r19685: Two changes inspired by problems with huge tdbs. tdbtool's list command now

prints the hash on every record for easier awk'ing, and tdbbackup allows a
different hash chain length on the backed up tdb.

Jeremy, Günther, this might be interesting for you huge domains. Not only
locking.tdb, also the winbind ones might grow huge.

In the installation I fixed with this winbind spent a huge amount of CPU
spinning through a degenerated winbindd_idmap.tdb with entries for more than
15.000 users. With a default number of hash chains of 131 on that tdb you can
imagine that the lists get large.

Not merging to 4, I don't get tdbbackup to compile there right now.

What about changing the global default hash chain number to be dramatically
larger? Disk is cheap these days.

Volker
(This used to be commit 577d0ff658596f8246f120e0342cc5c9e4077ece)

4 files changed, 19 insertions, 10 deletions

diff --git a/source3/tdb/common/dump.c b/source3/tdb/common/dump.c
index 577f23aac6..70382ca949 100644
--- a/source3/tdb/common/dump.c
+++ b/source3/tdb/common/dump.c
@@ -28,7 +28,8 @@
 
 #include "tdb_private.h"
 
-static tdb_off_t tdb_dump_record(struct tdb_context *tdb, tdb_off_t offset)
+static tdb_off_t tdb_dump_record(struct tdb_context *tdb, int hash,
+				 tdb_off_t offset)
 {
 	struct list_struct rec;
 	tdb_off_t tailer_ofs, tailer;
@@ -39,8 +40,10 @@ static tdb_off_t tdb_dump_record(struct tdb_context *tdb, tdb_off_t offset)
 		return 0;
 	}
 
-	printf(" rec: offset=0x%08x next=0x%08x rec_len=%d key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
-	       offset, rec.next, rec.rec_len, rec.key_len, rec.data_len, rec.full_hash, rec.magic);
+	printf(" rec: hash=%d, offset=0x%08x next=0x%08x rec_len=%d "
+	       "key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
+	       hash, offset, rec.next, rec.rec_len, rec.key_len, rec.data_len,
+	       rec.full_hash, rec.magic);
 
 	tailer_ofs = offset + sizeof(rec) + rec.rec_len - sizeof(tdb_off_t);
 
@@ -72,7 +75,7 @@ static int tdb_dump_chain(struct tdb_context *tdb, int i)
 		printf("hash=%d\n", i);
 
 	while (rec_ptr) {
-		rec_ptr = tdb_dump_record(tdb, rec_ptr);
+		rec_ptr = tdb_dump_record(tdb, i, rec_ptr);
 	}
 
 	return tdb_unlock(tdb, i, F_WRLCK);
diff --git a/source3/tdb/common/tdbback.c b/source3/tdb/common/tdbback.c
index 4698e6016b..28de85c29f 100644
--- a/source3/tdb/common/tdbback.c
+++ b/source3/tdb/common/tdbback.c
@@ -95,7 +95,7 @@ static int test_fn(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
   only doing the backup if its OK
   this function is also used for restore
 */
-int backup_tdb(const char *old_name, const char *new_name)
+int backup_tdb(const char *old_name, const char *new_name, int hash_size)
 {
 	TDB_CONTEXT *tdb;
 	TDB_CONTEXT *tdb_new;
@@ -122,7 +122,8 @@ int backup_tdb(const char *old_name, const char *new_name)
 
 	/* create the new tdb */
 	unlink(tmp_name);
-	tdb_new = tdb_open(tmp_name, tdb_hash_size(tdb),
+	tdb_new = tdb_open(tmp_name,
+			   hash_size ? hash_size : tdb_hash_size(tdb),
 			   TDB_DEFAULT, O_RDWR|O_CREAT|O_EXCL, 
 			   st.st_mode & 0777);
 	if (!tdb_new) {
@@ -217,7 +218,7 @@ int verify_tdb(const char *fname, const char *bak_name)
 	/* count is < 0 means an error */
 	if (count < 0) {
 		printf("restoring %s\n", fname);
-		return backup_tdb(bak_name, fname);
+		return backup_tdb(bak_name, fname, 0);
 	}
 
 	printf("%s : %d records\n", fname, count);
diff --git a/source3/tdb/include/tdbback.h b/source3/tdb/include/tdbback.h
index 7ebeaa494d..69d3bb6f91 100644
--- a/source3/tdb/include/tdbback.h
+++ b/source3/tdb/include/tdbback.h
@@ -19,5 +19,5 @@
 */
 
 char *add_suffix(const char *name, const char *suffix);
-int backup_tdb(const char *old_name, const char *new_name);
+int backup_tdb(const char *old_name, const char *new_name, int hash_size);
 int verify_tdb(const char *fname, const char *bak_name);
diff --git a/source3/tdb/tools/tdbbackup.c b/source3/tdb/tools/tdbbackup.c
index f49cd339c7..4ae5a4f921 100644
--- a/source3/tdb/tools/tdbbackup.c
+++ b/source3/tdb/tools/tdbbackup.c
@@ -93,6 +93,7 @@ static void usage(void)
 	printf("   -h            this help message\n");
 	printf("   -s suffix     set the backup suffix\n");
 	printf("   -v            verify mode (restore if corrupt)\n");
+	printf("   -n hashsize   set the new hash size for the backup\n");
 }
 		
 
@@ -102,9 +103,10 @@ static void usage(void)
 	int ret = 0;
 	int c;
 	int verify = 0;
+	int hashsize = 0;
 	const char *suffix = ".bak";
 
-	while ((c = getopt(argc, argv, "vhs:")) != -1) {
+	while ((c = getopt(argc, argv, "vhs:n:")) != -1) {
 		switch (c) {
 		case 'h':
 			usage();
@@ -115,6 +117,9 @@ static void usage(void)
 		case 's':
 			suffix = optarg;
 			break;
+		case 'n':
+			hashsize = atoi(optarg);
+			break;
 		}
 	}
 
@@ -138,7 +143,7 @@ static void usage(void)
 			}
 		} else {
 			if (file_newer(fname, bak_name) &&
-			    backup_tdb(fname, bak_name) != 0) {
+			    backup_tdb(fname, bak_name, hashsize) != 0) {
 				ret = 1;
 			}
 		}