ntdb: create initial database to be multiple of NTDB_PGSIZE.

As copied from tdb1, there is logic in the transaction code to handle
a non-PGSIZE multiple db, but in fact this only happens for a
completely unused database: as soon as we add anything to it, it is
expanded to a NTDB_PGSIZE multiple.

If we create the database with a free record which pads it out to
NTDB_PGSIZE, we can remove this last-page-is-different logic.

Of course, the fake ntdbs we create in our tests now also need to be
multiples of NTDB_PGSIZE, so we change some numbers there too.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

7 files changed, 58 insertions, 43 deletions

diff --git a/lib/ntdb/test/api-91-get-stats.c b/lib/ntdb/test/api-91-get-stats.c
index 786885b44c..e44b8a5061 100644
--- a/lib/ntdb/test/api-91-get-stats.c
+++ b/lib/ntdb/test/api-91-get-stats.c
@@ -19,12 +19,16 @@ int main(int argc, char *argv[])
 
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
 		union ntdb_attribute *attr;
-		NTDB_DATA key = ntdb_mkdata("key", 3);
+		NTDB_DATA key = ntdb_mkdata("key", 3), data;
 
 		ntdb = ntdb_open("run-91-get-stats.ntdb", flags[i],
 			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
 		ok1(ntdb);
-		ok1(ntdb_store(ntdb, key, key, NTDB_REPLACE) == 0);
+		/* Force an expansion */
+		data.dsize = 65536;
+		data.dptr = calloc(data.dsize, 1);
+		ok1(ntdb_store(ntdb, key, data, NTDB_REPLACE) == 0);
+		free(data.dptr);
 
 		/* Use malloc so valgrind will catch overruns. */
 		attr = malloc(sizeof *attr);
diff --git a/lib/ntdb/test/failtest_helper.h b/lib/ntdb/test/failtest_helper.h
index d347d75fba..5f9166d8d5 100644
--- a/lib/ntdb/test/failtest_helper.h
+++ b/lib/ntdb/test/failtest_helper.h
@@ -4,7 +4,7 @@
 #include <stdbool.h>
 
 /* FIXME: Check these! */
-#define INITIAL_NTDB_MALLOC	"open.c", 399, FAILTEST_MALLOC
+#define INITIAL_NTDB_MALLOC	"open.c", 425, FAILTEST_MALLOC
 #define URANDOM_OPEN		"open.c", 62, FAILTEST_OPEN
 #define URANDOM_READ		"open.c", 42, FAILTEST_READ
 
diff --git a/lib/ntdb/test/run-03-coalesce.c b/lib/ntdb/test/run-03-coalesce.c
index f93b33a1c3..22a6817881 100644
--- a/lib/ntdb/test/run-03-coalesce.c
+++ b/lib/ntdb/test/run-03-coalesce.c
@@ -34,7 +34,7 @@ int main(int argc, char *argv[])
 	/* No coalescing can be done due to EOF */
 	layout = new_ntdb_layout();
 	ntdb_layout_add_freetable(layout);
-	len = 1024;
+	len = 56544;
 	ntdb_layout_add_free(layout, len, 0);
 	ntdb_layout_write(layout, free, &tap_log_attr, "run-03-coalesce.ntdb");
 	/* NOMMAP is for lockcheck. */
@@ -60,24 +60,24 @@ int main(int argc, char *argv[])
 	/* No coalescing can be done due to used record */
 	layout = new_ntdb_layout();
 	ntdb_layout_add_freetable(layout);
-	ntdb_layout_add_free(layout, 1024, 0);
+	ntdb_layout_add_free(layout, 56512, 0);
 	ntdb_layout_add_used(layout, key, data, 6);
 	ntdb_layout_write(layout, free, &tap_log_attr, "run-03-coalesce.ntdb");
 	/* NOMMAP is for lockcheck. */
 	ntdb = ntdb_open("run-03-coalesce.ntdb", NTDB_NOMMAP, O_RDWR, 0,
 		       &tap_log_attr);
-	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 1024);
+	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 56512);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 
 	/* Figure out which bucket free entry is. */
-	b_off = bucket_off(ntdb->ftable_off, size_to_bucket(1024));
+	b_off = bucket_off(ntdb->ftable_off, size_to_bucket(56512));
 	/* Lock and fail to coalesce. */
 	ok1(ntdb_lock_free_bucket(ntdb, b_off, NTDB_LOCK_WAIT) == 0);
 	test = layout->elem[1].base.off;
-	ok1(coalesce(ntdb, layout->elem[1].base.off, b_off, 1024, &test)
+	ok1(coalesce(ntdb, layout->elem[1].base.off, b_off, 56512, &test)
 	    == 0);
 	ntdb_unlock_free_bucket(ntdb, b_off);
-	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 1024);
+	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 56512);
 	ok1(test == layout->elem[1].base.off);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 	ntdb_close(ntdb);
@@ -87,13 +87,13 @@ int main(int argc, char *argv[])
 	layout = new_ntdb_layout();
 	ntdb_layout_add_freetable(layout);
 	ntdb_layout_add_free(layout, 1024, 0);
-	ntdb_layout_add_free(layout, 2048, 0);
+	ntdb_layout_add_free(layout, 55504, 0);
 	ntdb_layout_write(layout, free, &tap_log_attr, "run-03-coalesce.ntdb");
 	/* NOMMAP is for lockcheck. */
 	ntdb = ntdb_open("run-03-coalesce.ntdb", NTDB_NOMMAP, O_RDWR, 0,
 		       &tap_log_attr);
 	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 1024);
-	ok1(free_record_length(ntdb, layout->elem[2].base.off) == 2048);
+	ok1(free_record_length(ntdb, layout->elem[2].base.off) == 55504);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 
 	/* Figure out which bucket (first) free entry is. */
@@ -102,12 +102,12 @@ int main(int argc, char *argv[])
 	ok1(ntdb_lock_free_bucket(ntdb, b_off, NTDB_LOCK_WAIT) == 0);
 	test = layout->elem[2].base.off;
 	ok1(coalesce(ntdb, layout->elem[1].base.off, b_off, 1024, &test)
-	    == 1024 + sizeof(struct ntdb_used_record) + 2048);
+	    == 1024 + sizeof(struct ntdb_used_record) + 55504);
 	/* Should tell us it's erased this one... */
 	ok1(test == NTDB_ERR_NOEXIST);
 	ok1(ntdb->file->allrecord_lock.count == 0 && ntdb->file->num_lockrecs == 0);
 	ok1(free_record_length(ntdb, layout->elem[1].base.off)
-	    == 1024 + sizeof(struct ntdb_used_record) + 2048);
+	    == 1024 + sizeof(struct ntdb_used_record) + 55504);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 	ntdb_close(ntdb);
 	ntdb_layout_free(layout);
@@ -116,14 +116,14 @@ int main(int argc, char *argv[])
 	layout = new_ntdb_layout();
 	ntdb_layout_add_freetable(layout);
 	ntdb_layout_add_free(layout, 1024, 0);
-	ntdb_layout_add_free(layout, 512, 0);
+	ntdb_layout_add_free(layout, 55472, 0);
 	ntdb_layout_add_used(layout, key, data, 6);
 	ntdb_layout_write(layout, free, &tap_log_attr, "run-03-coalesce.ntdb");
 	/* NOMMAP is for lockcheck. */
 	ntdb = ntdb_open("run-03-coalesce.ntdb", NTDB_NOMMAP, O_RDWR, 0,
 		       &tap_log_attr);
 	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 1024);
-	ok1(free_record_length(ntdb, layout->elem[2].base.off) == 512);
+	ok1(free_record_length(ntdb, layout->elem[2].base.off) == 55472);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 
 	/* Figure out which bucket free entry is. */
@@ -132,10 +132,10 @@ int main(int argc, char *argv[])
 	ok1(ntdb_lock_free_bucket(ntdb, b_off, NTDB_LOCK_WAIT) == 0);
 	test = layout->elem[2].base.off;
 	ok1(coalesce(ntdb, layout->elem[1].base.off, b_off, 1024, &test)
-	    == 1024 + sizeof(struct ntdb_used_record) + 512);
+	    == 1024 + sizeof(struct ntdb_used_record) + 55472);
 	ok1(ntdb->file->allrecord_lock.count == 0 && ntdb->file->num_lockrecs == 0);
 	ok1(free_record_length(ntdb, layout->elem[1].base.off)
-	    == 1024 + sizeof(struct ntdb_used_record) + 512);
+	    == 1024 + sizeof(struct ntdb_used_record) + 55472);
 	ok1(test == NTDB_ERR_NOEXIST);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 	ntdb_close(ntdb);
@@ -146,14 +146,14 @@ int main(int argc, char *argv[])
 	ntdb_layout_add_freetable(layout);
 	ntdb_layout_add_free(layout, 1024, 0);
 	ntdb_layout_add_free(layout, 512, 0);
-	ntdb_layout_add_free(layout, 256, 0);
+	ntdb_layout_add_free(layout, 54976, 0);
 	ntdb_layout_write(layout, free, &tap_log_attr, "run-03-coalesce.ntdb");
 	/* NOMMAP is for lockcheck. */
 	ntdb = ntdb_open("run-03-coalesce.ntdb", NTDB_NOMMAP, O_RDWR, 0,
 		       &tap_log_attr);
 	ok1(free_record_length(ntdb, layout->elem[1].base.off) == 1024);
 	ok1(free_record_length(ntdb, layout->elem[2].base.off) == 512);
-	ok1(free_record_length(ntdb, layout->elem[3].base.off) == 256);
+	ok1(free_record_length(ntdb, layout->elem[3].base.off) == 54976);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 
 	/* Figure out which bucket free entry is. */
@@ -163,12 +163,12 @@ int main(int argc, char *argv[])
 	test = layout->elem[2].base.off;
 	ok1(coalesce(ntdb, layout->elem[1].base.off, b_off, 1024, &test)
 	    == 1024 + sizeof(struct ntdb_used_record) + 512
-	    + sizeof(struct ntdb_used_record) + 256);
+	    + sizeof(struct ntdb_used_record) + 54976);
 	ok1(ntdb->file->allrecord_lock.count == 0
 	    && ntdb->file->num_lockrecs == 0);
 	ok1(free_record_length(ntdb, layout->elem[1].base.off)
 	    == 1024 + sizeof(struct ntdb_used_record) + 512
-	    + sizeof(struct ntdb_used_record) + 256);
+	    + sizeof(struct ntdb_used_record) + 54976);
 	ok1(ntdb_check(ntdb, NULL, NULL) == 0);
 	ntdb_close(ntdb);
 	ntdb_layout_free(layout);
diff --git a/lib/ntdb/test/run-30-exhaust-before-expand.c b/lib/ntdb/test/run-30-exhaust-before-expand.c
index e3831f51a7..24c48b005a 100644
--- a/lib/ntdb/test/run-30-exhaust-before-expand.c
+++ b/lib/ntdb/test/run-30-exhaust-before-expand.c
@@ -27,11 +27,11 @@ int main(int argc, char *argv[])
 			NTDB_INTERNAL|NTDB_CONVERT, NTDB_CONVERT,
 			NTDB_NOMMAP|NTDB_CONVERT };
 
-	plan_tests(sizeof(flags) / sizeof(flags[0]) * 11 + 1);
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 7 + 1);
 
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
 		NTDB_DATA k, d;
-		uint64_t size, old_size;
+		uint64_t size;
 		bool was_empty = false;
 
 		k.dptr = (void *)&j;
@@ -43,22 +43,15 @@ int main(int argc, char *argv[])
 		if (!ntdb)
 			continue;
 
-		old_size = ntdb->file->map_size;
-
-		ok1(empty_freetable(ntdb));
-		/* Need some hash lock for expand. */
-		ok1(ntdb_lock_hashes(ntdb, 0, 1, F_WRLCK, NTDB_LOCK_WAIT) == 0);
-		/* Create some free space. */
-		ok1(ntdb_expand(ntdb, 1) == 0);
-		ok1(ntdb_unlock_hashes(ntdb, 0, 1, F_WRLCK) == 0);
 		ok1(ntdb_check(ntdb, NULL, NULL) == 0);
+		/* There's one empty record in initial db. */
 		ok1(!empty_freetable(ntdb));
 
 		size = ntdb->file->map_size;
 
 		/* Create one record to chew up most space. */
-		d.dsize = (size - old_size - 32);
-		d.dptr = malloc(d.dsize);
+		d.dsize = size - sizeof(struct new_database) - 32;
+		d.dptr = calloc(d.dsize, 1);
 		j = 0;
 		ok1(ntdb_store(ntdb, k, d, NTDB_INSERT) == 0);
 		ok1(ntdb->file->map_size == size);
diff --git a/lib/ntdb/test/run-64-bit-tdb.c b/lib/ntdb/test/run-64-bit-tdb.c
index b36f422a97..a85a4af56c 100644
--- a/lib/ntdb/test/run-64-bit-tdb.c
+++ b/lib/ntdb/test/run-64-bit-tdb.c
@@ -21,7 +21,7 @@ int main(int argc, char *argv[])
 		return exit_status();
 	}
 
-	plan_tests(sizeof(flags) / sizeof(flags[0]) * 14);
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 16);
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
 		off_t old_size;
 		NTDB_DATA k, d;
@@ -37,6 +37,15 @@ int main(int argc, char *argv[])
 
 		old_size = ntdb->file->map_size;
 
+		/* Add a fake record to chew up the existing free space. */
+		k = ntdb_mkdata("fake", 4);
+		d.dsize = ntdb->file->map_size - sizeof(struct new_database)- 8;
+		d.dptr = malloc(d.dsize);
+		memset(d.dptr, 0, d.dsize);
+		ok1(ntdb_store(ntdb, k, d, NTDB_INSERT) == 0);
+		ok1(ntdb->file->map_size == old_size);
+		free(d.dptr);
+
 		/* This makes a sparse file */
 		ok1(ftruncate(ntdb->file->fd, ALMOST_4G) == 0);
 		ok1(add_free_record(ntdb, old_size, ALMOST_4G - old_size,
diff --git a/lib/ntdb/test/run-capabilities.c b/lib/ntdb/test/run-capabilities.c
index c2c6aa15db..52c4fac0e6 100644
--- a/lib/ntdb/test/run-capabilities.c
+++ b/lib/ntdb/test/run-capabilities.c
@@ -29,7 +29,7 @@ static void create_ntdb(const char *name,
 	va_list ap;
 	struct ntdb_layout *layout;
 	struct ntdb_context *ntdb;
-	int fd;
+	int fd, clen;
 
 	key = ntdb_mkdata("Hello", 5);
 	data = ntdb_mkdata("world", 5);
@@ -38,10 +38,11 @@ static void create_ntdb(const char *name,
 	layout = new_ntdb_layout();
 	ntdb_layout_add_freetable(layout);
 	ntdb_layout_add_used(layout, key, data, 6);
-	ntdb_layout_add_free(layout, 80, 0);
+	clen = len_of(breaks_check, breaks_write, breaks_open);
+	ntdb_layout_add_free(layout, 56480 - clen, 0);
 	ntdb_layout_add_capability(layout, cap,
-				  breaks_write, breaks_check, breaks_open,
-				  len_of(breaks_check, breaks_write, breaks_open));
+				   breaks_write, breaks_check, breaks_open,
+				   clen);
 
 	va_start(ap, breaks_open);
 	while ((cap = va_arg(ap, int)) != 0) {
@@ -51,12 +52,11 @@ static void create_ntdb(const char *name,
 
 		key.dsize--;
 		ntdb_layout_add_used(layout, key, data, 11 - key.dsize);
-		ntdb_layout_add_free(layout, 80, 0);
+		clen = len_of(breaks_check, breaks_write, breaks_open);
+		ntdb_layout_add_free(layout, 65456 - clen, 0);
 		ntdb_layout_add_capability(layout, cap,
 					  breaks_write, breaks_check,
-					  breaks_open,
-					  len_of(breaks_check, breaks_write,
-						 breaks_open));
+					  breaks_open, clen);
 	}
 	va_end(ap);
 
diff --git a/lib/ntdb/test/run-expand-in-transaction.c b/lib/ntdb/test/run-expand-in-transaction.c
index dadbec7922..54f9d81dec 100644
--- a/lib/ntdb/test/run-expand-in-transaction.c
+++ b/lib/ntdb/test/run-expand-in-transaction.c
@@ -11,10 +11,11 @@ int main(int argc, char *argv[])
 	NTDB_DATA key = ntdb_mkdata("key", 3);
 	NTDB_DATA data = ntdb_mkdata("data", 4);
 
-	plan_tests(sizeof(flags) / sizeof(flags[0]) * 7 + 1);
+	plan_tests(sizeof(flags) / sizeof(flags[0]) * 9 + 1);
 
 	for (i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
 		size_t size;
+		NTDB_DATA k, d;
 		ntdb = ntdb_open("run-expand-in-transaction.ntdb", flags[i],
 			       O_RDWR|O_CREAT|O_TRUNC, 0600, &tap_log_attr);
 		ok1(ntdb);
@@ -22,6 +23,14 @@ int main(int argc, char *argv[])
 			continue;
 
 		size = ntdb->file->map_size;
+		/* Add a fake record to chew up the existing free space. */
+		k = ntdb_mkdata("fake", 4);
+		d.dsize = ntdb->file->map_size - sizeof(struct new_database)- 8;
+		d.dptr = malloc(d.dsize);
+		memset(d.dptr, 0, d.dsize);
+		ok1(ntdb_store(ntdb, k, d, NTDB_INSERT) == 0);
+		ok1(ntdb->file->map_size == size);
+		free(d.dptr);
 		ok1(ntdb_transaction_start(ntdb) == 0);
 		ok1(ntdb_store(ntdb, key, data, NTDB_INSERT) == 0);
 		ok1(ntdb->file->map_size > size);