TDB2: Goodbye TDB2, Hello NTDB.

This renames everything from tdb2 to ntdb: importantly, we no longer
use the tdb_ namespace, so you can link against both ntdb and tdb if
you want to.

This also enables building of standalone ntdb by the autobuild script.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

9 files changed, 2634 insertions, 0 deletions

diff --git a/lib/ntdb/tools/Makefile b/lib/ntdb/tools/Makefile
new file mode 100644
index 0000000000..087c256d7f
--- /dev/null
+++ b/lib/ntdb/tools/Makefile
@@ -0,0 +1,16 @@
+OBJS:=../../ntdb.o ../../hash.o ../../tally.o
+CFLAGS:=-I../../.. -I.. -Wall -g -O3 #-g -pg
+LDFLAGS:=-L../../..
+
+default: ntdbtorture ntdbtool ntdbdump ntdbrestore mkntdb speed growtdb-bench
+
+ntdbdump: ntdbdump.c $(OBJS)
+ntdbrestore: ntdbrestore.c $(OBJS)
+ntdbtorture: ntdbtorture.c $(OBJS)
+ntdbtool: ntdbtool.c $(OBJS)
+mkntdb: mkntdb.c $(OBJS)
+speed: speed.c $(OBJS)
+growtdb-bench: growtdb-bench.c $(OBJS)
+
+clean:
+	rm -f ntdbtorture ntdbdump ntdbrestore ntdbtool mkntdb speed growtdb-bench
diff --git a/lib/ntdb/tools/growtdb-bench.c b/lib/ntdb/tools/growtdb-bench.c
new file mode 100644
index 0000000000..640f87af5a
--- /dev/null
+++ b/lib/ntdb/tools/growtdb-bench.c
@@ -0,0 +1,114 @@
+#include "ntdb.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <ccan/err/err.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+static void logfn(struct ntdb_context *ntdb,
+		  enum ntdb_log_level level,
+		  enum NTDB_ERROR ecode,
+		  const char *message,
+		  void *data)
+{
+	fprintf(stderr, "ntdb:%s:%s:%s\n",
+		ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, users, groups;
+	NTDB_DATA idxkey, idxdata;
+	NTDB_DATA k, d, gk;
+	char cmd[100];
+	struct ntdb_context *ntdb;
+	enum NTDB_ERROR ecode;
+	union ntdb_attribute log;
+
+	if (argc != 3) {
+		printf("Usage: growtdb-bench <users> <groups>\n");
+		exit(1);
+	}
+	users = atoi(argv[1]);
+	groups = atoi(argv[2]);
+
+	sprintf(cmd, "cat /proc/%i/statm", getpid());
+
+	log.base.attr = NTDB_ATTRIBUTE_LOG;
+	log.base.next = NULL;
+	log.log.fn = logfn;
+
+	ntdb = ntdb_open("/tmp/growtdb.ntdb", NTDB_DEFAULT,
+		       O_RDWR|O_CREAT|O_TRUNC, 0600, &log);
+
+	idxkey.dptr = (unsigned char *)"User index";
+	idxkey.dsize = strlen("User index");
+	idxdata.dsize = 51;
+	idxdata.dptr = calloc(idxdata.dsize, 1);
+
+	/* Create users. */
+	k.dsize = 48;
+	k.dptr = calloc(k.dsize, 1);
+	d.dsize = 64;
+	d.dptr = calloc(d.dsize, 1);
+
+	ntdb_transaction_start(ntdb);
+	for (i = 0; i < users; i++) {
+		memcpy(k.dptr, &i, sizeof(i));
+		ecode = ntdb_store(ntdb, k, d, NTDB_INSERT);
+		if (ecode != NTDB_SUCCESS)
+			errx(1, "ntdb insert failed: %s", ntdb_errorstr(ecode));
+
+		/* This simulates a growing index record. */
+		ecode = ntdb_append(ntdb, idxkey, idxdata);
+		if (ecode != NTDB_SUCCESS)
+			errx(1, "ntdb append failed: %s", ntdb_errorstr(ecode));
+	}
+	if ((ecode = ntdb_transaction_commit(ntdb)) != 0)
+		errx(1, "ntdb commit1 failed: %s", ntdb_errorstr(ecode));
+
+	if ((ecode = ntdb_check(ntdb, NULL, NULL)) != 0)
+		errx(1, "ntdb_check failed after initial insert!");
+
+	system(cmd);
+
+	/* Now put them all in groups: add 32 bytes to each record for
+	 * a group. */
+	gk.dsize = 48;
+	gk.dptr = calloc(k.dsize, 1);
+	gk.dptr[gk.dsize-1] = 1;
+
+	d.dsize = 32;
+	for (i = 0; i < groups; i++) {
+		ntdb_transaction_start(ntdb);
+		/* Create the "group". */
+		memcpy(gk.dptr, &i, sizeof(i));
+		ecode = ntdb_store(ntdb, gk, d, NTDB_INSERT);
+		if (ecode != NTDB_SUCCESS)
+			errx(1, "ntdb insert failed: %s", ntdb_errorstr(ecode));
+
+		/* Now populate it. */
+		for (j = 0; j < users; j++) {
+			/* Append to the user. */
+			memcpy(k.dptr, &j, sizeof(j));
+			if ((ecode = ntdb_append(ntdb, k, d)) != 0)
+				errx(1, "ntdb append failed: %s",
+				     ntdb_errorstr(ecode));
+
+			/* Append to the group. */
+			if ((ecode = ntdb_append(ntdb, gk, d)) != 0)
+				errx(1, "ntdb append failed: %s",
+				     ntdb_errorstr(ecode));
+		}
+		if ((ecode = ntdb_transaction_commit(ntdb)) != 0)
+			errx(1, "ntdb commit2 failed: %s", ntdb_errorstr(ecode));
+		if ((ecode = ntdb_check(ntdb, NULL, NULL)) != 0)
+			errx(1, "ntdb_check failed after iteration %i!", i);
+		system(cmd);
+	}
+
+	return 0;
+}
diff --git a/lib/ntdb/tools/mkntdb.c b/lib/ntdb/tools/mkntdb.c
new file mode 100644
index 0000000000..e728987a53
--- /dev/null
+++ b/lib/ntdb/tools/mkntdb.c
@@ -0,0 +1,29 @@
+#include "ntdb.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <ccan/err/err.h>
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, num_recs;
+	struct ntdb_context *ntdb;
+
+	if (argc != 3 || (num_recs = atoi(argv[2])) == 0)
+		errx(1, "Usage: mktdb <tdbfile> <numrecords>");
+
+	ntdb = ntdb_open(argv[1], NTDB_DEFAULT, O_CREAT|O_TRUNC|O_RDWR, 0600,NULL);
+	if (!ntdb)
+		err(1, "Opening %s", argv[1]);
+
+	for (i = 0; i < num_recs; i++) {
+		NTDB_DATA d;
+
+		d.dptr = (void *)&i;
+		d.dsize = sizeof(i);
+		if (ntdb_store(ntdb, d, d, NTDB_INSERT) != 0)
+			err(1, "Failed to store record %i", i);
+	}
+	printf("Done\n");
+	return 0;
+}
diff --git a/lib/ntdb/tools/ntdbbackup.c b/lib/ntdb/tools/ntdbbackup.c
new file mode 100644
index 0000000000..a76f18491b
--- /dev/null
+++ b/lib/ntdb/tools/ntdbbackup.c
@@ -0,0 +1,340 @@
+/*
+   Unix SMB/CIFS implementation.
+   low level ntdb backup and restore utility
+   Copyright (C) Andrew Tridgell              2002
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+
+  This program is meant for backup/restore of ntdb databases. Typical usage would be:
+     tdbbackup *.ntdb
+  when Samba shuts down cleanly, which will make a backup of all the local databases
+  to *.bak files. Then on Samba startup you would use:
+     tdbbackup -v *.ntdb
+  and this will check the databases for corruption and if corruption is detected then
+  the backup will be restored.
+
+  You may also like to do a backup on a regular basis while Samba is
+  running, perhaps using cron.
+
+  The reason this program is needed is to cope with power failures
+  while Samba is running. A power failure could lead to database
+  corruption and Samba will then not start correctly.
+
+  Note that many of the databases in Samba are transient and thus
+  don't need to be backed up, so you can optimise the above a little
+  by only running the backup on the critical databases.
+
+ */
+
+#include "config.h"
+#include "ntdb.h"
+#include "system/filesys.h"
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+static int failed;
+
+static void ntdb_log(struct ntdb_context *ntdb,
+		    enum ntdb_log_level level,
+		    enum NTDB_ERROR ecode,
+		    const char *message,
+		    void *data)
+{
+	fprintf(stderr, "%s:%s\n", ntdb_errorstr(ecode), message);
+}
+
+static char *add_suffix(const char *name, const char *suffix)
+{
+	char *ret;
+	int len = strlen(name) + strlen(suffix) + 1;
+	ret = (char *)malloc(len);
+	if (!ret) {
+		fprintf(stderr,"Out of memory!\n");
+		exit(1);
+	}
+	snprintf(ret, len, "%s%s", name, suffix);
+	return ret;
+}
+
+static int copy_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	struct ntdb_context *ntdb_new = (struct ntdb_context *)state;
+	enum NTDB_ERROR err;
+
+	err = ntdb_store(ntdb_new, key, dbuf, NTDB_INSERT);
+	if (err) {
+		fprintf(stderr,"Failed to insert into %s: %s\n",
+			ntdb_name(ntdb_new), ntdb_errorstr(err));
+		failed = 1;
+		return 1;
+	}
+	return 0;
+}
+
+
+static int test_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	return 0;
+}
+
+/*
+  carefully backup a ntdb, validating the contents and
+  only doing the backup if its OK
+  this function is also used for restore
+*/
+static int backup_ntdb(const char *old_name, const char *new_name)
+{
+	struct ntdb_context *ntdb;
+	struct ntdb_context *ntdb_new;
+	char *tmp_name;
+	struct stat st;
+	int count1, count2;
+	enum NTDB_ERROR err;
+	union ntdb_attribute log_attr;
+
+	tmp_name = add_suffix(new_name, ".tmp");
+
+	/* stat the old ntdb to find its permissions */
+	if (stat(old_name, &st) != 0) {
+		perror(old_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+	log_attr.base.next = NULL;
+	log_attr.log.fn = ntdb_log;
+
+	/* open the old ntdb */
+	ntdb = ntdb_open(old_name, NTDB_DEFAULT, O_RDWR, 0, &log_attr);
+	if (!ntdb) {
+		printf("Failed to open %s\n", old_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	unlink(tmp_name);
+	ntdb_new = ntdb_open(tmp_name, NTDB_DEFAULT,
+			   O_RDWR|O_CREAT|O_EXCL, st.st_mode & 0777,
+			   &log_attr);
+	if (!ntdb_new) {
+		perror(tmp_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	err = ntdb_transaction_start(ntdb);
+	if (err) {
+		fprintf(stderr, "Failed to start transaction on old ntdb: %s\n",
+			ntdb_errorstr(err));
+		ntdb_close(ntdb);
+		ntdb_close(ntdb_new);
+		unlink(tmp_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	/* lock the backup ntdb so that nobody else can change it */
+	err = ntdb_lockall(ntdb_new);
+	if (err) {
+		fprintf(stderr, "Failed to lock backup ntdb: %s\n",
+			ntdb_errorstr(err));
+		ntdb_close(ntdb);
+		ntdb_close(ntdb_new);
+		unlink(tmp_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	failed = 0;
+
+	/* traverse and copy */
+	count1 = ntdb_traverse(ntdb, copy_fn, (void *)ntdb_new);
+	if (count1 < 0 || failed) {
+		fprintf(stderr,"failed to copy %s\n", old_name);
+		ntdb_close(ntdb);
+		ntdb_close(ntdb_new);
+		unlink(tmp_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	/* close the old ntdb */
+	ntdb_close(ntdb);
+
+	/* copy done, unlock the backup ntdb */
+	ntdb_unlockall(ntdb_new);
+
+#ifdef HAVE_FDATASYNC
+	if (fdatasync(ntdb_fd(ntdb_new)) != 0) {
+#else
+	if (fsync(ntdb_fd(ntdb_new)) != 0) {
+#endif
+		/* not fatal */
+		fprintf(stderr, "failed to fsync backup file\n");
+	}
+
+	/* close the new ntdb and re-open read-only */
+	ntdb_close(ntdb_new);
+
+	/* we don't need the hash attr any more */
+	log_attr.base.next = NULL;
+
+	ntdb_new = ntdb_open(tmp_name, NTDB_DEFAULT, O_RDONLY, 0, &log_attr);
+	if (!ntdb_new) {
+		fprintf(stderr,"failed to reopen %s\n", tmp_name);
+		unlink(tmp_name);
+		perror(tmp_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	/* traverse the new ntdb to confirm */
+	count2 = ntdb_traverse(ntdb_new, test_fn, NULL);
+	if (count2 != count1) {
+		fprintf(stderr,"failed to copy %s\n", old_name);
+		ntdb_close(ntdb_new);
+		unlink(tmp_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	/* close the new ntdb and rename it to .bak */
+	ntdb_close(ntdb_new);
+	if (rename(tmp_name, new_name) != 0) {
+		perror(new_name);
+		free(tmp_name);
+		return 1;
+	}
+
+	free(tmp_name);
+
+	return 0;
+}
+
+/*
+  verify a ntdb and if it is corrupt then restore from *.bak
+*/
+static int verify_ntdb(const char *fname, const char *bak_name)
+{
+	struct ntdb_context *ntdb;
+	int count = -1;
+	union ntdb_attribute log_attr;
+
+	log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+	log_attr.base.next = NULL;
+	log_attr.log.fn = ntdb_log;
+
+	/* open the ntdb */
+	ntdb = ntdb_open(fname, NTDB_DEFAULT, O_RDONLY, 0, &log_attr);
+
+	/* traverse the ntdb, then close it */
+	if (ntdb) {
+		count = ntdb_traverse(ntdb, test_fn, NULL);
+		ntdb_close(ntdb);
+	}
+
+	/* count is < 0 means an error */
+	if (count < 0) {
+		printf("restoring %s\n", fname);
+		return backup_ntdb(bak_name, fname);
+	}
+
+	printf("%s : %d records\n", fname, count);
+
+	return 0;
+}
+
+/*
+  see if one file is newer than another
+*/
+static int file_newer(const char *fname1, const char *fname2)
+{
+	struct stat st1, st2;
+	if (stat(fname1, &st1) != 0) {
+		return 0;
+	}
+	if (stat(fname2, &st2) != 0) {
+		return 1;
+	}
+	return (st1.st_mtime > st2.st_mtime);
+}
+
+static void usage(void)
+{
+	printf("Usage: ntdbbackup [options] <fname...>\n\n");
+	printf("   -h            this help message\n");
+	printf("   -v            verify mode (restore if corrupt)\n");
+	printf("   -s suffix     set the backup suffix\n");
+	printf("   -v            verify mode (restore if corrupt)\n");
+}
+
+
+ int main(int argc, char *argv[])
+{
+	int i;
+	int ret = 0;
+	int c;
+	int verify = 0;
+	const char *suffix = ".bak";
+
+	while ((c = getopt(argc, argv, "vhs:")) != -1) {
+		switch (c) {
+		case 'h':
+			usage();
+			exit(0);
+		case 'v':
+			verify = 1;
+			break;
+		case 's':
+			suffix = optarg;
+			break;
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 1) {
+		usage();
+		exit(1);
+	}
+
+	for (i=0; i<argc; i++) {
+		const char *fname = argv[i];
+		char *bak_name;
+
+		bak_name = add_suffix(fname, suffix);
+
+		if (verify) {
+			if (verify_ntdb(fname, bak_name) != 0) {
+				ret = 1;
+			}
+		} else {
+			if (file_newer(fname, bak_name) &&
+			    backup_ntdb(fname, bak_name) != 0) {
+				ret = 1;
+			}
+		}
+
+		free(bak_name);
+	}
+
+	return ret;
+}
diff --git a/lib/ntdb/tools/ntdbdump.c b/lib/ntdb/tools/ntdbdump.c
new file mode 100644
index 0000000000..1b1c59eae3
--- /dev/null
+++ b/lib/ntdb/tools/ntdbdump.c
@@ -0,0 +1,122 @@
+/*
+   simple ntdb dump util
+   Copyright (C) Andrew Tridgell              2001
+   Copyright (C) Rusty Russell                2011
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "config.h"
+#include "ntdb.h"
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#include <system/filesys.h>
+#include <system/locale.h>
+#else
+#include <ctype.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+#endif
+
+static void print_data(NTDB_DATA d)
+{
+	unsigned char *p = (unsigned char *)d.dptr;
+	int len = d.dsize;
+	while (len--) {
+		if (isprint(*p) && !strchr("\"\\", *p)) {
+			fputc(*p, stdout);
+		} else {
+			printf("\\%02X", *p);
+		}
+		p++;
+	}
+}
+
+static int traverse_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	printf("{\n");
+	printf("key(%d) = \"", (int)key.dsize);
+	print_data(key);
+	printf("\"\n");
+	printf("data(%d) = \"", (int)dbuf.dsize);
+	print_data(dbuf);
+	printf("\"\n");
+	printf("}\n");
+	return 0;
+}
+
+static int dump_ntdb(const char *fname, const char *keyname)
+{
+	struct ntdb_context *ntdb;
+	NTDB_DATA key, value;
+
+	ntdb = ntdb_open(fname, 0, O_RDONLY, 0, NULL);
+	if (!ntdb) {
+		printf("Failed to open %s\n", fname);
+		return 1;
+	}
+
+	if (!keyname) {
+		ntdb_traverse(ntdb, traverse_fn, NULL);
+	} else {
+		key = ntdb_mkdata(keyname, strlen(keyname));
+		if (ntdb_fetch(ntdb, key, &value) != 0) {
+			return 1;
+		} else {
+			print_data(value);
+			free(value.dptr);
+		}
+	}
+
+	return 0;
+}
+
+static void usage( void)
+{
+	printf( "Usage: ntdbdump [options] <filename>\n\n");
+	printf( "   -h          this help message\n");
+	printf( "   -k keyname  dumps value of keyname\n");
+}
+
+ int main(int argc, char *argv[])
+{
+	char *fname, *keyname=NULL;
+	int c;
+
+	if (argc < 2) {
+		printf("Usage: ntdbdump <fname>\n");
+		exit(1);
+	}
+
+	while ((c = getopt( argc, argv, "hk:")) != -1) {
+		switch (c) {
+		case 'h':
+			usage();
+			exit( 0);
+		case 'k':
+			keyname = optarg;
+			break;
+		default:
+			usage();
+			exit( 1);
+		}
+	}
+
+	fname = argv[optind];
+
+	return dump_ntdb(fname, keyname);
+}
diff --git a/lib/ntdb/tools/ntdbrestore.c b/lib/ntdb/tools/ntdbrestore.c
new file mode 100644
index 0000000000..dad591d562
--- /dev/null
+++ b/lib/ntdb/tools/ntdbrestore.c
@@ -0,0 +1,231 @@
+/*
+   ntdbrestore -- construct a ntdb from tdbdump output.
+   Copyright (C) Volker Lendecke		2010
+   Copyright (C) Simon McVittie			2005
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+#include "ntdb.h"
+#include <assert.h>
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#include <system/filesys.h>
+#else
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#endif
+
+static int read_linehead(FILE *f)
+{
+	int i, c;
+	int num_bytes;
+	char prefix[128];
+
+	while (1) {
+		c = getc(f);
+		if (c == EOF) {
+			return -1;
+		}
+		if (c == '(') {
+			break;
+		}
+	}
+	for (i=0; i<sizeof(prefix); i++) {
+		c = getc(f);
+		if (c == EOF) {
+			return -1;
+		}
+		prefix[i] = c;
+		if (c == '"') {
+			break;
+		}
+	}
+	if (i == sizeof(prefix)) {
+		return -1;
+	}
+	prefix[i] = '\0';
+
+	if (sscanf(prefix, "%d) = ", &num_bytes) != 1) {
+		return -1;
+	}
+	return num_bytes;
+}
+
+static int read_hex(void) {
+	int c;
+	c = getchar();
+	if (c == EOF) {
+		fprintf(stderr, "Unexpected EOF in data\n");
+		return -1;
+	} else if (c == '"') {
+		fprintf(stderr, "Unexpected \\\" sequence\n");
+		return -1;
+	} else if ('0' <= c && c <= '9')  {
+		return c - '0';
+	} else if ('A' <= c && c <= 'F')  {
+		return c - 'A' + 10;
+	} else if ('a' <= c && c <= 'f')  {
+		return c - 'a' + 10;
+	} else {
+		fprintf(stderr, "Invalid hex: %c\n", c);
+		return -1;
+	}
+}
+
+static int read_data(FILE *f, NTDB_DATA *d, size_t size) {
+	int c, low, high;
+	int i;
+
+	d->dptr = (unsigned char *)malloc(size);
+	if (d->dptr == NULL) {
+		return -1;
+	}
+	d->dsize = size;
+
+	for (i=0; i<size; i++) {
+		c = getc(f);
+		if (c == EOF) {
+			fprintf(stderr, "Unexpected EOF in data\n");
+			return 1;
+		} else if (c == '"') {
+			return 0;
+		} else if (c == '\\') {
+			high = read_hex();
+			if (high < 0) {
+				return -1;
+			}
+			high = high << 4;
+			assert(high == (high & 0xf0));
+			low = read_hex();
+			if (low < 0) {
+				return -1;
+			}
+			assert(low == (low & 0x0f));
+			d->dptr[i] = (low|high);
+		} else {
+			d->dptr[i] = c;
+		}
+	}
+	return 0;
+}
+
+static int swallow(FILE *f, const char *s, int *eof)
+{
+	char line[128];
+
+	if (fgets(line, sizeof(line), f) == NULL) {
+		if (eof != NULL) {
+			*eof = 1;
+		}
+		return -1;
+	}
+	if (strcmp(line, s) != 0) {
+		return -1;
+	}
+	return 0;
+}
+
+static bool read_rec(FILE *f, struct ntdb_context *ntdb, int *eof)
+{
+	int length;
+	NTDB_DATA key, data;
+	bool ret = false;
+	enum NTDB_ERROR e;
+
+	key.dptr = NULL;
+	data.dptr = NULL;
+
+	if (swallow(f, "{\n", eof) == -1) {
+		goto fail;
+	}
+	length = read_linehead(f);
+	if (length == -1) {
+		goto fail;
+	}
+	if (read_data(f, &key, length) == -1) {
+		goto fail;
+	}
+	if (swallow(f, "\"\n", NULL) == -1) {
+		goto fail;
+	}
+	length = read_linehead(f);
+	if (length == -1) {
+		goto fail;
+	}
+	if (read_data(f, &data, length) == -1) {
+		goto fail;
+	}
+	if ((swallow(f, "\"\n", NULL) == -1)
+	    || (swallow(f, "}\n", NULL) == -1)) {
+		goto fail;
+	}
+	e = ntdb_store(ntdb, key, data, NTDB_INSERT);
+	if (e != NTDB_SUCCESS) {
+		fprintf(stderr, "NTDB error: %s\n", ntdb_errorstr(e));
+		goto fail;
+	}
+
+	ret = true;
+fail:
+	free(key.dptr);
+	free(data.dptr);
+	return ret;
+}
+
+static int restore_ntdb(const char *fname)
+{
+	struct ntdb_context *ntdb;
+
+	ntdb = ntdb_open(fname, 0, O_RDWR|O_CREAT|O_EXCL, 0666, NULL);
+	if (!ntdb) {
+		perror("ntdb_open");
+		fprintf(stderr, "Failed to open %s\n", fname);
+		return 1;
+	}
+
+	while (1) {
+		int eof = 0;
+		if (!read_rec(stdin, ntdb, &eof)) {
+			if (eof) {
+				break;
+			}
+			return 1;
+		}
+	}
+	if (ntdb_close(ntdb)) {
+		fprintf(stderr, "Error closing ntdb\n");
+		return 1;
+	}
+	fprintf(stderr, "EOF\n");
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	char *fname;
+
+	if (argc < 2) {
+		printf("Usage: %s dbname < tdbdump_output\n", argv[0]);
+		exit(1);
+	}
+
+	fname = argv[1];
+
+	return restore_ntdb(fname);
+}
diff --git a/lib/ntdb/tools/ntdbtool.c b/lib/ntdb/tools/ntdbtool.c
new file mode 100644
index 0000000000..7c1ef7df7a
--- /dev/null
+++ b/lib/ntdb/tools/ntdbtool.c
@@ -0,0 +1,810 @@
+/*
+   Unix SMB/CIFS implementation.
+   Samba database functions
+   Copyright (C) Andrew Tridgell              1999-2000
+   Copyright (C) Paul `Rusty' Russell		   2000
+   Copyright (C) Jeremy Allison			   2000
+   Copyright (C) Andrew Esh                        2001
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+#include "ntdb.h"
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#include <system/filesys.h>
+#include <system/time.h>
+#include <system/locale.h>
+#else
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <stdarg.h>
+#endif
+
+static int do_command(void);
+const char *cmdname;
+char *arg1, *arg2;
+size_t arg1len, arg2len;
+int bIterate = 0;
+char *line;
+NTDB_DATA iterate_kbuf;
+char cmdline[1024];
+static int disable_mmap;
+
+enum commands {
+	CMD_CREATE_NTDB,
+	CMD_OPEN_NTDB,
+	CMD_TRANSACTION_START,
+	CMD_TRANSACTION_COMMIT,
+	CMD_TRANSACTION_CANCEL,
+	CMD_ERASE,
+	CMD_DUMP,
+	CMD_INSERT,
+	CMD_MOVE,
+	CMD_STORE,
+	CMD_SHOW,
+	CMD_KEYS,
+	CMD_HEXKEYS,
+	CMD_DELETE,
+#if 0
+	CMD_LIST_HASH_FREE,
+	CMD_LIST_FREE,
+#endif
+	CMD_INFO,
+	CMD_MMAP,
+	CMD_SPEED,
+	CMD_FIRST,
+	CMD_NEXT,
+	CMD_SYSTEM,
+	CMD_CHECK,
+	CMD_QUIT,
+	CMD_HELP
+};
+
+typedef struct {
+	const char *name;
+	enum commands cmd;
+} COMMAND_TABLE;
+
+COMMAND_TABLE cmd_table[] = {
+	{"create",	CMD_CREATE_NTDB},
+	{"open",	CMD_OPEN_NTDB},
+#if 0
+	{"transaction_start",	CMD_TRANSACTION_START},
+	{"transaction_commit",	CMD_TRANSACTION_COMMIT},
+	{"transaction_cancel",	CMD_TRANSACTION_CANCEL},
+#endif
+	{"erase",	CMD_ERASE},
+	{"dump",	CMD_DUMP},
+	{"insert",	CMD_INSERT},
+	{"move",	CMD_MOVE},
+	{"store",	CMD_STORE},
+	{"show",	CMD_SHOW},
+	{"keys",	CMD_KEYS},
+	{"hexkeys",	CMD_HEXKEYS},
+	{"delete",	CMD_DELETE},
+#if 0
+	{"list",	CMD_LIST_HASH_FREE},
+	{"free",	CMD_LIST_FREE},
+#endif
+	{"info",	CMD_INFO},
+	{"speed",	CMD_SPEED},
+	{"mmap",	CMD_MMAP},
+	{"first",	CMD_FIRST},
+	{"1",		CMD_FIRST},
+	{"next",	CMD_NEXT},
+	{"n",		CMD_NEXT},
+	{"check",	CMD_CHECK},
+	{"quit",	CMD_QUIT},
+	{"q",		CMD_QUIT},
+	{"!",		CMD_SYSTEM},
+	{NULL,		CMD_HELP}
+};
+
+struct timeval tp1,tp2;
+
+static void _start_timer(void)
+{
+	gettimeofday(&tp1,NULL);
+}
+
+static double _end_timer(void)
+{
+	gettimeofday(&tp2,NULL);
+	return((tp2.tv_sec - tp1.tv_sec) +
+	       (tp2.tv_usec - tp1.tv_usec)*1.0e-6);
+}
+
+static void ntdb_log(struct ntdb_context *ntdb,
+		    enum ntdb_log_level level,
+		    enum NTDB_ERROR ecode,
+		    const char *message,
+		    void *data)
+{
+	fprintf(stderr, "ntdb:%s:%s:%s\n",
+		ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+}
+
+/* a ntdb tool for manipulating a ntdb database */
+
+static struct ntdb_context *ntdb;
+
+static int print_rec(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state);
+static int print_key(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state);
+static int print_hexkey(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state);
+
+static void print_asc(const char *buf,int len)
+{
+	int i;
+
+	/* We're probably printing ASCII strings so don't try to display
+	   the trailing NULL character. */
+
+	if (buf[len - 1] == 0)
+	        len--;
+
+	for (i=0;i<len;i++)
+		printf("%c",isprint(buf[i])?buf[i]:'.');
+}
+
+static void print_data(const char *buf,int len)
+{
+	int i=0;
+	if (len<=0) return;
+	printf("[%03X] ",i);
+	for (i=0;i<len;) {
+		printf("%02X ",(int)((unsigned char)buf[i]));
+		i++;
+		if (i%8 == 0) printf(" ");
+		if (i%16 == 0) {
+			print_asc(&buf[i-16],8); printf(" ");
+			print_asc(&buf[i-8],8); printf("\n");
+			if (i<len) printf("[%03X] ",i);
+		}
+	}
+	if (i%16) {
+		int n;
+
+		n = 16 - (i%16);
+		printf(" ");
+		if (n>8) printf(" ");
+		while (n--) printf("   ");
+
+		n = i%16;
+		if (n > 8) n = 8;
+		print_asc(&buf[i-(i%16)],n); printf(" ");
+		n = (i%16) - n;
+		if (n>0) print_asc(&buf[i-n],n);
+		printf("\n");
+	}
+}
+
+static void help(void)
+{
+	printf("\n"
+"tdbtool: \n"
+"  create    dbname     : create a database\n"
+"  open      dbname     : open an existing database\n"
+"  openjh    dbname     : open an existing database (jenkins hash)\n"
+"  transaction_start    : start a transaction\n"
+"  transaction_commit   : commit a transaction\n"
+"  transaction_cancel   : cancel a transaction\n"
+"  erase                : erase the database\n"
+"  dump                 : dump the database as strings\n"
+"  keys                 : dump the database keys as strings\n"
+"  hexkeys              : dump the database keys as hex values\n"
+"  info                 : print summary info about the database\n"
+"  insert    key  data  : insert a record\n"
+"  move      key  file  : move a record to a destination ntdb\n"
+"  store     key  data  : store a record (replace)\n"
+"  show      key        : show a record by key\n"
+"  delete    key        : delete a record by key\n"
+#if 0
+"  list                 : print the database hash table and freelist\n"
+"  free                 : print the database freelist\n"
+#endif
+"  check                : check the integrity of an opened database\n"
+"  speed                : perform speed tests on the database\n"
+"  ! command            : execute system command\n"
+"  1 | first            : print the first record\n"
+"  n | next             : print the next record\n"
+"  q | quit             : terminate\n"
+"  \\n                   : repeat 'next' command\n"
+"\n");
+}
+
+static void terror(enum NTDB_ERROR err, const char *why)
+{
+	if (err != NTDB_SUCCESS)
+		printf("%s:%s\n", ntdb_errorstr(err), why);
+	else
+		printf("%s\n", why);
+}
+
+static void create_ntdb(const char *tdbname)
+{
+	union ntdb_attribute log_attr;
+	log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+	log_attr.base.next = NULL;
+	log_attr.log.fn = ntdb_log;
+
+	if (ntdb) ntdb_close(ntdb);
+	ntdb = ntdb_open(tdbname, (disable_mmap?NTDB_NOMMAP:0),
+		       O_RDWR | O_CREAT | O_TRUNC, 0600, &log_attr);
+	if (!ntdb) {
+		printf("Could not create %s: %s\n", tdbname, strerror(errno));
+	}
+}
+
+static void open_ntdb(const char *tdbname)
+{
+	union ntdb_attribute log_attr;
+	log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+	log_attr.base.next = NULL;
+	log_attr.log.fn = ntdb_log;
+
+	if (ntdb) ntdb_close(ntdb);
+	ntdb = ntdb_open(tdbname, disable_mmap?NTDB_NOMMAP:0, O_RDWR, 0600,
+		       &log_attr);
+	if (!ntdb) {
+		printf("Could not open %s: %s\n", tdbname, strerror(errno));
+	}
+}
+
+static void insert_ntdb(char *keyname, size_t keylen, char* data, size_t datalen)
+{
+	NTDB_DATA key, dbuf;
+	enum NTDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(NTDB_SUCCESS, "need key");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+	dbuf.dptr = (unsigned char *)data;
+	dbuf.dsize = datalen;
+
+	ecode = ntdb_store(ntdb, key, dbuf, NTDB_INSERT);
+	if (ecode) {
+		terror(ecode, "insert failed");
+	}
+}
+
+static void store_ntdb(char *keyname, size_t keylen, char* data, size_t datalen)
+{
+	NTDB_DATA key, dbuf;
+	enum NTDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(NTDB_SUCCESS, "need key");
+		return;
+	}
+
+	if ((data == NULL) || (datalen == 0)) {
+		terror(NTDB_SUCCESS, "need data");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+	dbuf.dptr = (unsigned char *)data;
+	dbuf.dsize = datalen;
+
+	printf("Storing key:\n");
+	print_rec(ntdb, key, dbuf, NULL);
+
+	ecode = ntdb_store(ntdb, key, dbuf, NTDB_REPLACE);
+	if (ecode) {
+		terror(ecode, "store failed");
+	}
+}
+
+static void show_ntdb(char *keyname, size_t keylen)
+{
+	NTDB_DATA key, dbuf;
+	enum NTDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(NTDB_SUCCESS, "need key");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+
+	ecode = ntdb_fetch(ntdb, key, &dbuf);
+	if (ecode) {
+		terror(ecode, "fetch failed");
+		return;
+	}
+
+	print_rec(ntdb, key, dbuf, NULL);
+
+	free( dbuf.dptr );
+}
+
+static void delete_ntdb(char *keyname, size_t keylen)
+{
+	NTDB_DATA key;
+	enum NTDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(NTDB_SUCCESS, "need key");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+
+	ecode = ntdb_delete(ntdb, key);
+	if (ecode) {
+		terror(ecode, "delete failed");
+	}
+}
+
+static void move_rec(char *keyname, size_t keylen, char* tdbname)
+{
+	NTDB_DATA key, dbuf;
+	struct ntdb_context *dst_ntdb;
+	enum NTDB_ERROR ecode;
+
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror(NTDB_SUCCESS, "need key");
+		return;
+	}
+
+	if ( !tdbname ) {
+		terror(NTDB_SUCCESS, "need destination ntdb name");
+		return;
+	}
+
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+
+	ecode = ntdb_fetch(ntdb, key, &dbuf);
+	if (ecode) {
+		terror(ecode, "fetch failed");
+		return;
+	}
+
+	print_rec(ntdb, key, dbuf, NULL);
+
+	dst_ntdb = ntdb_open(tdbname, 0, O_RDWR, 0600, NULL);
+	if ( !dst_ntdb ) {
+		terror(NTDB_SUCCESS, "unable to open destination ntdb");
+		return;
+	}
+
+	ecode = ntdb_store( dst_ntdb, key, dbuf, NTDB_REPLACE);
+	if (ecode)
+		terror(ecode, "failed to move record");
+	else
+		printf("record moved\n");
+
+	ntdb_close( dst_ntdb );
+}
+
+static int print_rec(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	printf("\nkey %d bytes\n", (int)key.dsize);
+	print_asc((const char *)key.dptr, key.dsize);
+	printf("\ndata %d bytes\n", (int)dbuf.dsize);
+	print_data((const char *)dbuf.dptr, dbuf.dsize);
+	return 0;
+}
+
+static int print_key(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	printf("key %d bytes: ", (int)key.dsize);
+	print_asc((const char *)key.dptr, key.dsize);
+	printf("\n");
+	return 0;
+}
+
+static int print_hexkey(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	printf("key %d bytes\n", (int)key.dsize);
+	print_data((const char *)key.dptr, key.dsize);
+	printf("\n");
+	return 0;
+}
+
+static int total_bytes;
+
+static int traverse_fn(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+	total_bytes += dbuf.dsize;
+	return 0;
+}
+
+static void info_ntdb(void)
+{
+	enum NTDB_ERROR ecode;
+	char *summary;
+
+	ecode = ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &summary);
+
+	if (ecode) {
+		terror(ecode, "Getting summary");
+	} else {
+		printf("%s", summary);
+		free(summary);
+	}
+}
+
+static void speed_ntdb(const char *tlimit)
+{
+	unsigned timelimit = tlimit?atoi(tlimit):0;
+	double t;
+	int ops;
+	if (timelimit == 0) timelimit = 5;
+
+	ops = 0;
+	printf("Testing store speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		long int r = random();
+		NTDB_DATA key, dbuf;
+		key = ntdb_mkdata("store test", strlen("store test"));
+		dbuf.dptr = (unsigned char *)&r;
+		dbuf.dsize = sizeof(r);
+		ntdb_store(ntdb, key, dbuf, NTDB_REPLACE);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+
+	ops = 0;
+	printf("Testing fetch speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		long int r = random();
+		NTDB_DATA key, dbuf;
+		key = ntdb_mkdata("store test", strlen("store test"));
+		dbuf.dptr = (unsigned char *)&r;
+		dbuf.dsize = sizeof(r);
+		ntdb_fetch(ntdb, key, &dbuf);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+
+	ops = 0;
+	printf("Testing transaction speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		long int r = random();
+		NTDB_DATA key, dbuf;
+		key = ntdb_mkdata("transaction test", strlen("transaction test"));
+		dbuf.dptr = (unsigned char *)&r;
+		dbuf.dsize = sizeof(r);
+		ntdb_transaction_start(ntdb);
+		ntdb_store(ntdb, key, dbuf, NTDB_REPLACE);
+		ntdb_transaction_commit(ntdb);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+
+	ops = 0;
+	printf("Testing traverse speed for %u seconds\n", timelimit);
+	_start_timer();
+	do {
+		ntdb_traverse(ntdb, traverse_fn, NULL);
+		t = _end_timer();
+		ops++;
+	} while (t < timelimit);
+	printf("%10.3f ops/sec\n", ops/t);
+}
+
+static void toggle_mmap(void)
+{
+	disable_mmap = !disable_mmap;
+	if (disable_mmap) {
+		printf("mmap is disabled\n");
+	} else {
+		printf("mmap is enabled\n");
+	}
+}
+
+static char *ntdb_getline(const char *prompt)
+{
+	static char thisline[1024];
+	char *p;
+	fputs(prompt, stdout);
+	thisline[0] = 0;
+	p = fgets(thisline, sizeof(thisline)-1, stdin);
+	if (p) p = strchr(p, '\n');
+	if (p) *p = 0;
+	return p?thisline:NULL;
+}
+
+static int do_delete_fn(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf,
+                     void *state)
+{
+    return ntdb_delete(the_ntdb, key);
+}
+
+static void first_record(struct ntdb_context *the_ntdb, NTDB_DATA *pkey)
+{
+	NTDB_DATA dbuf;
+	enum NTDB_ERROR ecode;
+	ecode = ntdb_firstkey(the_ntdb, pkey);
+	if (!ecode)
+		ecode = ntdb_fetch(the_ntdb, *pkey, &dbuf);
+	if (ecode) terror(ecode, "fetch failed");
+	else {
+		print_rec(the_ntdb, *pkey, dbuf, NULL);
+	}
+}
+
+static void next_record(struct ntdb_context *the_ntdb, NTDB_DATA *pkey)
+{
+	NTDB_DATA dbuf;
+	enum NTDB_ERROR ecode;
+	ecode = ntdb_nextkey(the_ntdb, pkey);
+
+	if (!ecode)
+		ecode = ntdb_fetch(the_ntdb, *pkey, &dbuf);
+	if (ecode)
+		terror(ecode, "fetch failed");
+	else
+		print_rec(the_ntdb, *pkey, dbuf, NULL);
+}
+
+static void check_db(struct ntdb_context *the_ntdb)
+{
+	if (!the_ntdb) {
+		printf("Error: No database opened!\n");
+	} else {
+		if (ntdb_check(the_ntdb, NULL, NULL) != 0)
+			printf("Integrity check for the opened database failed.\n");
+		else
+			printf("Database integrity is OK.\n");
+	}
+}
+
+static int do_command(void)
+{
+	COMMAND_TABLE *ctp = cmd_table;
+	enum commands mycmd = CMD_HELP;
+	int cmd_len;
+
+	if (cmdname && strlen(cmdname) == 0) {
+		mycmd = CMD_NEXT;
+	} else {
+		while (ctp->name) {
+			cmd_len = strlen(ctp->name);
+			if (strncmp(ctp->name,cmdname,cmd_len) == 0) {
+				mycmd = ctp->cmd;
+				break;
+			}
+			ctp++;
+		}
+	}
+
+	switch (mycmd) {
+	case CMD_CREATE_NTDB:
+		bIterate = 0;
+		create_ntdb(arg1);
+		return 0;
+	case CMD_OPEN_NTDB:
+		bIterate = 0;
+		open_ntdb(arg1);
+		return 0;
+	case CMD_SYSTEM:
+		/* Shell command */
+		if (system(arg1) == -1) {
+			terror(NTDB_SUCCESS, "system() call failed\n");
+		}
+		return 0;
+	case CMD_QUIT:
+		return 1;
+	default:
+		/* all the rest require a open database */
+		if (!ntdb) {
+			bIterate = 0;
+			terror(NTDB_SUCCESS, "database not open");
+			help();
+			return 0;
+		}
+		switch (mycmd) {
+		case CMD_TRANSACTION_START:
+			bIterate = 0;
+			ntdb_transaction_start(ntdb);
+			return 0;
+		case CMD_TRANSACTION_COMMIT:
+			bIterate = 0;
+			ntdb_transaction_commit(ntdb);
+			return 0;
+		case CMD_TRANSACTION_CANCEL:
+			bIterate = 0;
+			ntdb_transaction_cancel(ntdb);
+			return 0;
+		case CMD_ERASE:
+			bIterate = 0;
+			ntdb_traverse(ntdb, do_delete_fn, NULL);
+			return 0;
+		case CMD_DUMP:
+			bIterate = 0;
+			ntdb_traverse(ntdb, print_rec, NULL);
+			return 0;
+		case CMD_INSERT:
+			bIterate = 0;
+			insert_ntdb(arg1, arg1len,arg2,arg2len);
+			return 0;
+		case CMD_MOVE:
+			bIterate = 0;
+			move_rec(arg1,arg1len,arg2);
+			return 0;
+		case CMD_STORE:
+			bIterate = 0;
+			store_ntdb(arg1,arg1len,arg2,arg2len);
+			return 0;
+		case CMD_SHOW:
+			bIterate = 0;
+			show_ntdb(arg1, arg1len);
+			return 0;
+		case CMD_KEYS:
+			ntdb_traverse(ntdb, print_key, NULL);
+			return 0;
+		case CMD_HEXKEYS:
+			ntdb_traverse(ntdb, print_hexkey, NULL);
+			return 0;
+		case CMD_DELETE:
+			bIterate = 0;
+			delete_ntdb(arg1,arg1len);
+			return 0;
+#if 0
+		case CMD_LIST_HASH_FREE:
+			ntdb_dump_all(ntdb);
+			return 0;
+		case CMD_LIST_FREE:
+			ntdb_printfreelist(ntdb);
+			return 0;
+#endif
+		case CMD_INFO:
+			info_ntdb();
+			return 0;
+		case CMD_SPEED:
+			speed_ntdb(arg1);
+			return 0;
+		case CMD_MMAP:
+			toggle_mmap();
+			return 0;
+		case CMD_FIRST:
+			bIterate = 1;
+			first_record(ntdb, &iterate_kbuf);
+			return 0;
+		case CMD_NEXT:
+			if (bIterate)
+				next_record(ntdb, &iterate_kbuf);
+			return 0;
+		case CMD_CHECK:
+			check_db(ntdb);
+			return 0;
+		case CMD_HELP:
+			help();
+			return 0;
+		case CMD_CREATE_NTDB:
+		case CMD_OPEN_NTDB:
+		case CMD_SYSTEM:
+		case CMD_QUIT:
+			/*
+			 * unhandled commands.  cases included here to avoid compiler
+			 * warnings.
+			 */
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+static char *convert_string(char *instring, size_t *sizep)
+{
+	size_t length = 0;
+	char *outp, *inp;
+	char temp[3];
+
+	outp = inp = instring;
+
+	while (*inp) {
+		if (*inp == '\\') {
+			inp++;
+			if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+				temp[0] = *inp++;
+				temp[1] = '\0';
+				if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+					temp[1] = *inp++;
+					temp[2] = '\0';
+				}
+				*outp++ = (char)strtol((const char *)temp,NULL,16);
+			} else {
+				*outp++ = *inp++;
+			}
+		} else {
+			*outp++ = *inp++;
+		}
+		length++;
+	}
+	*sizep = length;
+	return instring;
+}
+
+int main(int argc, char *argv[])
+{
+	cmdname = "";
+	arg1 = NULL;
+	arg1len = 0;
+	arg2 = NULL;
+	arg2len = 0;
+
+	if (argv[1]) {
+		cmdname = "open";
+		arg1 = argv[1];
+		do_command();
+		cmdname =  "";
+		arg1 = NULL;
+	}
+
+	switch (argc) {
+	case 1:
+	case 2:
+		/* Interactive mode */
+		while ((cmdname = ntdb_getline("ntdb> "))) {
+			arg2 = arg1 = NULL;
+			if ((arg1 = strchr((const char *)cmdname,' ')) != NULL) {
+				arg1++;
+				arg2 = arg1;
+				while (*arg2) {
+					if (*arg2 == ' ') {
+						*arg2++ = '\0';
+						break;
+					}
+					if ((*arg2++ == '\\') && (*arg2 == ' ')) {
+						arg2++;
+					}
+				}
+			}
+			if (arg1) arg1 = convert_string(arg1,&arg1len);
+			if (arg2) arg2 = convert_string(arg2,&arg2len);
+			if (do_command()) break;
+		}
+		break;
+	case 5:
+		arg2 = convert_string(argv[4],&arg2len);
+	case 4:
+		arg1 = convert_string(argv[3],&arg1len);
+	case 3:
+		cmdname = argv[2];
+	default:
+		do_command();
+		break;
+	}
+
+	if (ntdb) ntdb_close(ntdb);
+
+	return 0;
+}
diff --git a/lib/ntdb/tools/ntdbtorture.c b/lib/ntdb/tools/ntdbtorture.c
new file mode 100644
index 0000000000..c7b249db06
--- /dev/null
+++ b/lib/ntdb/tools/ntdbtorture.c
@@ -0,0 +1,529 @@
+/* this tests ntdb by doing lots of ops from several simultaneous
+   writers - that stresses the locking code.
+*/
+
+#include "config.h"
+#include "ntdb.h"
+#include <ccan/err/err.h>
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#else
+#include <stdlib.h>
+#include <getopt.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <time.h>
+#include <sys/wait.h>
+#endif
+
+//#define REOPEN_PROB 30
+#define DELETE_PROB 8
+#define STORE_PROB 4
+#define APPEND_PROB 6
+#define TRANSACTION_PROB 10
+#define TRANSACTION_PREPARE_PROB 2
+#define LOCKSTORE_PROB 5
+#define TRAVERSE_PROB 20
+#define TRAVERSE_MOD_PROB 100
+#define TRAVERSE_ABORT_PROB 500
+#define CULL_PROB 100
+#define KEYLEN 3
+#define DATALEN 100
+
+static struct ntdb_context *db;
+static int in_transaction;
+static int in_traverse;
+static int error_count;
+#if TRANSACTION_PROB
+static int always_transaction = 0;
+#endif
+static int loopnum;
+static int count_pipe;
+static union ntdb_attribute log_attr;
+static union ntdb_attribute seed_attr;
+
+static void ntdb_log(struct ntdb_context *ntdb,
+		    enum ntdb_log_level level,
+		    enum NTDB_ERROR ecode,
+		    const char *message,
+		    void *data)
+{
+	printf("ntdb:%s:%s:%s\n",
+	       ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+	fflush(stdout);
+#if 0
+	{
+		char str[200];
+		signal(SIGUSR1, SIG_IGN);
+		sprintf(str,"xterm -e gdb /proc/%d/exe %d", getpid(), getpid());
+		system(str);
+	}
+#endif
+}
+
+#include "../private.h"
+
+static void segv_handler(int sig, siginfo_t *info, void *p)
+{
+	char string[100];
+
+	sprintf(string, "%u: death at %p (map_ptr %p, map_size %zu)\n",
+		getpid(), info->si_addr, db->file->map_ptr,
+		(size_t)db->file->map_size);
+	if (write(2, string, strlen(string)) > 0)
+		sleep(60);
+	_exit(11);
+}
+
+static void fatal(struct ntdb_context *ntdb, const char *why)
+{
+	fprintf(stderr, "%u:%s:%s\n", getpid(), why,
+		ntdb ? ntdb_errorstr(ntdb_error(ntdb)) : "(no ntdb)");
+	error_count++;
+}
+
+static char *randbuf(int len)
+{
+	char *buf;
+	int i;
+	buf = (char *)malloc(len+1);
+
+	for (i=0;i<len;i++) {
+		buf[i] = 'a' + (rand() % 26);
+	}
+	buf[i] = 0;
+	return buf;
+}
+
+static void addrec_db(void);
+static int modify_traverse(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf,
+			   void *state)
+{
+#if CULL_PROB
+	if (random() % CULL_PROB == 0) {
+		ntdb_delete(ntdb, key);
+	}
+#endif
+
+#if TRAVERSE_MOD_PROB
+	if (random() % TRAVERSE_MOD_PROB == 0) {
+		addrec_db();
+	}
+#endif
+
+#if TRAVERSE_ABORT_PROB
+	if (random() % TRAVERSE_ABORT_PROB == 0)
+		return 1;
+#endif
+
+	return 0;
+}
+
+static void addrec_db(void)
+{
+	int klen, dlen;
+	char *k, *d;
+	NTDB_DATA key, data;
+
+	klen = 1 + (rand() % KEYLEN);
+	dlen = 1 + (rand() % DATALEN);
+
+	k = randbuf(klen);
+	d = randbuf(dlen);
+
+	key.dptr = (unsigned char *)k;
+	key.dsize = klen+1;
+
+	data.dptr = (unsigned char *)d;
+	data.dsize = dlen+1;
+
+#if REOPEN_PROB
+	if (in_traverse == 0 && in_transaction == 0 && random() % REOPEN_PROB == 0) {
+		ntdb_reopen_all(0);
+		goto next;
+	}
+#endif
+
+#if TRANSACTION_PROB
+	if (in_traverse == 0 && in_transaction == 0 && (always_transaction || random() % TRANSACTION_PROB == 0)) {
+		if (ntdb_transaction_start(db) != 0) {
+			fatal(db, "ntdb_transaction_start failed");
+		}
+		in_transaction++;
+		goto next;
+	}
+	if (in_traverse == 0 && in_transaction && random() % TRANSACTION_PROB == 0) {
+		if (random() % TRANSACTION_PREPARE_PROB == 0) {
+			if (ntdb_transaction_prepare_commit(db) != 0) {
+				fatal(db, "ntdb_transaction_prepare_commit failed");
+			}
+		}
+		if (ntdb_transaction_commit(db) != 0) {
+			fatal(db, "ntdb_transaction_commit failed");
+		}
+		in_transaction--;
+		goto next;
+	}
+
+	if (in_traverse == 0 && in_transaction && random() % TRANSACTION_PROB == 0) {
+		ntdb_transaction_cancel(db);
+		in_transaction--;
+		goto next;
+	}
+#endif
+
+#if DELETE_PROB
+	if (random() % DELETE_PROB == 0) {
+		ntdb_delete(db, key);
+		goto next;
+	}
+#endif
+
+#if STORE_PROB
+	if (random() % STORE_PROB == 0) {
+		if (ntdb_store(db, key, data, NTDB_REPLACE) != 0) {
+			fatal(db, "ntdb_store failed");
+		}
+		goto next;
+	}
+#endif
+
+#if APPEND_PROB
+	if (random() % APPEND_PROB == 0) {
+		if (ntdb_append(db, key, data) != 0) {
+			fatal(db, "ntdb_append failed");
+		}
+		goto next;
+	}
+#endif
+
+#if LOCKSTORE_PROB
+	if (random() % LOCKSTORE_PROB == 0) {
+		ntdb_chainlock(db, key);
+		if (ntdb_fetch(db, key, &data) != NTDB_SUCCESS) {
+			data.dsize = 0;
+			data.dptr = NULL;
+		}
+		if (ntdb_store(db, key, data, NTDB_REPLACE) != 0) {
+			fatal(db, "ntdb_store failed");
+		}
+		if (data.dptr) free(data.dptr);
+		ntdb_chainunlock(db, key);
+		goto next;
+	}
+#endif
+
+#if TRAVERSE_PROB
+	/* FIXME: recursive traverses break transactions? */
+	if (in_traverse == 0 && random() % TRAVERSE_PROB == 0) {
+		in_traverse++;
+		ntdb_traverse(db, modify_traverse, NULL);
+		in_traverse--;
+		goto next;
+	}
+#endif
+
+	if (ntdb_fetch(db, key, &data) == NTDB_SUCCESS)
+		free(data.dptr);
+
+next:
+	free(k);
+	free(d);
+}
+
+static int traverse_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf,
+                       void *state)
+{
+	ntdb_delete(ntdb, key);
+	return 0;
+}
+
+static void usage(void)
+{
+	printf("Usage: ntdbtorture"
+#if TRANSACTION_PROB
+	       " [-t]"
+#endif
+	       " [-k] [-n NUM_PROCS] [-l NUM_LOOPS] [-s SEED] [-S]\n");
+	exit(0);
+}
+
+static void send_count_and_suicide(int sig)
+{
+	/* This ensures our successor can continue where we left off. */
+	if (write(count_pipe, &loopnum, sizeof(loopnum)) != sizeof(loopnum))
+		exit(2);
+	/* This gives a unique signature. */
+	kill(getpid(), SIGUSR2);
+}
+
+static int run_child(const char *filename, int i, int seed, unsigned num_loops,
+		     unsigned start, int ntdb_flags)
+{
+	struct sigaction act = { .sa_sigaction = segv_handler,
+				 .sa_flags = SA_SIGINFO };
+	sigaction(11, &act, NULL);
+
+	db = ntdb_open(filename, ntdb_flags, O_RDWR | O_CREAT, 0600,
+		      &log_attr);
+	if (!db) {
+		fatal(NULL, "db open failed");
+	}
+
+#if 0
+	if (i == 0) {
+		printf("pid %i\n", getpid());
+		sleep(9);
+	} else
+		sleep(10);
+#endif
+
+	srand(seed + i);
+	srandom(seed + i);
+
+	/* Set global, then we're ready to handle being killed. */
+	loopnum = start;
+	signal(SIGUSR1, send_count_and_suicide);
+
+	for (;loopnum<num_loops && error_count == 0;loopnum++) {
+		addrec_db();
+	}
+
+	if (error_count == 0) {
+		ntdb_traverse(db, NULL, NULL);
+#if TRANSACTION_PROB
+		if (always_transaction) {
+			while (in_transaction) {
+				ntdb_transaction_cancel(db);
+				in_transaction--;
+			}
+			if (ntdb_transaction_start(db) != 0)
+				fatal(db, "ntdb_transaction_start failed");
+		}
+#endif
+		ntdb_traverse(db, traverse_fn, NULL);
+		ntdb_traverse(db, traverse_fn, NULL);
+
+#if TRANSACTION_PROB
+		if (always_transaction) {
+			if (ntdb_transaction_commit(db) != 0)
+				fatal(db, "ntdb_transaction_commit failed");
+		}
+#endif
+	}
+
+	ntdb_close(db);
+
+	return (error_count < 100 ? error_count : 100);
+}
+
+static char *test_path(const char *filename)
+{
+	const char *prefix = getenv("TEST_DATA_PREFIX");
+
+	if (prefix) {
+		char *path = NULL;
+		int ret;
+
+		ret = asprintf(&path, "%s/%s", prefix, filename);
+		if (ret == -1) {
+			return NULL;
+		}
+		return path;
+	}
+
+	return strdup(filename);
+}
+
+int main(int argc, char * const *argv)
+{
+	int i, seed = -1;
+	int num_loops = 5000;
+	int num_procs = 3;
+	int c, pfds[2];
+	extern char *optarg;
+	pid_t *pids;
+	int kill_random = 0;
+	int *done;
+	int ntdb_flags = NTDB_DEFAULT;
+	char *test_ntdb;
+
+	log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+	log_attr.base.next = &seed_attr;
+	log_attr.log.fn = ntdb_log;
+	seed_attr.base.attr = NTDB_ATTRIBUTE_SEED;
+	seed_attr.base.next = NULL;
+
+	while ((c = getopt(argc, argv, "n:l:s:thkS")) != -1) {
+		switch (c) {
+		case 'n':
+			num_procs = strtol(optarg, NULL, 0);
+			break;
+		case 'l':
+			num_loops = strtol(optarg, NULL, 0);
+			break;
+		case 's':
+			seed = strtol(optarg, NULL, 0);
+			break;
+		case 'S':
+			ntdb_flags = NTDB_NOSYNC;
+			break;
+		case 't':
+#if TRANSACTION_PROB
+			always_transaction = 1;
+#else
+			fprintf(stderr, "Transactions not supported\n");
+			usage();
+#endif
+			break;
+		case 'k':
+			kill_random = 1;
+			break;
+		default:
+			usage();
+		}
+	}
+
+	test_ntdb = test_path("torture.ntdb");
+
+	unlink(test_ntdb);
+
+	if (seed == -1) {
+		seed = (getpid() + time(NULL)) & 0x7FFFFFFF;
+	}
+	seed_attr.seed.seed = (((uint64_t)seed) << 32) | seed;
+
+	if (num_procs == 1 && !kill_random) {
+		/* Don't fork for this case, makes debugging easier. */
+		error_count = run_child(test_ntdb, 0, seed, num_loops, 0,
+					ntdb_flags);
+		goto done;
+	}
+
+	pids = (pid_t *)calloc(sizeof(pid_t), num_procs);
+	done = (int *)calloc(sizeof(int), num_procs);
+
+	if (pipe(pfds) != 0) {
+		perror("Creating pipe");
+		exit(1);
+	}
+	count_pipe = pfds[1];
+
+	for (i=0;i<num_procs;i++) {
+		if ((pids[i]=fork()) == 0) {
+			close(pfds[0]);
+			if (i == 0) {
+				printf("testing with %d processes, %d loops, seed=%d%s\n",
+				       num_procs, num_loops, seed,
+#if TRANSACTION_PROB
+				       always_transaction ? " (all within transactions)" : ""
+#else
+				       ""
+#endif
+					);
+			}
+			exit(run_child(test_ntdb, i, seed, num_loops, 0,
+				       ntdb_flags));
+		}
+	}
+
+	while (num_procs) {
+		int status, j;
+		pid_t pid;
+
+		if (error_count != 0) {
+			/* try and stop the test on any failure */
+			for (j=0;j<num_procs;j++) {
+				if (pids[j] != 0) {
+					kill(pids[j], SIGTERM);
+				}
+			}
+		}
+
+		pid = waitpid(-1, &status, kill_random ? WNOHANG : 0);
+		if (pid == 0) {
+			struct timespec ts;
+
+			/* Sleep for 1/10 second. */
+			ts.tv_sec = 0;
+			ts.tv_nsec = 100000000;
+			nanosleep(&ts, NULL);
+
+			/* Kill someone. */
+			kill(pids[random() % num_procs], SIGUSR1);
+			continue;
+		}
+
+		if (pid == -1) {
+			perror("failed to wait for child\n");
+			exit(1);
+		}
+
+		for (j=0;j<num_procs;j++) {
+			if (pids[j] == pid) break;
+		}
+		if (j == num_procs) {
+			printf("unknown child %d exited!?\n", (int)pid);
+			exit(1);
+		}
+		if (WIFSIGNALED(status)) {
+			if (WTERMSIG(status) == SIGUSR2
+			    || WTERMSIG(status) == SIGUSR1) {
+				/* SIGUSR2 means they wrote to pipe. */
+				if (WTERMSIG(status) == SIGUSR2) {
+					if (read(pfds[0], &done[j],
+						 sizeof(done[j]))
+					    != sizeof(done[j]))
+						err(1,
+						    "Short read from child?");
+				}
+				pids[j] = fork();
+				if (pids[j] == 0)
+					exit(run_child(test_ntdb, j, seed,
+						       num_loops, done[j],
+						       ntdb_flags));
+				printf("Restarting child %i for %u-%u\n",
+				       j, done[j], num_loops);
+				continue;
+			}
+			printf("child %d exited with signal %d\n",
+			       (int)pid, WTERMSIG(status));
+			error_count++;
+		} else {
+			if (WEXITSTATUS(status) != 0) {
+				printf("child %d exited with status %d\n",
+				       (int)pid, WEXITSTATUS(status));
+				error_count++;
+			}
+		}
+		memmove(&pids[j], &pids[j+1],
+			(num_procs - j - 1)*sizeof(pids[0]));
+		num_procs--;
+	}
+
+	free(pids);
+
+done:
+	if (error_count == 0) {
+		db = ntdb_open(test_ntdb, NTDB_DEFAULT, O_RDWR | O_CREAT,
+			      0600, &log_attr);
+		if (!db) {
+			fatal(db, "db open failed");
+			exit(1);
+		}
+		if (ntdb_check(db, NULL, NULL) != 0) {
+			fatal(db, "db check failed");
+			exit(1);
+		}
+		ntdb_close(db);
+		printf("OK\n");
+	}
+
+	free(test_ntdb);
+	return error_count;
+}
diff --git a/lib/ntdb/tools/speed.c b/lib/ntdb/tools/speed.c
new file mode 100644
index 0000000000..868494b898
--- /dev/null
+++ b/lib/ntdb/tools/speed.c
@@ -0,0 +1,443 @@
+/* Simple speed test for NTDB */
+#include <ccan/err/err.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include "ntdb.h"
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeval *start,
+			const struct timeval *stop,
+			unsigned int num)
+{
+	struct timeval diff;
+
+	timersub(stop, start, &diff);
+
+	/* Floating point is more accurate here. */
+	return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
+		/ num * 1000;
+}
+
+static size_t file_size(void)
+{
+	struct stat st;
+
+	if (stat("/tmp/speed.ntdb", &st) != 0)
+		return -1;
+	return st.st_size;
+}
+
+static int count_record(struct ntdb_context *ntdb,
+			NTDB_DATA key, NTDB_DATA data, void *p)
+{
+	int *total = p;
+	*total += *(int *)data.dptr;
+	return 0;
+}
+
+static void dump_and_clear_stats(struct ntdb_context **ntdb,
+				 int flags,
+				 union ntdb_attribute *attr)
+{
+	union ntdb_attribute stats;
+	enum NTDB_ERROR ecode;
+
+	stats.base.attr = NTDB_ATTRIBUTE_STATS;
+	stats.stats.size = sizeof(stats.stats);
+	ecode = ntdb_get_attribute(*ntdb, &stats);
+	if (ecode != NTDB_SUCCESS)
+		errx(1, "Getting stats: %s", ntdb_errorstr(ecode));
+
+	printf("allocs = %llu\n",
+	       (unsigned long long)stats.stats.allocs);
+	printf("  alloc_subhash = %llu\n",
+	       (unsigned long long)stats.stats.alloc_subhash);
+	printf("  alloc_chain = %llu\n",
+	       (unsigned long long)stats.stats.alloc_chain);
+	printf("  alloc_bucket_exact = %llu\n",
+	       (unsigned long long)stats.stats.alloc_bucket_exact);
+	printf("  alloc_bucket_max = %llu\n",
+	       (unsigned long long)stats.stats.alloc_bucket_max);
+	printf("  alloc_leftover = %llu\n",
+	       (unsigned long long)stats.stats.alloc_leftover);
+	printf("  alloc_coalesce_tried = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_tried);
+	printf("    alloc_coalesce_iterate_clash = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_iterate_clash);
+	printf("    alloc_coalesce_lockfail = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_lockfail);
+	printf("    alloc_coalesce_race = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_race);
+	printf("    alloc_coalesce_succeeded = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_succeeded);
+	printf("      alloc_coalesce_num_merged = %llu\n",
+	       (unsigned long long)stats.stats.alloc_coalesce_num_merged);
+	printf("compares = %llu\n",
+	       (unsigned long long)stats.stats.compares);
+	printf("  compare_wrong_bucket = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_bucket);
+	printf("  compare_wrong_offsetbits = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_offsetbits);
+	printf("  compare_wrong_keylen = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_keylen);
+	printf("  compare_wrong_rechash = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_rechash);
+	printf("  compare_wrong_keycmp = %llu\n",
+	       (unsigned long long)stats.stats.compare_wrong_keycmp);
+	printf("transactions = %llu\n",
+	       (unsigned long long)stats.stats.transactions);
+	printf("  transaction_cancel = %llu\n",
+	       (unsigned long long)stats.stats.transaction_cancel);
+	printf("  transaction_nest = %llu\n",
+	       (unsigned long long)stats.stats.transaction_nest);
+	printf("  transaction_expand_file = %llu\n",
+	       (unsigned long long)stats.stats.transaction_expand_file);
+	printf("  transaction_read_direct = %llu\n",
+	       (unsigned long long)stats.stats.transaction_read_direct);
+	printf("    transaction_read_direct_fail = %llu\n",
+	       (unsigned long long)stats.stats.transaction_read_direct_fail);
+	printf("  transaction_write_direct = %llu\n",
+	       (unsigned long long)stats.stats.transaction_write_direct);
+	printf("    transaction_write_direct_fail = %llu\n",
+	       (unsigned long long)stats.stats.transaction_write_direct_fail);
+	printf("expands = %llu\n",
+	       (unsigned long long)stats.stats.expands);
+	printf("frees = %llu\n",
+	       (unsigned long long)stats.stats.frees);
+	printf("locks = %llu\n",
+	       (unsigned long long)stats.stats.locks);
+	printf("  lock_lowlevel = %llu\n",
+	       (unsigned long long)stats.stats.lock_lowlevel);
+	printf("  lock_nonblock = %llu\n",
+	       (unsigned long long)stats.stats.lock_nonblock);
+	printf("    lock_nonblock_fail = %llu\n",
+	       (unsigned long long)stats.stats.lock_nonblock_fail);
+
+	/* Now clear. */
+	ntdb_close(*ntdb);
+	*ntdb = ntdb_open("/tmp/speed.ntdb", flags, O_RDWR, 0, attr);
+}
+
+static void ntdb_log(struct ntdb_context *ntdb,
+		    enum ntdb_log_level level,
+		    enum NTDB_ERROR ecode,
+		    const char *message,
+		    void *data)
+{
+	fprintf(stderr, "ntdb:%s:%s:%s\n",
+		ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i, j, num = 1000, stage = 0, stopat = -1;
+	int flags = NTDB_DEFAULT;
+	bool transaction = false, summary = false;
+	NTDB_DATA key, data;
+	struct ntdb_context *ntdb;
+	struct timeval start, stop;
+	union ntdb_attribute seed, log;
+	bool do_stats = false;
+	enum NTDB_ERROR ecode;
+
+	/* Try to keep benchmarks even. */
+	seed.base.attr = NTDB_ATTRIBUTE_SEED;
+	seed.base.next = NULL;
+	seed.seed.seed = 0;
+
+	log.base.attr = NTDB_ATTRIBUTE_LOG;
+	log.base.next = &seed;
+	log.log.fn = ntdb_log;
+
+	if (argv[1] && strcmp(argv[1], "--internal") == 0) {
+		flags = NTDB_INTERNAL;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--transaction") == 0) {
+		transaction = true;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--no-sync") == 0) {
+		flags |= NTDB_NOSYNC;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--summary") == 0) {
+		summary = true;
+		argc--;
+		argv++;
+	}
+	if (argv[1] && strcmp(argv[1], "--stats") == 0) {
+		do_stats = true;
+		argc--;
+		argv++;
+	}
+
+	ntdb = ntdb_open("/tmp/speed.ntdb", flags, O_RDWR|O_CREAT|O_TRUNC,
+		       0600, &log);
+	if (!ntdb)
+		err(1, "Opening /tmp/speed.ntdb");
+
+	key.dptr = (void *)&i;
+	key.dsize = sizeof(i);
+	data = key;
+
+	if (argv[1]) {
+		num = atoi(argv[1]);
+		argv++;
+		argc--;
+	}
+
+	if (argv[1]) {
+		stopat = atoi(argv[1]);
+		argv++;
+		argc--;
+	}
+
+	/* Add 1000 records. */
+	printf("Adding %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
+			errx(1, "Inserting key %u in ntdb: %s",
+			     i, ntdb_errorstr(ecode));
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+
+	if (++stage == stopat)
+		exit(0);
+
+	/* Finding 1000 records. */
+	printf("Finding %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		NTDB_DATA dbuf;
+		if ((ecode = ntdb_fetch(ntdb, key, &dbuf)) != NTDB_SUCCESS
+		    || *(int *)dbuf.dptr != i) {
+			errx(1, "Fetching key %u in ntdb gave %u",
+			     i, ecode ? ecode : *(int *)dbuf.dptr);
+		}
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Missing 1000 records. */
+	printf("Missing %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (i = num; i < num*2; i++) {
+		NTDB_DATA dbuf;
+		ecode = ntdb_fetch(ntdb, key, &dbuf);
+		if (ecode != NTDB_ERR_NOEXIST)
+			errx(1, "Fetching key %u in ntdb gave %s",
+			     i, ntdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Traverse 1000 records. */
+	printf("Traversing %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	i = 0;
+	gettimeofday(&start, NULL);
+	if (ntdb_traverse(ntdb, count_record, &i) != num)
+		errx(1, "Traverse returned wrong number of records");
+	if (i != (num - 1) * (num / 2))
+		errx(1, "Traverse tallied to %u", i);
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Delete 1000 records (not in order). */
+	printf("Deleting %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (j = 0; j < num; j++) {
+		i = (j + 100003) % num;
+		if ((ecode = ntdb_delete(ntdb, key)) != NTDB_SUCCESS)
+			errx(1, "Deleting key %u in ntdb: %s",
+			     i, ntdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Re-add 1000 records (not in order). */
+	printf("Re-adding %u records: ", num); fflush(stdout);
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	gettimeofday(&start, NULL);
+	for (j = 0; j < num; j++) {
+		i = (j + 100003) % num;
+		if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
+			errx(1, "Inserting key %u in ntdb: %s",
+			     i, ntdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	/* Append 1000 records. */
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	printf("Appending %u records: ", num); fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if ((ecode = ntdb_append(ntdb, key, data)) != NTDB_SUCCESS)
+			errx(1, "Appending key %u in ntdb: %s",
+			     i, ntdb_errorstr(ecode));
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (++stage == stopat)
+		exit(0);
+
+	/* Churn 1000 records: not in order! */
+	if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+		errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+	printf("Churning %u records: ", num); fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (j = 0; j < num; j++) {
+		i = (j + 1000019) % num;
+		if ((ecode = ntdb_delete(ntdb, key)) != NTDB_SUCCESS)
+			errx(1, "Deleting key %u in ntdb: %s",
+			     i, ntdb_errorstr(ecode));
+		i += num;
+		if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
+			errx(1, "Inserting key %u in ntdb: %s",
+			     i, ntdb_errorstr(ecode));
+	}
+	gettimeofday(&stop, NULL);
+	if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+		errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+	printf(" %zu ns (%zu bytes)\n",
+	       normalize(&start, &stop, num), file_size());
+
+	if (ntdb_check(ntdb, NULL, NULL))
+		errx(1, "ntdb_check failed!");
+	if (summary) {
+		char *sumstr = NULL;
+		ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+		printf("%s\n", sumstr);
+		free(sumstr);
+	}
+	if (do_stats)
+		dump_and_clear_stats(&ntdb, flags, &log);
+	if (++stage == stopat)
+		exit(0);
+
+	return 0;
+}