summaryrefslogtreecommitdiff
path: root/lib/ntdb/tools
diff options
context:
space:
mode:
Diffstat (limited to 'lib/ntdb/tools')
-rw-r--r--lib/ntdb/tools/Makefile16
-rw-r--r--lib/ntdb/tools/growtdb-bench.c114
-rw-r--r--lib/ntdb/tools/mkntdb.c29
-rw-r--r--lib/ntdb/tools/ntdbbackup.c340
-rw-r--r--lib/ntdb/tools/ntdbdump.c122
-rw-r--r--lib/ntdb/tools/ntdbrestore.c231
-rw-r--r--lib/ntdb/tools/ntdbtool.c810
-rw-r--r--lib/ntdb/tools/ntdbtorture.c529
-rw-r--r--lib/ntdb/tools/speed.c443
9 files changed, 2634 insertions, 0 deletions
diff --git a/lib/ntdb/tools/Makefile b/lib/ntdb/tools/Makefile
new file mode 100644
index 0000000000..087c256d7f
--- /dev/null
+++ b/lib/ntdb/tools/Makefile
@@ -0,0 +1,16 @@
+OBJS:=../../ntdb.o ../../hash.o ../../tally.o
+CFLAGS:=-I../../.. -I.. -Wall -g -O3 #-g -pg
+LDFLAGS:=-L../../..
+
+default: ntdbtorture ntdbtool ntdbdump ntdbrestore mkntdb speed growtdb-bench
+
+ntdbdump: ntdbdump.c $(OBJS)
+ntdbrestore: ntdbrestore.c $(OBJS)
+ntdbtorture: ntdbtorture.c $(OBJS)
+ntdbtool: ntdbtool.c $(OBJS)
+mkntdb: mkntdb.c $(OBJS)
+speed: speed.c $(OBJS)
+growtdb-bench: growtdb-bench.c $(OBJS)
+
+clean:
+ rm -f ntdbtorture ntdbdump ntdbrestore ntdbtool mkntdb speed growtdb-bench
diff --git a/lib/ntdb/tools/growtdb-bench.c b/lib/ntdb/tools/growtdb-bench.c
new file mode 100644
index 0000000000..640f87af5a
--- /dev/null
+++ b/lib/ntdb/tools/growtdb-bench.c
@@ -0,0 +1,114 @@
+#include "ntdb.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <ccan/err/err.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+static void logfn(struct ntdb_context *ntdb,
+ enum ntdb_log_level level,
+ enum NTDB_ERROR ecode,
+ const char *message,
+ void *data)
+{
+ fprintf(stderr, "ntdb:%s:%s:%s\n",
+ ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+}
+
+int main(int argc, char *argv[])
+{
+ unsigned int i, j, users, groups;
+ NTDB_DATA idxkey, idxdata;
+ NTDB_DATA k, d, gk;
+ char cmd[100];
+ struct ntdb_context *ntdb;
+ enum NTDB_ERROR ecode;
+ union ntdb_attribute log;
+
+ if (argc != 3) {
+ printf("Usage: growtdb-bench <users> <groups>\n");
+ exit(1);
+ }
+ users = atoi(argv[1]);
+ groups = atoi(argv[2]);
+
+ sprintf(cmd, "cat /proc/%i/statm", getpid());
+
+ log.base.attr = NTDB_ATTRIBUTE_LOG;
+ log.base.next = NULL;
+ log.log.fn = logfn;
+
+ ntdb = ntdb_open("/tmp/growtdb.ntdb", NTDB_DEFAULT,
+ O_RDWR|O_CREAT|O_TRUNC, 0600, &log);
+
+ idxkey.dptr = (unsigned char *)"User index";
+ idxkey.dsize = strlen("User index");
+ idxdata.dsize = 51;
+ idxdata.dptr = calloc(idxdata.dsize, 1);
+
+ /* Create users. */
+ k.dsize = 48;
+ k.dptr = calloc(k.dsize, 1);
+ d.dsize = 64;
+ d.dptr = calloc(d.dsize, 1);
+
+ ntdb_transaction_start(ntdb);
+ for (i = 0; i < users; i++) {
+ memcpy(k.dptr, &i, sizeof(i));
+ ecode = ntdb_store(ntdb, k, d, NTDB_INSERT);
+ if (ecode != NTDB_SUCCESS)
+ errx(1, "ntdb insert failed: %s", ntdb_errorstr(ecode));
+
+ /* This simulates a growing index record. */
+ ecode = ntdb_append(ntdb, idxkey, idxdata);
+ if (ecode != NTDB_SUCCESS)
+ errx(1, "ntdb append failed: %s", ntdb_errorstr(ecode));
+ }
+ if ((ecode = ntdb_transaction_commit(ntdb)) != 0)
+ errx(1, "ntdb commit1 failed: %s", ntdb_errorstr(ecode));
+
+ if ((ecode = ntdb_check(ntdb, NULL, NULL)) != 0)
+ errx(1, "ntdb_check failed after initial insert!");
+
+ system(cmd);
+
+ /* Now put them all in groups: add 32 bytes to each record for
+ * a group. */
+ gk.dsize = 48;
+ gk.dptr = calloc(k.dsize, 1);
+ gk.dptr[gk.dsize-1] = 1;
+
+ d.dsize = 32;
+ for (i = 0; i < groups; i++) {
+ ntdb_transaction_start(ntdb);
+ /* Create the "group". */
+ memcpy(gk.dptr, &i, sizeof(i));
+ ecode = ntdb_store(ntdb, gk, d, NTDB_INSERT);
+ if (ecode != NTDB_SUCCESS)
+ errx(1, "ntdb insert failed: %s", ntdb_errorstr(ecode));
+
+ /* Now populate it. */
+ for (j = 0; j < users; j++) {
+ /* Append to the user. */
+ memcpy(k.dptr, &j, sizeof(j));
+ if ((ecode = ntdb_append(ntdb, k, d)) != 0)
+ errx(1, "ntdb append failed: %s",
+ ntdb_errorstr(ecode));
+
+ /* Append to the group. */
+ if ((ecode = ntdb_append(ntdb, gk, d)) != 0)
+ errx(1, "ntdb append failed: %s",
+ ntdb_errorstr(ecode));
+ }
+ if ((ecode = ntdb_transaction_commit(ntdb)) != 0)
+ errx(1, "ntdb commit2 failed: %s", ntdb_errorstr(ecode));
+ if ((ecode = ntdb_check(ntdb, NULL, NULL)) != 0)
+ errx(1, "ntdb_check failed after iteration %i!", i);
+ system(cmd);
+ }
+
+ return 0;
+}
diff --git a/lib/ntdb/tools/mkntdb.c b/lib/ntdb/tools/mkntdb.c
new file mode 100644
index 0000000000..e728987a53
--- /dev/null
+++ b/lib/ntdb/tools/mkntdb.c
@@ -0,0 +1,29 @@
+#include "ntdb.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <ccan/err/err.h>
+
+int main(int argc, char *argv[])
+{
+ unsigned int i, num_recs;
+ struct ntdb_context *ntdb;
+
+ if (argc != 3 || (num_recs = atoi(argv[2])) == 0)
+ errx(1, "Usage: mktdb <tdbfile> <numrecords>");
+
+ ntdb = ntdb_open(argv[1], NTDB_DEFAULT, O_CREAT|O_TRUNC|O_RDWR, 0600,NULL);
+ if (!ntdb)
+ err(1, "Opening %s", argv[1]);
+
+ for (i = 0; i < num_recs; i++) {
+ NTDB_DATA d;
+
+ d.dptr = (void *)&i;
+ d.dsize = sizeof(i);
+ if (ntdb_store(ntdb, d, d, NTDB_INSERT) != 0)
+ err(1, "Failed to store record %i", i);
+ }
+ printf("Done\n");
+ return 0;
+}
diff --git a/lib/ntdb/tools/ntdbbackup.c b/lib/ntdb/tools/ntdbbackup.c
new file mode 100644
index 0000000000..a76f18491b
--- /dev/null
+++ b/lib/ntdb/tools/ntdbbackup.c
@@ -0,0 +1,340 @@
+/*
+ Unix SMB/CIFS implementation.
+ low level ntdb backup and restore utility
+ Copyright (C) Andrew Tridgell 2002
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+
+ This program is meant for backup/restore of ntdb databases. Typical usage would be:
+ tdbbackup *.ntdb
+ when Samba shuts down cleanly, which will make a backup of all the local databases
+ to *.bak files. Then on Samba startup you would use:
+ tdbbackup -v *.ntdb
+ and this will check the databases for corruption and if corruption is detected then
+ the backup will be restored.
+
+ You may also like to do a backup on a regular basis while Samba is
+ running, perhaps using cron.
+
+ The reason this program is needed is to cope with power failures
+ while Samba is running. A power failure could lead to database
+ corruption and Samba will then not start correctly.
+
+ Note that many of the databases in Samba are transient and thus
+ don't need to be backed up, so you can optimise the above a little
+ by only running the backup on the critical databases.
+
+ */
+
+#include "config.h"
+#include "ntdb.h"
+#include "system/filesys.h"
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
+
+static int failed;
+
+static void ntdb_log(struct ntdb_context *ntdb,
+ enum ntdb_log_level level,
+ enum NTDB_ERROR ecode,
+ const char *message,
+ void *data)
+{
+ fprintf(stderr, "%s:%s\n", ntdb_errorstr(ecode), message);
+}
+
+static char *add_suffix(const char *name, const char *suffix)
+{
+ char *ret;
+ int len = strlen(name) + strlen(suffix) + 1;
+ ret = (char *)malloc(len);
+ if (!ret) {
+ fprintf(stderr,"Out of memory!\n");
+ exit(1);
+ }
+ snprintf(ret, len, "%s%s", name, suffix);
+ return ret;
+}
+
+static int copy_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ struct ntdb_context *ntdb_new = (struct ntdb_context *)state;
+ enum NTDB_ERROR err;
+
+ err = ntdb_store(ntdb_new, key, dbuf, NTDB_INSERT);
+ if (err) {
+ fprintf(stderr,"Failed to insert into %s: %s\n",
+ ntdb_name(ntdb_new), ntdb_errorstr(err));
+ failed = 1;
+ return 1;
+ }
+ return 0;
+}
+
+
+static int test_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ return 0;
+}
+
+/*
+ carefully backup a ntdb, validating the contents and
+ only doing the backup if its OK
+ this function is also used for restore
+*/
+static int backup_ntdb(const char *old_name, const char *new_name)
+{
+ struct ntdb_context *ntdb;
+ struct ntdb_context *ntdb_new;
+ char *tmp_name;
+ struct stat st;
+ int count1, count2;
+ enum NTDB_ERROR err;
+ union ntdb_attribute log_attr;
+
+ tmp_name = add_suffix(new_name, ".tmp");
+
+ /* stat the old ntdb to find its permissions */
+ if (stat(old_name, &st) != 0) {
+ perror(old_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+ log_attr.base.next = NULL;
+ log_attr.log.fn = ntdb_log;
+
+ /* open the old ntdb */
+ ntdb = ntdb_open(old_name, NTDB_DEFAULT, O_RDWR, 0, &log_attr);
+ if (!ntdb) {
+ printf("Failed to open %s\n", old_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ unlink(tmp_name);
+ ntdb_new = ntdb_open(tmp_name, NTDB_DEFAULT,
+ O_RDWR|O_CREAT|O_EXCL, st.st_mode & 0777,
+ &log_attr);
+ if (!ntdb_new) {
+ perror(tmp_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ err = ntdb_transaction_start(ntdb);
+ if (err) {
+ fprintf(stderr, "Failed to start transaction on old ntdb: %s\n",
+ ntdb_errorstr(err));
+ ntdb_close(ntdb);
+ ntdb_close(ntdb_new);
+ unlink(tmp_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ /* lock the backup ntdb so that nobody else can change it */
+ err = ntdb_lockall(ntdb_new);
+ if (err) {
+ fprintf(stderr, "Failed to lock backup ntdb: %s\n",
+ ntdb_errorstr(err));
+ ntdb_close(ntdb);
+ ntdb_close(ntdb_new);
+ unlink(tmp_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ failed = 0;
+
+ /* traverse and copy */
+ count1 = ntdb_traverse(ntdb, copy_fn, (void *)ntdb_new);
+ if (count1 < 0 || failed) {
+ fprintf(stderr,"failed to copy %s\n", old_name);
+ ntdb_close(ntdb);
+ ntdb_close(ntdb_new);
+ unlink(tmp_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ /* close the old ntdb */
+ ntdb_close(ntdb);
+
+ /* copy done, unlock the backup ntdb */
+ ntdb_unlockall(ntdb_new);
+
+#ifdef HAVE_FDATASYNC
+ if (fdatasync(ntdb_fd(ntdb_new)) != 0) {
+#else
+ if (fsync(ntdb_fd(ntdb_new)) != 0) {
+#endif
+ /* not fatal */
+ fprintf(stderr, "failed to fsync backup file\n");
+ }
+
+ /* close the new ntdb and re-open read-only */
+ ntdb_close(ntdb_new);
+
+ /* we don't need the hash attr any more */
+ log_attr.base.next = NULL;
+
+ ntdb_new = ntdb_open(tmp_name, NTDB_DEFAULT, O_RDONLY, 0, &log_attr);
+ if (!ntdb_new) {
+ fprintf(stderr,"failed to reopen %s\n", tmp_name);
+ unlink(tmp_name);
+ perror(tmp_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ /* traverse the new ntdb to confirm */
+ count2 = ntdb_traverse(ntdb_new, test_fn, NULL);
+ if (count2 != count1) {
+ fprintf(stderr,"failed to copy %s\n", old_name);
+ ntdb_close(ntdb_new);
+ unlink(tmp_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ /* close the new ntdb and rename it to .bak */
+ ntdb_close(ntdb_new);
+ if (rename(tmp_name, new_name) != 0) {
+ perror(new_name);
+ free(tmp_name);
+ return 1;
+ }
+
+ free(tmp_name);
+
+ return 0;
+}
+
+/*
+ verify a ntdb and if it is corrupt then restore from *.bak
+*/
+static int verify_ntdb(const char *fname, const char *bak_name)
+{
+ struct ntdb_context *ntdb;
+ int count = -1;
+ union ntdb_attribute log_attr;
+
+ log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+ log_attr.base.next = NULL;
+ log_attr.log.fn = ntdb_log;
+
+ /* open the ntdb */
+ ntdb = ntdb_open(fname, NTDB_DEFAULT, O_RDONLY, 0, &log_attr);
+
+ /* traverse the ntdb, then close it */
+ if (ntdb) {
+ count = ntdb_traverse(ntdb, test_fn, NULL);
+ ntdb_close(ntdb);
+ }
+
+ /* count is < 0 means an error */
+ if (count < 0) {
+ printf("restoring %s\n", fname);
+ return backup_ntdb(bak_name, fname);
+ }
+
+ printf("%s : %d records\n", fname, count);
+
+ return 0;
+}
+
+/*
+ see if one file is newer than another
+*/
+static int file_newer(const char *fname1, const char *fname2)
+{
+ struct stat st1, st2;
+ if (stat(fname1, &st1) != 0) {
+ return 0;
+ }
+ if (stat(fname2, &st2) != 0) {
+ return 1;
+ }
+ return (st1.st_mtime > st2.st_mtime);
+}
+
+static void usage(void)
+{
+ printf("Usage: ntdbbackup [options] <fname...>\n\n");
+ printf(" -h this help message\n");
+ printf(" -v verify mode (restore if corrupt)\n");
+ printf(" -s suffix set the backup suffix\n");
+ printf(" -v verify mode (restore if corrupt)\n");
+}
+
+
+ int main(int argc, char *argv[])
+{
+ int i;
+ int ret = 0;
+ int c;
+ int verify = 0;
+ const char *suffix = ".bak";
+
+ while ((c = getopt(argc, argv, "vhs:")) != -1) {
+ switch (c) {
+ case 'h':
+ usage();
+ exit(0);
+ case 'v':
+ verify = 1;
+ break;
+ case 's':
+ suffix = optarg;
+ break;
+ }
+ }
+
+ argc -= optind;
+ argv += optind;
+
+ if (argc < 1) {
+ usage();
+ exit(1);
+ }
+
+ for (i=0; i<argc; i++) {
+ const char *fname = argv[i];
+ char *bak_name;
+
+ bak_name = add_suffix(fname, suffix);
+
+ if (verify) {
+ if (verify_ntdb(fname, bak_name) != 0) {
+ ret = 1;
+ }
+ } else {
+ if (file_newer(fname, bak_name) &&
+ backup_ntdb(fname, bak_name) != 0) {
+ ret = 1;
+ }
+ }
+
+ free(bak_name);
+ }
+
+ return ret;
+}
diff --git a/lib/ntdb/tools/ntdbdump.c b/lib/ntdb/tools/ntdbdump.c
new file mode 100644
index 0000000000..1b1c59eae3
--- /dev/null
+++ b/lib/ntdb/tools/ntdbdump.c
@@ -0,0 +1,122 @@
+/*
+ simple ntdb dump util
+ Copyright (C) Andrew Tridgell 2001
+ Copyright (C) Rusty Russell 2011
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "config.h"
+#include "ntdb.h"
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#include <system/filesys.h>
+#include <system/locale.h>
+#else
+#include <ctype.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+#endif
+
+static void print_data(NTDB_DATA d)
+{
+ unsigned char *p = (unsigned char *)d.dptr;
+ int len = d.dsize;
+ while (len--) {
+ if (isprint(*p) && !strchr("\"\\", *p)) {
+ fputc(*p, stdout);
+ } else {
+ printf("\\%02X", *p);
+ }
+ p++;
+ }
+}
+
+static int traverse_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ printf("{\n");
+ printf("key(%d) = \"", (int)key.dsize);
+ print_data(key);
+ printf("\"\n");
+ printf("data(%d) = \"", (int)dbuf.dsize);
+ print_data(dbuf);
+ printf("\"\n");
+ printf("}\n");
+ return 0;
+}
+
+static int dump_ntdb(const char *fname, const char *keyname)
+{
+ struct ntdb_context *ntdb;
+ NTDB_DATA key, value;
+
+ ntdb = ntdb_open(fname, 0, O_RDONLY, 0, NULL);
+ if (!ntdb) {
+ printf("Failed to open %s\n", fname);
+ return 1;
+ }
+
+ if (!keyname) {
+ ntdb_traverse(ntdb, traverse_fn, NULL);
+ } else {
+ key = ntdb_mkdata(keyname, strlen(keyname));
+ if (ntdb_fetch(ntdb, key, &value) != 0) {
+ return 1;
+ } else {
+ print_data(value);
+ free(value.dptr);
+ }
+ }
+
+ return 0;
+}
+
+static void usage( void)
+{
+ printf( "Usage: ntdbdump [options] <filename>\n\n");
+ printf( " -h this help message\n");
+ printf( " -k keyname dumps value of keyname\n");
+}
+
+ int main(int argc, char *argv[])
+{
+ char *fname, *keyname=NULL;
+ int c;
+
+ if (argc < 2) {
+ printf("Usage: ntdbdump <fname>\n");
+ exit(1);
+ }
+
+ while ((c = getopt( argc, argv, "hk:")) != -1) {
+ switch (c) {
+ case 'h':
+ usage();
+ exit( 0);
+ case 'k':
+ keyname = optarg;
+ break;
+ default:
+ usage();
+ exit( 1);
+ }
+ }
+
+ fname = argv[optind];
+
+ return dump_ntdb(fname, keyname);
+}
diff --git a/lib/ntdb/tools/ntdbrestore.c b/lib/ntdb/tools/ntdbrestore.c
new file mode 100644
index 0000000000..dad591d562
--- /dev/null
+++ b/lib/ntdb/tools/ntdbrestore.c
@@ -0,0 +1,231 @@
+/*
+ ntdbrestore -- construct a ntdb from tdbdump output.
+ Copyright (C) Volker Lendecke 2010
+ Copyright (C) Simon McVittie 2005
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+#include "ntdb.h"
+#include <assert.h>
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#include <system/filesys.h>
+#else
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#endif
+
+static int read_linehead(FILE *f)
+{
+ int i, c;
+ int num_bytes;
+ char prefix[128];
+
+ while (1) {
+ c = getc(f);
+ if (c == EOF) {
+ return -1;
+ }
+ if (c == '(') {
+ break;
+ }
+ }
+ for (i=0; i<sizeof(prefix); i++) {
+ c = getc(f);
+ if (c == EOF) {
+ return -1;
+ }
+ prefix[i] = c;
+ if (c == '"') {
+ break;
+ }
+ }
+ if (i == sizeof(prefix)) {
+ return -1;
+ }
+ prefix[i] = '\0';
+
+ if (sscanf(prefix, "%d) = ", &num_bytes) != 1) {
+ return -1;
+ }
+ return num_bytes;
+}
+
+static int read_hex(void) {
+ int c;
+ c = getchar();
+ if (c == EOF) {
+ fprintf(stderr, "Unexpected EOF in data\n");
+ return -1;
+ } else if (c == '"') {
+ fprintf(stderr, "Unexpected \\\" sequence\n");
+ return -1;
+ } else if ('0' <= c && c <= '9') {
+ return c - '0';
+ } else if ('A' <= c && c <= 'F') {
+ return c - 'A' + 10;
+ } else if ('a' <= c && c <= 'f') {
+ return c - 'a' + 10;
+ } else {
+ fprintf(stderr, "Invalid hex: %c\n", c);
+ return -1;
+ }
+}
+
+static int read_data(FILE *f, NTDB_DATA *d, size_t size) {
+ int c, low, high;
+ int i;
+
+ d->dptr = (unsigned char *)malloc(size);
+ if (d->dptr == NULL) {
+ return -1;
+ }
+ d->dsize = size;
+
+ for (i=0; i<size; i++) {
+ c = getc(f);
+ if (c == EOF) {
+ fprintf(stderr, "Unexpected EOF in data\n");
+ return 1;
+ } else if (c == '"') {
+ return 0;
+ } else if (c == '\\') {
+ high = read_hex();
+ if (high < 0) {
+ return -1;
+ }
+ high = high << 4;
+ assert(high == (high & 0xf0));
+ low = read_hex();
+ if (low < 0) {
+ return -1;
+ }
+ assert(low == (low & 0x0f));
+ d->dptr[i] = (low|high);
+ } else {
+ d->dptr[i] = c;
+ }
+ }
+ return 0;
+}
+
+static int swallow(FILE *f, const char *s, int *eof)
+{
+ char line[128];
+
+ if (fgets(line, sizeof(line), f) == NULL) {
+ if (eof != NULL) {
+ *eof = 1;
+ }
+ return -1;
+ }
+ if (strcmp(line, s) != 0) {
+ return -1;
+ }
+ return 0;
+}
+
+static bool read_rec(FILE *f, struct ntdb_context *ntdb, int *eof)
+{
+ int length;
+ NTDB_DATA key, data;
+ bool ret = false;
+ enum NTDB_ERROR e;
+
+ key.dptr = NULL;
+ data.dptr = NULL;
+
+ if (swallow(f, "{\n", eof) == -1) {
+ goto fail;
+ }
+ length = read_linehead(f);
+ if (length == -1) {
+ goto fail;
+ }
+ if (read_data(f, &key, length) == -1) {
+ goto fail;
+ }
+ if (swallow(f, "\"\n", NULL) == -1) {
+ goto fail;
+ }
+ length = read_linehead(f);
+ if (length == -1) {
+ goto fail;
+ }
+ if (read_data(f, &data, length) == -1) {
+ goto fail;
+ }
+ if ((swallow(f, "\"\n", NULL) == -1)
+ || (swallow(f, "}\n", NULL) == -1)) {
+ goto fail;
+ }
+ e = ntdb_store(ntdb, key, data, NTDB_INSERT);
+ if (e != NTDB_SUCCESS) {
+ fprintf(stderr, "NTDB error: %s\n", ntdb_errorstr(e));
+ goto fail;
+ }
+
+ ret = true;
+fail:
+ free(key.dptr);
+ free(data.dptr);
+ return ret;
+}
+
+static int restore_ntdb(const char *fname)
+{
+ struct ntdb_context *ntdb;
+
+ ntdb = ntdb_open(fname, 0, O_RDWR|O_CREAT|O_EXCL, 0666, NULL);
+ if (!ntdb) {
+ perror("ntdb_open");
+ fprintf(stderr, "Failed to open %s\n", fname);
+ return 1;
+ }
+
+ while (1) {
+ int eof = 0;
+ if (!read_rec(stdin, ntdb, &eof)) {
+ if (eof) {
+ break;
+ }
+ return 1;
+ }
+ }
+ if (ntdb_close(ntdb)) {
+ fprintf(stderr, "Error closing ntdb\n");
+ return 1;
+ }
+ fprintf(stderr, "EOF\n");
+ return 0;
+}
+
+int main(int argc, char *argv[])
+{
+ char *fname;
+
+ if (argc < 2) {
+ printf("Usage: %s dbname < tdbdump_output\n", argv[0]);
+ exit(1);
+ }
+
+ fname = argv[1];
+
+ return restore_ntdb(fname);
+}
diff --git a/lib/ntdb/tools/ntdbtool.c b/lib/ntdb/tools/ntdbtool.c
new file mode 100644
index 0000000000..7c1ef7df7a
--- /dev/null
+++ b/lib/ntdb/tools/ntdbtool.c
@@ -0,0 +1,810 @@
+/*
+ Unix SMB/CIFS implementation.
+ Samba database functions
+ Copyright (C) Andrew Tridgell 1999-2000
+ Copyright (C) Paul `Rusty' Russell 2000
+ Copyright (C) Jeremy Allison 2000
+ Copyright (C) Andrew Esh 2001
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "config.h"
+#include "ntdb.h"
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#include <system/filesys.h>
+#include <system/time.h>
+#include <system/locale.h>
+#else
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <string.h>
+#include <stdarg.h>
+#endif
+
+static int do_command(void);
+const char *cmdname;
+char *arg1, *arg2;
+size_t arg1len, arg2len;
+int bIterate = 0;
+char *line;
+NTDB_DATA iterate_kbuf;
+char cmdline[1024];
+static int disable_mmap;
+
+enum commands {
+ CMD_CREATE_NTDB,
+ CMD_OPEN_NTDB,
+ CMD_TRANSACTION_START,
+ CMD_TRANSACTION_COMMIT,
+ CMD_TRANSACTION_CANCEL,
+ CMD_ERASE,
+ CMD_DUMP,
+ CMD_INSERT,
+ CMD_MOVE,
+ CMD_STORE,
+ CMD_SHOW,
+ CMD_KEYS,
+ CMD_HEXKEYS,
+ CMD_DELETE,
+#if 0
+ CMD_LIST_HASH_FREE,
+ CMD_LIST_FREE,
+#endif
+ CMD_INFO,
+ CMD_MMAP,
+ CMD_SPEED,
+ CMD_FIRST,
+ CMD_NEXT,
+ CMD_SYSTEM,
+ CMD_CHECK,
+ CMD_QUIT,
+ CMD_HELP
+};
+
+typedef struct {
+ const char *name;
+ enum commands cmd;
+} COMMAND_TABLE;
+
+COMMAND_TABLE cmd_table[] = {
+ {"create", CMD_CREATE_NTDB},
+ {"open", CMD_OPEN_NTDB},
+#if 0
+ {"transaction_start", CMD_TRANSACTION_START},
+ {"transaction_commit", CMD_TRANSACTION_COMMIT},
+ {"transaction_cancel", CMD_TRANSACTION_CANCEL},
+#endif
+ {"erase", CMD_ERASE},
+ {"dump", CMD_DUMP},
+ {"insert", CMD_INSERT},
+ {"move", CMD_MOVE},
+ {"store", CMD_STORE},
+ {"show", CMD_SHOW},
+ {"keys", CMD_KEYS},
+ {"hexkeys", CMD_HEXKEYS},
+ {"delete", CMD_DELETE},
+#if 0
+ {"list", CMD_LIST_HASH_FREE},
+ {"free", CMD_LIST_FREE},
+#endif
+ {"info", CMD_INFO},
+ {"speed", CMD_SPEED},
+ {"mmap", CMD_MMAP},
+ {"first", CMD_FIRST},
+ {"1", CMD_FIRST},
+ {"next", CMD_NEXT},
+ {"n", CMD_NEXT},
+ {"check", CMD_CHECK},
+ {"quit", CMD_QUIT},
+ {"q", CMD_QUIT},
+ {"!", CMD_SYSTEM},
+ {NULL, CMD_HELP}
+};
+
+struct timeval tp1,tp2;
+
+static void _start_timer(void)
+{
+ gettimeofday(&tp1,NULL);
+}
+
+static double _end_timer(void)
+{
+ gettimeofday(&tp2,NULL);
+ return((tp2.tv_sec - tp1.tv_sec) +
+ (tp2.tv_usec - tp1.tv_usec)*1.0e-6);
+}
+
+static void ntdb_log(struct ntdb_context *ntdb,
+ enum ntdb_log_level level,
+ enum NTDB_ERROR ecode,
+ const char *message,
+ void *data)
+{
+ fprintf(stderr, "ntdb:%s:%s:%s\n",
+ ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+}
+
+/* a ntdb tool for manipulating a ntdb database */
+
+static struct ntdb_context *ntdb;
+
+static int print_rec(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state);
+static int print_key(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state);
+static int print_hexkey(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state);
+
+static void print_asc(const char *buf,int len)
+{
+ int i;
+
+ /* We're probably printing ASCII strings so don't try to display
+ the trailing NULL character. */
+
+ if (buf[len - 1] == 0)
+ len--;
+
+ for (i=0;i<len;i++)
+ printf("%c",isprint(buf[i])?buf[i]:'.');
+}
+
+static void print_data(const char *buf,int len)
+{
+ int i=0;
+ if (len<=0) return;
+ printf("[%03X] ",i);
+ for (i=0;i<len;) {
+ printf("%02X ",(int)((unsigned char)buf[i]));
+ i++;
+ if (i%8 == 0) printf(" ");
+ if (i%16 == 0) {
+ print_asc(&buf[i-16],8); printf(" ");
+ print_asc(&buf[i-8],8); printf("\n");
+ if (i<len) printf("[%03X] ",i);
+ }
+ }
+ if (i%16) {
+ int n;
+
+ n = 16 - (i%16);
+ printf(" ");
+ if (n>8) printf(" ");
+ while (n--) printf(" ");
+
+ n = i%16;
+ if (n > 8) n = 8;
+ print_asc(&buf[i-(i%16)],n); printf(" ");
+ n = (i%16) - n;
+ if (n>0) print_asc(&buf[i-n],n);
+ printf("\n");
+ }
+}
+
+static void help(void)
+{
+ printf("\n"
+"tdbtool: \n"
+" create dbname : create a database\n"
+" open dbname : open an existing database\n"
+" openjh dbname : open an existing database (jenkins hash)\n"
+" transaction_start : start a transaction\n"
+" transaction_commit : commit a transaction\n"
+" transaction_cancel : cancel a transaction\n"
+" erase : erase the database\n"
+" dump : dump the database as strings\n"
+" keys : dump the database keys as strings\n"
+" hexkeys : dump the database keys as hex values\n"
+" info : print summary info about the database\n"
+" insert key data : insert a record\n"
+" move key file : move a record to a destination ntdb\n"
+" store key data : store a record (replace)\n"
+" show key : show a record by key\n"
+" delete key : delete a record by key\n"
+#if 0
+" list : print the database hash table and freelist\n"
+" free : print the database freelist\n"
+#endif
+" check : check the integrity of an opened database\n"
+" speed : perform speed tests on the database\n"
+" ! command : execute system command\n"
+" 1 | first : print the first record\n"
+" n | next : print the next record\n"
+" q | quit : terminate\n"
+" \\n : repeat 'next' command\n"
+"\n");
+}
+
+static void terror(enum NTDB_ERROR err, const char *why)
+{
+ if (err != NTDB_SUCCESS)
+ printf("%s:%s\n", ntdb_errorstr(err), why);
+ else
+ printf("%s\n", why);
+}
+
+static void create_ntdb(const char *tdbname)
+{
+ union ntdb_attribute log_attr;
+ log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+ log_attr.base.next = NULL;
+ log_attr.log.fn = ntdb_log;
+
+ if (ntdb) ntdb_close(ntdb);
+ ntdb = ntdb_open(tdbname, (disable_mmap?NTDB_NOMMAP:0),
+ O_RDWR | O_CREAT | O_TRUNC, 0600, &log_attr);
+ if (!ntdb) {
+ printf("Could not create %s: %s\n", tdbname, strerror(errno));
+ }
+}
+
+static void open_ntdb(const char *tdbname)
+{
+ union ntdb_attribute log_attr;
+ log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+ log_attr.base.next = NULL;
+ log_attr.log.fn = ntdb_log;
+
+ if (ntdb) ntdb_close(ntdb);
+ ntdb = ntdb_open(tdbname, disable_mmap?NTDB_NOMMAP:0, O_RDWR, 0600,
+ &log_attr);
+ if (!ntdb) {
+ printf("Could not open %s: %s\n", tdbname, strerror(errno));
+ }
+}
+
+static void insert_ntdb(char *keyname, size_t keylen, char* data, size_t datalen)
+{
+ NTDB_DATA key, dbuf;
+ enum NTDB_ERROR ecode;
+
+ if ((keyname == NULL) || (keylen == 0)) {
+ terror(NTDB_SUCCESS, "need key");
+ return;
+ }
+
+ key.dptr = (unsigned char *)keyname;
+ key.dsize = keylen;
+ dbuf.dptr = (unsigned char *)data;
+ dbuf.dsize = datalen;
+
+ ecode = ntdb_store(ntdb, key, dbuf, NTDB_INSERT);
+ if (ecode) {
+ terror(ecode, "insert failed");
+ }
+}
+
+static void store_ntdb(char *keyname, size_t keylen, char* data, size_t datalen)
+{
+ NTDB_DATA key, dbuf;
+ enum NTDB_ERROR ecode;
+
+ if ((keyname == NULL) || (keylen == 0)) {
+ terror(NTDB_SUCCESS, "need key");
+ return;
+ }
+
+ if ((data == NULL) || (datalen == 0)) {
+ terror(NTDB_SUCCESS, "need data");
+ return;
+ }
+
+ key.dptr = (unsigned char *)keyname;
+ key.dsize = keylen;
+ dbuf.dptr = (unsigned char *)data;
+ dbuf.dsize = datalen;
+
+ printf("Storing key:\n");
+ print_rec(ntdb, key, dbuf, NULL);
+
+ ecode = ntdb_store(ntdb, key, dbuf, NTDB_REPLACE);
+ if (ecode) {
+ terror(ecode, "store failed");
+ }
+}
+
+static void show_ntdb(char *keyname, size_t keylen)
+{
+ NTDB_DATA key, dbuf;
+ enum NTDB_ERROR ecode;
+
+ if ((keyname == NULL) || (keylen == 0)) {
+ terror(NTDB_SUCCESS, "need key");
+ return;
+ }
+
+ key.dptr = (unsigned char *)keyname;
+ key.dsize = keylen;
+
+ ecode = ntdb_fetch(ntdb, key, &dbuf);
+ if (ecode) {
+ terror(ecode, "fetch failed");
+ return;
+ }
+
+ print_rec(ntdb, key, dbuf, NULL);
+
+ free( dbuf.dptr );
+}
+
+static void delete_ntdb(char *keyname, size_t keylen)
+{
+ NTDB_DATA key;
+ enum NTDB_ERROR ecode;
+
+ if ((keyname == NULL) || (keylen == 0)) {
+ terror(NTDB_SUCCESS, "need key");
+ return;
+ }
+
+ key.dptr = (unsigned char *)keyname;
+ key.dsize = keylen;
+
+ ecode = ntdb_delete(ntdb, key);
+ if (ecode) {
+ terror(ecode, "delete failed");
+ }
+}
+
+static void move_rec(char *keyname, size_t keylen, char* tdbname)
+{
+ NTDB_DATA key, dbuf;
+ struct ntdb_context *dst_ntdb;
+ enum NTDB_ERROR ecode;
+
+ if ((keyname == NULL) || (keylen == 0)) {
+ terror(NTDB_SUCCESS, "need key");
+ return;
+ }
+
+ if ( !tdbname ) {
+ terror(NTDB_SUCCESS, "need destination ntdb name");
+ return;
+ }
+
+ key.dptr = (unsigned char *)keyname;
+ key.dsize = keylen;
+
+ ecode = ntdb_fetch(ntdb, key, &dbuf);
+ if (ecode) {
+ terror(ecode, "fetch failed");
+ return;
+ }
+
+ print_rec(ntdb, key, dbuf, NULL);
+
+ dst_ntdb = ntdb_open(tdbname, 0, O_RDWR, 0600, NULL);
+ if ( !dst_ntdb ) {
+ terror(NTDB_SUCCESS, "unable to open destination ntdb");
+ return;
+ }
+
+ ecode = ntdb_store( dst_ntdb, key, dbuf, NTDB_REPLACE);
+ if (ecode)
+ terror(ecode, "failed to move record");
+ else
+ printf("record moved\n");
+
+ ntdb_close( dst_ntdb );
+}
+
+static int print_rec(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ printf("\nkey %d bytes\n", (int)key.dsize);
+ print_asc((const char *)key.dptr, key.dsize);
+ printf("\ndata %d bytes\n", (int)dbuf.dsize);
+ print_data((const char *)dbuf.dptr, dbuf.dsize);
+ return 0;
+}
+
+static int print_key(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ printf("key %d bytes: ", (int)key.dsize);
+ print_asc((const char *)key.dptr, key.dsize);
+ printf("\n");
+ return 0;
+}
+
+static int print_hexkey(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ printf("key %d bytes\n", (int)key.dsize);
+ print_data((const char *)key.dptr, key.dsize);
+ printf("\n");
+ return 0;
+}
+
+static int total_bytes;
+
+static int traverse_fn(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf, void *state)
+{
+ total_bytes += dbuf.dsize;
+ return 0;
+}
+
+static void info_ntdb(void)
+{
+ enum NTDB_ERROR ecode;
+ char *summary;
+
+ ecode = ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &summary);
+
+ if (ecode) {
+ terror(ecode, "Getting summary");
+ } else {
+ printf("%s", summary);
+ free(summary);
+ }
+}
+
+static void speed_ntdb(const char *tlimit)
+{
+ unsigned timelimit = tlimit?atoi(tlimit):0;
+ double t;
+ int ops;
+ if (timelimit == 0) timelimit = 5;
+
+ ops = 0;
+ printf("Testing store speed for %u seconds\n", timelimit);
+ _start_timer();
+ do {
+ long int r = random();
+ NTDB_DATA key, dbuf;
+ key = ntdb_mkdata("store test", strlen("store test"));
+ dbuf.dptr = (unsigned char *)&r;
+ dbuf.dsize = sizeof(r);
+ ntdb_store(ntdb, key, dbuf, NTDB_REPLACE);
+ t = _end_timer();
+ ops++;
+ } while (t < timelimit);
+ printf("%10.3f ops/sec\n", ops/t);
+
+ ops = 0;
+ printf("Testing fetch speed for %u seconds\n", timelimit);
+ _start_timer();
+ do {
+ long int r = random();
+ NTDB_DATA key, dbuf;
+ key = ntdb_mkdata("store test", strlen("store test"));
+ dbuf.dptr = (unsigned char *)&r;
+ dbuf.dsize = sizeof(r);
+ ntdb_fetch(ntdb, key, &dbuf);
+ t = _end_timer();
+ ops++;
+ } while (t < timelimit);
+ printf("%10.3f ops/sec\n", ops/t);
+
+ ops = 0;
+ printf("Testing transaction speed for %u seconds\n", timelimit);
+ _start_timer();
+ do {
+ long int r = random();
+ NTDB_DATA key, dbuf;
+ key = ntdb_mkdata("transaction test", strlen("transaction test"));
+ dbuf.dptr = (unsigned char *)&r;
+ dbuf.dsize = sizeof(r);
+ ntdb_transaction_start(ntdb);
+ ntdb_store(ntdb, key, dbuf, NTDB_REPLACE);
+ ntdb_transaction_commit(ntdb);
+ t = _end_timer();
+ ops++;
+ } while (t < timelimit);
+ printf("%10.3f ops/sec\n", ops/t);
+
+ ops = 0;
+ printf("Testing traverse speed for %u seconds\n", timelimit);
+ _start_timer();
+ do {
+ ntdb_traverse(ntdb, traverse_fn, NULL);
+ t = _end_timer();
+ ops++;
+ } while (t < timelimit);
+ printf("%10.3f ops/sec\n", ops/t);
+}
+
+static void toggle_mmap(void)
+{
+ disable_mmap = !disable_mmap;
+ if (disable_mmap) {
+ printf("mmap is disabled\n");
+ } else {
+ printf("mmap is enabled\n");
+ }
+}
+
+static char *ntdb_getline(const char *prompt)
+{
+ static char thisline[1024];
+ char *p;
+ fputs(prompt, stdout);
+ thisline[0] = 0;
+ p = fgets(thisline, sizeof(thisline)-1, stdin);
+ if (p) p = strchr(p, '\n');
+ if (p) *p = 0;
+ return p?thisline:NULL;
+}
+
+static int do_delete_fn(struct ntdb_context *the_ntdb, NTDB_DATA key, NTDB_DATA dbuf,
+ void *state)
+{
+ return ntdb_delete(the_ntdb, key);
+}
+
+static void first_record(struct ntdb_context *the_ntdb, NTDB_DATA *pkey)
+{
+ NTDB_DATA dbuf;
+ enum NTDB_ERROR ecode;
+ ecode = ntdb_firstkey(the_ntdb, pkey);
+ if (!ecode)
+ ecode = ntdb_fetch(the_ntdb, *pkey, &dbuf);
+ if (ecode) terror(ecode, "fetch failed");
+ else {
+ print_rec(the_ntdb, *pkey, dbuf, NULL);
+ }
+}
+
+static void next_record(struct ntdb_context *the_ntdb, NTDB_DATA *pkey)
+{
+ NTDB_DATA dbuf;
+ enum NTDB_ERROR ecode;
+ ecode = ntdb_nextkey(the_ntdb, pkey);
+
+ if (!ecode)
+ ecode = ntdb_fetch(the_ntdb, *pkey, &dbuf);
+ if (ecode)
+ terror(ecode, "fetch failed");
+ else
+ print_rec(the_ntdb, *pkey, dbuf, NULL);
+}
+
+static void check_db(struct ntdb_context *the_ntdb)
+{
+ if (!the_ntdb) {
+ printf("Error: No database opened!\n");
+ } else {
+ if (ntdb_check(the_ntdb, NULL, NULL) != 0)
+ printf("Integrity check for the opened database failed.\n");
+ else
+ printf("Database integrity is OK.\n");
+ }
+}
+
+static int do_command(void)
+{
+ COMMAND_TABLE *ctp = cmd_table;
+ enum commands mycmd = CMD_HELP;
+ int cmd_len;
+
+ if (cmdname && strlen(cmdname) == 0) {
+ mycmd = CMD_NEXT;
+ } else {
+ while (ctp->name) {
+ cmd_len = strlen(ctp->name);
+ if (strncmp(ctp->name,cmdname,cmd_len) == 0) {
+ mycmd = ctp->cmd;
+ break;
+ }
+ ctp++;
+ }
+ }
+
+ switch (mycmd) {
+ case CMD_CREATE_NTDB:
+ bIterate = 0;
+ create_ntdb(arg1);
+ return 0;
+ case CMD_OPEN_NTDB:
+ bIterate = 0;
+ open_ntdb(arg1);
+ return 0;
+ case CMD_SYSTEM:
+ /* Shell command */
+ if (system(arg1) == -1) {
+ terror(NTDB_SUCCESS, "system() call failed\n");
+ }
+ return 0;
+ case CMD_QUIT:
+ return 1;
+ default:
+ /* all the rest require a open database */
+ if (!ntdb) {
+ bIterate = 0;
+ terror(NTDB_SUCCESS, "database not open");
+ help();
+ return 0;
+ }
+ switch (mycmd) {
+ case CMD_TRANSACTION_START:
+ bIterate = 0;
+ ntdb_transaction_start(ntdb);
+ return 0;
+ case CMD_TRANSACTION_COMMIT:
+ bIterate = 0;
+ ntdb_transaction_commit(ntdb);
+ return 0;
+ case CMD_TRANSACTION_CANCEL:
+ bIterate = 0;
+ ntdb_transaction_cancel(ntdb);
+ return 0;
+ case CMD_ERASE:
+ bIterate = 0;
+ ntdb_traverse(ntdb, do_delete_fn, NULL);
+ return 0;
+ case CMD_DUMP:
+ bIterate = 0;
+ ntdb_traverse(ntdb, print_rec, NULL);
+ return 0;
+ case CMD_INSERT:
+ bIterate = 0;
+ insert_ntdb(arg1, arg1len,arg2,arg2len);
+ return 0;
+ case CMD_MOVE:
+ bIterate = 0;
+ move_rec(arg1,arg1len,arg2);
+ return 0;
+ case CMD_STORE:
+ bIterate = 0;
+ store_ntdb(arg1,arg1len,arg2,arg2len);
+ return 0;
+ case CMD_SHOW:
+ bIterate = 0;
+ show_ntdb(arg1, arg1len);
+ return 0;
+ case CMD_KEYS:
+ ntdb_traverse(ntdb, print_key, NULL);
+ return 0;
+ case CMD_HEXKEYS:
+ ntdb_traverse(ntdb, print_hexkey, NULL);
+ return 0;
+ case CMD_DELETE:
+ bIterate = 0;
+ delete_ntdb(arg1,arg1len);
+ return 0;
+#if 0
+ case CMD_LIST_HASH_FREE:
+ ntdb_dump_all(ntdb);
+ return 0;
+ case CMD_LIST_FREE:
+ ntdb_printfreelist(ntdb);
+ return 0;
+#endif
+ case CMD_INFO:
+ info_ntdb();
+ return 0;
+ case CMD_SPEED:
+ speed_ntdb(arg1);
+ return 0;
+ case CMD_MMAP:
+ toggle_mmap();
+ return 0;
+ case CMD_FIRST:
+ bIterate = 1;
+ first_record(ntdb, &iterate_kbuf);
+ return 0;
+ case CMD_NEXT:
+ if (bIterate)
+ next_record(ntdb, &iterate_kbuf);
+ return 0;
+ case CMD_CHECK:
+ check_db(ntdb);
+ return 0;
+ case CMD_HELP:
+ help();
+ return 0;
+ case CMD_CREATE_NTDB:
+ case CMD_OPEN_NTDB:
+ case CMD_SYSTEM:
+ case CMD_QUIT:
+ /*
+ * unhandled commands. cases included here to avoid compiler
+ * warnings.
+ */
+ return 0;
+ }
+ }
+
+ return 0;
+}
+
+static char *convert_string(char *instring, size_t *sizep)
+{
+ size_t length = 0;
+ char *outp, *inp;
+ char temp[3];
+
+ outp = inp = instring;
+
+ while (*inp) {
+ if (*inp == '\\') {
+ inp++;
+ if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+ temp[0] = *inp++;
+ temp[1] = '\0';
+ if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+ temp[1] = *inp++;
+ temp[2] = '\0';
+ }
+ *outp++ = (char)strtol((const char *)temp,NULL,16);
+ } else {
+ *outp++ = *inp++;
+ }
+ } else {
+ *outp++ = *inp++;
+ }
+ length++;
+ }
+ *sizep = length;
+ return instring;
+}
+
+int main(int argc, char *argv[])
+{
+ cmdname = "";
+ arg1 = NULL;
+ arg1len = 0;
+ arg2 = NULL;
+ arg2len = 0;
+
+ if (argv[1]) {
+ cmdname = "open";
+ arg1 = argv[1];
+ do_command();
+ cmdname = "";
+ arg1 = NULL;
+ }
+
+ switch (argc) {
+ case 1:
+ case 2:
+ /* Interactive mode */
+ while ((cmdname = ntdb_getline("ntdb> "))) {
+ arg2 = arg1 = NULL;
+ if ((arg1 = strchr((const char *)cmdname,' ')) != NULL) {
+ arg1++;
+ arg2 = arg1;
+ while (*arg2) {
+ if (*arg2 == ' ') {
+ *arg2++ = '\0';
+ break;
+ }
+ if ((*arg2++ == '\\') && (*arg2 == ' ')) {
+ arg2++;
+ }
+ }
+ }
+ if (arg1) arg1 = convert_string(arg1,&arg1len);
+ if (arg2) arg2 = convert_string(arg2,&arg2len);
+ if (do_command()) break;
+ }
+ break;
+ case 5:
+ arg2 = convert_string(argv[4],&arg2len);
+ case 4:
+ arg1 = convert_string(argv[3],&arg1len);
+ case 3:
+ cmdname = argv[2];
+ default:
+ do_command();
+ break;
+ }
+
+ if (ntdb) ntdb_close(ntdb);
+
+ return 0;
+}
diff --git a/lib/ntdb/tools/ntdbtorture.c b/lib/ntdb/tools/ntdbtorture.c
new file mode 100644
index 0000000000..c7b249db06
--- /dev/null
+++ b/lib/ntdb/tools/ntdbtorture.c
@@ -0,0 +1,529 @@
+/* this tests ntdb by doing lots of ops from several simultaneous
+ writers - that stresses the locking code.
+*/
+
+#include "config.h"
+#include "ntdb.h"
+#include <ccan/err/err.h>
+#ifdef HAVE_LIBREPLACE
+#include <replace.h>
+#else
+#include <stdlib.h>
+#include <getopt.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <time.h>
+#include <sys/wait.h>
+#endif
+
+//#define REOPEN_PROB 30
+#define DELETE_PROB 8
+#define STORE_PROB 4
+#define APPEND_PROB 6
+#define TRANSACTION_PROB 10
+#define TRANSACTION_PREPARE_PROB 2
+#define LOCKSTORE_PROB 5
+#define TRAVERSE_PROB 20
+#define TRAVERSE_MOD_PROB 100
+#define TRAVERSE_ABORT_PROB 500
+#define CULL_PROB 100
+#define KEYLEN 3
+#define DATALEN 100
+
+static struct ntdb_context *db;
+static int in_transaction;
+static int in_traverse;
+static int error_count;
+#if TRANSACTION_PROB
+static int always_transaction = 0;
+#endif
+static int loopnum;
+static int count_pipe;
+static union ntdb_attribute log_attr;
+static union ntdb_attribute seed_attr;
+
+static void ntdb_log(struct ntdb_context *ntdb,
+ enum ntdb_log_level level,
+ enum NTDB_ERROR ecode,
+ const char *message,
+ void *data)
+{
+ printf("ntdb:%s:%s:%s\n",
+ ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+ fflush(stdout);
+#if 0
+ {
+ char str[200];
+ signal(SIGUSR1, SIG_IGN);
+ sprintf(str,"xterm -e gdb /proc/%d/exe %d", getpid(), getpid());
+ system(str);
+ }
+#endif
+}
+
+#include "../private.h"
+
+static void segv_handler(int sig, siginfo_t *info, void *p)
+{
+ char string[100];
+
+ sprintf(string, "%u: death at %p (map_ptr %p, map_size %zu)\n",
+ getpid(), info->si_addr, db->file->map_ptr,
+ (size_t)db->file->map_size);
+ if (write(2, string, strlen(string)) > 0)
+ sleep(60);
+ _exit(11);
+}
+
+static void fatal(struct ntdb_context *ntdb, const char *why)
+{
+ fprintf(stderr, "%u:%s:%s\n", getpid(), why,
+ ntdb ? ntdb_errorstr(ntdb_error(ntdb)) : "(no ntdb)");
+ error_count++;
+}
+
+static char *randbuf(int len)
+{
+ char *buf;
+ int i;
+ buf = (char *)malloc(len+1);
+
+ for (i=0;i<len;i++) {
+ buf[i] = 'a' + (rand() % 26);
+ }
+ buf[i] = 0;
+ return buf;
+}
+
+static void addrec_db(void);
+static int modify_traverse(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf,
+ void *state)
+{
+#if CULL_PROB
+ if (random() % CULL_PROB == 0) {
+ ntdb_delete(ntdb, key);
+ }
+#endif
+
+#if TRAVERSE_MOD_PROB
+ if (random() % TRAVERSE_MOD_PROB == 0) {
+ addrec_db();
+ }
+#endif
+
+#if TRAVERSE_ABORT_PROB
+ if (random() % TRAVERSE_ABORT_PROB == 0)
+ return 1;
+#endif
+
+ return 0;
+}
+
+static void addrec_db(void)
+{
+ int klen, dlen;
+ char *k, *d;
+ NTDB_DATA key, data;
+
+ klen = 1 + (rand() % KEYLEN);
+ dlen = 1 + (rand() % DATALEN);
+
+ k = randbuf(klen);
+ d = randbuf(dlen);
+
+ key.dptr = (unsigned char *)k;
+ key.dsize = klen+1;
+
+ data.dptr = (unsigned char *)d;
+ data.dsize = dlen+1;
+
+#if REOPEN_PROB
+ if (in_traverse == 0 && in_transaction == 0 && random() % REOPEN_PROB == 0) {
+ ntdb_reopen_all(0);
+ goto next;
+ }
+#endif
+
+#if TRANSACTION_PROB
+ if (in_traverse == 0 && in_transaction == 0 && (always_transaction || random() % TRANSACTION_PROB == 0)) {
+ if (ntdb_transaction_start(db) != 0) {
+ fatal(db, "ntdb_transaction_start failed");
+ }
+ in_transaction++;
+ goto next;
+ }
+ if (in_traverse == 0 && in_transaction && random() % TRANSACTION_PROB == 0) {
+ if (random() % TRANSACTION_PREPARE_PROB == 0) {
+ if (ntdb_transaction_prepare_commit(db) != 0) {
+ fatal(db, "ntdb_transaction_prepare_commit failed");
+ }
+ }
+ if (ntdb_transaction_commit(db) != 0) {
+ fatal(db, "ntdb_transaction_commit failed");
+ }
+ in_transaction--;
+ goto next;
+ }
+
+ if (in_traverse == 0 && in_transaction && random() % TRANSACTION_PROB == 0) {
+ ntdb_transaction_cancel(db);
+ in_transaction--;
+ goto next;
+ }
+#endif
+
+#if DELETE_PROB
+ if (random() % DELETE_PROB == 0) {
+ ntdb_delete(db, key);
+ goto next;
+ }
+#endif
+
+#if STORE_PROB
+ if (random() % STORE_PROB == 0) {
+ if (ntdb_store(db, key, data, NTDB_REPLACE) != 0) {
+ fatal(db, "ntdb_store failed");
+ }
+ goto next;
+ }
+#endif
+
+#if APPEND_PROB
+ if (random() % APPEND_PROB == 0) {
+ if (ntdb_append(db, key, data) != 0) {
+ fatal(db, "ntdb_append failed");
+ }
+ goto next;
+ }
+#endif
+
+#if LOCKSTORE_PROB
+ if (random() % LOCKSTORE_PROB == 0) {
+ ntdb_chainlock(db, key);
+ if (ntdb_fetch(db, key, &data) != NTDB_SUCCESS) {
+ data.dsize = 0;
+ data.dptr = NULL;
+ }
+ if (ntdb_store(db, key, data, NTDB_REPLACE) != 0) {
+ fatal(db, "ntdb_store failed");
+ }
+ if (data.dptr) free(data.dptr);
+ ntdb_chainunlock(db, key);
+ goto next;
+ }
+#endif
+
+#if TRAVERSE_PROB
+ /* FIXME: recursive traverses break transactions? */
+ if (in_traverse == 0 && random() % TRAVERSE_PROB == 0) {
+ in_traverse++;
+ ntdb_traverse(db, modify_traverse, NULL);
+ in_traverse--;
+ goto next;
+ }
+#endif
+
+ if (ntdb_fetch(db, key, &data) == NTDB_SUCCESS)
+ free(data.dptr);
+
+next:
+ free(k);
+ free(d);
+}
+
+static int traverse_fn(struct ntdb_context *ntdb, NTDB_DATA key, NTDB_DATA dbuf,
+ void *state)
+{
+ ntdb_delete(ntdb, key);
+ return 0;
+}
+
+static void usage(void)
+{
+ printf("Usage: ntdbtorture"
+#if TRANSACTION_PROB
+ " [-t]"
+#endif
+ " [-k] [-n NUM_PROCS] [-l NUM_LOOPS] [-s SEED] [-S]\n");
+ exit(0);
+}
+
+static void send_count_and_suicide(int sig)
+{
+ /* This ensures our successor can continue where we left off. */
+ if (write(count_pipe, &loopnum, sizeof(loopnum)) != sizeof(loopnum))
+ exit(2);
+ /* This gives a unique signature. */
+ kill(getpid(), SIGUSR2);
+}
+
+static int run_child(const char *filename, int i, int seed, unsigned num_loops,
+ unsigned start, int ntdb_flags)
+{
+ struct sigaction act = { .sa_sigaction = segv_handler,
+ .sa_flags = SA_SIGINFO };
+ sigaction(11, &act, NULL);
+
+ db = ntdb_open(filename, ntdb_flags, O_RDWR | O_CREAT, 0600,
+ &log_attr);
+ if (!db) {
+ fatal(NULL, "db open failed");
+ }
+
+#if 0
+ if (i == 0) {
+ printf("pid %i\n", getpid());
+ sleep(9);
+ } else
+ sleep(10);
+#endif
+
+ srand(seed + i);
+ srandom(seed + i);
+
+ /* Set global, then we're ready to handle being killed. */
+ loopnum = start;
+ signal(SIGUSR1, send_count_and_suicide);
+
+ for (;loopnum<num_loops && error_count == 0;loopnum++) {
+ addrec_db();
+ }
+
+ if (error_count == 0) {
+ ntdb_traverse(db, NULL, NULL);
+#if TRANSACTION_PROB
+ if (always_transaction) {
+ while (in_transaction) {
+ ntdb_transaction_cancel(db);
+ in_transaction--;
+ }
+ if (ntdb_transaction_start(db) != 0)
+ fatal(db, "ntdb_transaction_start failed");
+ }
+#endif
+ ntdb_traverse(db, traverse_fn, NULL);
+ ntdb_traverse(db, traverse_fn, NULL);
+
+#if TRANSACTION_PROB
+ if (always_transaction) {
+ if (ntdb_transaction_commit(db) != 0)
+ fatal(db, "ntdb_transaction_commit failed");
+ }
+#endif
+ }
+
+ ntdb_close(db);
+
+ return (error_count < 100 ? error_count : 100);
+}
+
+static char *test_path(const char *filename)
+{
+ const char *prefix = getenv("TEST_DATA_PREFIX");
+
+ if (prefix) {
+ char *path = NULL;
+ int ret;
+
+ ret = asprintf(&path, "%s/%s", prefix, filename);
+ if (ret == -1) {
+ return NULL;
+ }
+ return path;
+ }
+
+ return strdup(filename);
+}
+
+int main(int argc, char * const *argv)
+{
+ int i, seed = -1;
+ int num_loops = 5000;
+ int num_procs = 3;
+ int c, pfds[2];
+ extern char *optarg;
+ pid_t *pids;
+ int kill_random = 0;
+ int *done;
+ int ntdb_flags = NTDB_DEFAULT;
+ char *test_ntdb;
+
+ log_attr.base.attr = NTDB_ATTRIBUTE_LOG;
+ log_attr.base.next = &seed_attr;
+ log_attr.log.fn = ntdb_log;
+ seed_attr.base.attr = NTDB_ATTRIBUTE_SEED;
+ seed_attr.base.next = NULL;
+
+ while ((c = getopt(argc, argv, "n:l:s:thkS")) != -1) {
+ switch (c) {
+ case 'n':
+ num_procs = strtol(optarg, NULL, 0);
+ break;
+ case 'l':
+ num_loops = strtol(optarg, NULL, 0);
+ break;
+ case 's':
+ seed = strtol(optarg, NULL, 0);
+ break;
+ case 'S':
+ ntdb_flags = NTDB_NOSYNC;
+ break;
+ case 't':
+#if TRANSACTION_PROB
+ always_transaction = 1;
+#else
+ fprintf(stderr, "Transactions not supported\n");
+ usage();
+#endif
+ break;
+ case 'k':
+ kill_random = 1;
+ break;
+ default:
+ usage();
+ }
+ }
+
+ test_ntdb = test_path("torture.ntdb");
+
+ unlink(test_ntdb);
+
+ if (seed == -1) {
+ seed = (getpid() + time(NULL)) & 0x7FFFFFFF;
+ }
+ seed_attr.seed.seed = (((uint64_t)seed) << 32) | seed;
+
+ if (num_procs == 1 && !kill_random) {
+ /* Don't fork for this case, makes debugging easier. */
+ error_count = run_child(test_ntdb, 0, seed, num_loops, 0,
+ ntdb_flags);
+ goto done;
+ }
+
+ pids = (pid_t *)calloc(sizeof(pid_t), num_procs);
+ done = (int *)calloc(sizeof(int), num_procs);
+
+ if (pipe(pfds) != 0) {
+ perror("Creating pipe");
+ exit(1);
+ }
+ count_pipe = pfds[1];
+
+ for (i=0;i<num_procs;i++) {
+ if ((pids[i]=fork()) == 0) {
+ close(pfds[0]);
+ if (i == 0) {
+ printf("testing with %d processes, %d loops, seed=%d%s\n",
+ num_procs, num_loops, seed,
+#if TRANSACTION_PROB
+ always_transaction ? " (all within transactions)" : ""
+#else
+ ""
+#endif
+ );
+ }
+ exit(run_child(test_ntdb, i, seed, num_loops, 0,
+ ntdb_flags));
+ }
+ }
+
+ while (num_procs) {
+ int status, j;
+ pid_t pid;
+
+ if (error_count != 0) {
+ /* try and stop the test on any failure */
+ for (j=0;j<num_procs;j++) {
+ if (pids[j] != 0) {
+ kill(pids[j], SIGTERM);
+ }
+ }
+ }
+
+ pid = waitpid(-1, &status, kill_random ? WNOHANG : 0);
+ if (pid == 0) {
+ struct timespec ts;
+
+ /* Sleep for 1/10 second. */
+ ts.tv_sec = 0;
+ ts.tv_nsec = 100000000;
+ nanosleep(&ts, NULL);
+
+ /* Kill someone. */
+ kill(pids[random() % num_procs], SIGUSR1);
+ continue;
+ }
+
+ if (pid == -1) {
+ perror("failed to wait for child\n");
+ exit(1);
+ }
+
+ for (j=0;j<num_procs;j++) {
+ if (pids[j] == pid) break;
+ }
+ if (j == num_procs) {
+ printf("unknown child %d exited!?\n", (int)pid);
+ exit(1);
+ }
+ if (WIFSIGNALED(status)) {
+ if (WTERMSIG(status) == SIGUSR2
+ || WTERMSIG(status) == SIGUSR1) {
+ /* SIGUSR2 means they wrote to pipe. */
+ if (WTERMSIG(status) == SIGUSR2) {
+ if (read(pfds[0], &done[j],
+ sizeof(done[j]))
+ != sizeof(done[j]))
+ err(1,
+ "Short read from child?");
+ }
+ pids[j] = fork();
+ if (pids[j] == 0)
+ exit(run_child(test_ntdb, j, seed,
+ num_loops, done[j],
+ ntdb_flags));
+ printf("Restarting child %i for %u-%u\n",
+ j, done[j], num_loops);
+ continue;
+ }
+ printf("child %d exited with signal %d\n",
+ (int)pid, WTERMSIG(status));
+ error_count++;
+ } else {
+ if (WEXITSTATUS(status) != 0) {
+ printf("child %d exited with status %d\n",
+ (int)pid, WEXITSTATUS(status));
+ error_count++;
+ }
+ }
+ memmove(&pids[j], &pids[j+1],
+ (num_procs - j - 1)*sizeof(pids[0]));
+ num_procs--;
+ }
+
+ free(pids);
+
+done:
+ if (error_count == 0) {
+ db = ntdb_open(test_ntdb, NTDB_DEFAULT, O_RDWR | O_CREAT,
+ 0600, &log_attr);
+ if (!db) {
+ fatal(db, "db open failed");
+ exit(1);
+ }
+ if (ntdb_check(db, NULL, NULL) != 0) {
+ fatal(db, "db check failed");
+ exit(1);
+ }
+ ntdb_close(db);
+ printf("OK\n");
+ }
+
+ free(test_ntdb);
+ return error_count;
+}
diff --git a/lib/ntdb/tools/speed.c b/lib/ntdb/tools/speed.c
new file mode 100644
index 0000000000..868494b898
--- /dev/null
+++ b/lib/ntdb/tools/speed.c
@@ -0,0 +1,443 @@
+/* Simple speed test for NTDB */
+#include <ccan/err/err.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include "ntdb.h"
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeval *start,
+ const struct timeval *stop,
+ unsigned int num)
+{
+ struct timeval diff;
+
+ timersub(stop, start, &diff);
+
+ /* Floating point is more accurate here. */
+ return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
+ / num * 1000;
+}
+
+static size_t file_size(void)
+{
+ struct stat st;
+
+ if (stat("/tmp/speed.ntdb", &st) != 0)
+ return -1;
+ return st.st_size;
+}
+
+static int count_record(struct ntdb_context *ntdb,
+ NTDB_DATA key, NTDB_DATA data, void *p)
+{
+ int *total = p;
+ *total += *(int *)data.dptr;
+ return 0;
+}
+
+static void dump_and_clear_stats(struct ntdb_context **ntdb,
+ int flags,
+ union ntdb_attribute *attr)
+{
+ union ntdb_attribute stats;
+ enum NTDB_ERROR ecode;
+
+ stats.base.attr = NTDB_ATTRIBUTE_STATS;
+ stats.stats.size = sizeof(stats.stats);
+ ecode = ntdb_get_attribute(*ntdb, &stats);
+ if (ecode != NTDB_SUCCESS)
+ errx(1, "Getting stats: %s", ntdb_errorstr(ecode));
+
+ printf("allocs = %llu\n",
+ (unsigned long long)stats.stats.allocs);
+ printf(" alloc_subhash = %llu\n",
+ (unsigned long long)stats.stats.alloc_subhash);
+ printf(" alloc_chain = %llu\n",
+ (unsigned long long)stats.stats.alloc_chain);
+ printf(" alloc_bucket_exact = %llu\n",
+ (unsigned long long)stats.stats.alloc_bucket_exact);
+ printf(" alloc_bucket_max = %llu\n",
+ (unsigned long long)stats.stats.alloc_bucket_max);
+ printf(" alloc_leftover = %llu\n",
+ (unsigned long long)stats.stats.alloc_leftover);
+ printf(" alloc_coalesce_tried = %llu\n",
+ (unsigned long long)stats.stats.alloc_coalesce_tried);
+ printf(" alloc_coalesce_iterate_clash = %llu\n",
+ (unsigned long long)stats.stats.alloc_coalesce_iterate_clash);
+ printf(" alloc_coalesce_lockfail = %llu\n",
+ (unsigned long long)stats.stats.alloc_coalesce_lockfail);
+ printf(" alloc_coalesce_race = %llu\n",
+ (unsigned long long)stats.stats.alloc_coalesce_race);
+ printf(" alloc_coalesce_succeeded = %llu\n",
+ (unsigned long long)stats.stats.alloc_coalesce_succeeded);
+ printf(" alloc_coalesce_num_merged = %llu\n",
+ (unsigned long long)stats.stats.alloc_coalesce_num_merged);
+ printf("compares = %llu\n",
+ (unsigned long long)stats.stats.compares);
+ printf(" compare_wrong_bucket = %llu\n",
+ (unsigned long long)stats.stats.compare_wrong_bucket);
+ printf(" compare_wrong_offsetbits = %llu\n",
+ (unsigned long long)stats.stats.compare_wrong_offsetbits);
+ printf(" compare_wrong_keylen = %llu\n",
+ (unsigned long long)stats.stats.compare_wrong_keylen);
+ printf(" compare_wrong_rechash = %llu\n",
+ (unsigned long long)stats.stats.compare_wrong_rechash);
+ printf(" compare_wrong_keycmp = %llu\n",
+ (unsigned long long)stats.stats.compare_wrong_keycmp);
+ printf("transactions = %llu\n",
+ (unsigned long long)stats.stats.transactions);
+ printf(" transaction_cancel = %llu\n",
+ (unsigned long long)stats.stats.transaction_cancel);
+ printf(" transaction_nest = %llu\n",
+ (unsigned long long)stats.stats.transaction_nest);
+ printf(" transaction_expand_file = %llu\n",
+ (unsigned long long)stats.stats.transaction_expand_file);
+ printf(" transaction_read_direct = %llu\n",
+ (unsigned long long)stats.stats.transaction_read_direct);
+ printf(" transaction_read_direct_fail = %llu\n",
+ (unsigned long long)stats.stats.transaction_read_direct_fail);
+ printf(" transaction_write_direct = %llu\n",
+ (unsigned long long)stats.stats.transaction_write_direct);
+ printf(" transaction_write_direct_fail = %llu\n",
+ (unsigned long long)stats.stats.transaction_write_direct_fail);
+ printf("expands = %llu\n",
+ (unsigned long long)stats.stats.expands);
+ printf("frees = %llu\n",
+ (unsigned long long)stats.stats.frees);
+ printf("locks = %llu\n",
+ (unsigned long long)stats.stats.locks);
+ printf(" lock_lowlevel = %llu\n",
+ (unsigned long long)stats.stats.lock_lowlevel);
+ printf(" lock_nonblock = %llu\n",
+ (unsigned long long)stats.stats.lock_nonblock);
+ printf(" lock_nonblock_fail = %llu\n",
+ (unsigned long long)stats.stats.lock_nonblock_fail);
+
+ /* Now clear. */
+ ntdb_close(*ntdb);
+ *ntdb = ntdb_open("/tmp/speed.ntdb", flags, O_RDWR, 0, attr);
+}
+
+static void ntdb_log(struct ntdb_context *ntdb,
+ enum ntdb_log_level level,
+ enum NTDB_ERROR ecode,
+ const char *message,
+ void *data)
+{
+ fprintf(stderr, "ntdb:%s:%s:%s\n",
+ ntdb_name(ntdb), ntdb_errorstr(ecode), message);
+}
+
+int main(int argc, char *argv[])
+{
+ unsigned int i, j, num = 1000, stage = 0, stopat = -1;
+ int flags = NTDB_DEFAULT;
+ bool transaction = false, summary = false;
+ NTDB_DATA key, data;
+ struct ntdb_context *ntdb;
+ struct timeval start, stop;
+ union ntdb_attribute seed, log;
+ bool do_stats = false;
+ enum NTDB_ERROR ecode;
+
+ /* Try to keep benchmarks even. */
+ seed.base.attr = NTDB_ATTRIBUTE_SEED;
+ seed.base.next = NULL;
+ seed.seed.seed = 0;
+
+ log.base.attr = NTDB_ATTRIBUTE_LOG;
+ log.base.next = &seed;
+ log.log.fn = ntdb_log;
+
+ if (argv[1] && strcmp(argv[1], "--internal") == 0) {
+ flags = NTDB_INTERNAL;
+ argc--;
+ argv++;
+ }
+ if (argv[1] && strcmp(argv[1], "--transaction") == 0) {
+ transaction = true;
+ argc--;
+ argv++;
+ }
+ if (argv[1] && strcmp(argv[1], "--no-sync") == 0) {
+ flags |= NTDB_NOSYNC;
+ argc--;
+ argv++;
+ }
+ if (argv[1] && strcmp(argv[1], "--summary") == 0) {
+ summary = true;
+ argc--;
+ argv++;
+ }
+ if (argv[1] && strcmp(argv[1], "--stats") == 0) {
+ do_stats = true;
+ argc--;
+ argv++;
+ }
+
+ ntdb = ntdb_open("/tmp/speed.ntdb", flags, O_RDWR|O_CREAT|O_TRUNC,
+ 0600, &log);
+ if (!ntdb)
+ err(1, "Opening /tmp/speed.ntdb");
+
+ key.dptr = (void *)&i;
+ key.dsize = sizeof(i);
+ data = key;
+
+ if (argv[1]) {
+ num = atoi(argv[1]);
+ argv++;
+ argc--;
+ }
+
+ if (argv[1]) {
+ stopat = atoi(argv[1]);
+ argv++;
+ argc--;
+ }
+
+ /* Add 1000 records. */
+ printf("Adding %u records: ", num); fflush(stdout);
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ gettimeofday(&start, NULL);
+ for (i = 0; i < num; i++)
+ if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
+ errx(1, "Inserting key %u in ntdb: %s",
+ i, ntdb_errorstr(ecode));
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+
+ if (++stage == stopat)
+ exit(0);
+
+ /* Finding 1000 records. */
+ printf("Finding %u records: ", num); fflush(stdout);
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ gettimeofday(&start, NULL);
+ for (i = 0; i < num; i++) {
+ NTDB_DATA dbuf;
+ if ((ecode = ntdb_fetch(ntdb, key, &dbuf)) != NTDB_SUCCESS
+ || *(int *)dbuf.dptr != i) {
+ errx(1, "Fetching key %u in ntdb gave %u",
+ i, ecode ? ecode : *(int *)dbuf.dptr);
+ }
+ }
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+ if (++stage == stopat)
+ exit(0);
+
+ /* Missing 1000 records. */
+ printf("Missing %u records: ", num); fflush(stdout);
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ gettimeofday(&start, NULL);
+ for (i = num; i < num*2; i++) {
+ NTDB_DATA dbuf;
+ ecode = ntdb_fetch(ntdb, key, &dbuf);
+ if (ecode != NTDB_ERR_NOEXIST)
+ errx(1, "Fetching key %u in ntdb gave %s",
+ i, ntdb_errorstr(ecode));
+ }
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+ if (++stage == stopat)
+ exit(0);
+
+ /* Traverse 1000 records. */
+ printf("Traversing %u records: ", num); fflush(stdout);
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ i = 0;
+ gettimeofday(&start, NULL);
+ if (ntdb_traverse(ntdb, count_record, &i) != num)
+ errx(1, "Traverse returned wrong number of records");
+ if (i != (num - 1) * (num / 2))
+ errx(1, "Traverse tallied to %u", i);
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+ if (++stage == stopat)
+ exit(0);
+
+ /* Delete 1000 records (not in order). */
+ printf("Deleting %u records: ", num); fflush(stdout);
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ gettimeofday(&start, NULL);
+ for (j = 0; j < num; j++) {
+ i = (j + 100003) % num;
+ if ((ecode = ntdb_delete(ntdb, key)) != NTDB_SUCCESS)
+ errx(1, "Deleting key %u in ntdb: %s",
+ i, ntdb_errorstr(ecode));
+ }
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+ if (++stage == stopat)
+ exit(0);
+
+ /* Re-add 1000 records (not in order). */
+ printf("Re-adding %u records: ", num); fflush(stdout);
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ gettimeofday(&start, NULL);
+ for (j = 0; j < num; j++) {
+ i = (j + 100003) % num;
+ if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
+ errx(1, "Inserting key %u in ntdb: %s",
+ i, ntdb_errorstr(ecode));
+ }
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+ if (++stage == stopat)
+ exit(0);
+
+ /* Append 1000 records. */
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ printf("Appending %u records: ", num); fflush(stdout);
+ gettimeofday(&start, NULL);
+ for (i = 0; i < num; i++)
+ if ((ecode = ntdb_append(ntdb, key, data)) != NTDB_SUCCESS)
+ errx(1, "Appending key %u in ntdb: %s",
+ i, ntdb_errorstr(ecode));
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (++stage == stopat)
+ exit(0);
+
+ /* Churn 1000 records: not in order! */
+ if (transaction && (ecode = ntdb_transaction_start(ntdb)))
+ errx(1, "starting transaction: %s", ntdb_errorstr(ecode));
+ printf("Churning %u records: ", num); fflush(stdout);
+ gettimeofday(&start, NULL);
+ for (j = 0; j < num; j++) {
+ i = (j + 1000019) % num;
+ if ((ecode = ntdb_delete(ntdb, key)) != NTDB_SUCCESS)
+ errx(1, "Deleting key %u in ntdb: %s",
+ i, ntdb_errorstr(ecode));
+ i += num;
+ if ((ecode = ntdb_store(ntdb, key, data, NTDB_INSERT)) != 0)
+ errx(1, "Inserting key %u in ntdb: %s",
+ i, ntdb_errorstr(ecode));
+ }
+ gettimeofday(&stop, NULL);
+ if (transaction && (ecode = ntdb_transaction_commit(ntdb)))
+ errx(1, "committing transaction: %s", ntdb_errorstr(ecode));
+ printf(" %zu ns (%zu bytes)\n",
+ normalize(&start, &stop, num), file_size());
+
+ if (ntdb_check(ntdb, NULL, NULL))
+ errx(1, "ntdb_check failed!");
+ if (summary) {
+ char *sumstr = NULL;
+ ntdb_summary(ntdb, NTDB_SUMMARY_HISTOGRAMS, &sumstr);
+ printf("%s\n", sumstr);
+ free(sumstr);
+ }
+ if (do_stats)
+ dump_and_clear_stats(&ntdb, flags, &log);
+ if (++stage == stopat)
+ exit(0);
+
+ return 0;
+}
generated by cgit (git 2.34.1) at 2024-11-22 22:09:05 +0000