r19426: merge nearly all the differences between Samba3 tdb and Samba4

tdb. This includes:

 - the new tdb_lockall and tdb_lockall_read code, which will be needed
   for the ldb speedups

 - the tdb logging changes. This is an intermediate step to keep the
   differences between the two branches small. The plan is still to
   move to a tdb_init()/tdb_set_logging_function()/tdb_attach() style
   of open which will make things much cleaner.

 - the updated test suites and standalone tdb build code

 - use libreplace headers

There are still some small differences I haven't merged. I'll discuss
those on the list.
(This used to be commit 48903c75edfaf75dbd3e9d052e615552cdff39b4)

2 files changed, 191 insertions, 103 deletions

diff --git a/source3/tdb/tools/tdbtest.c b/source3/tdb/tools/tdbtest.c
index 89295a3291..c7a09789fe 100644
--- a/source3/tdb/tools/tdbtest.c
+++ b/source3/tdb/tools/tdbtest.c
@@ -1,48 +1,43 @@
-#include <stdlib.h>
-#include <stdio.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <string.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <signal.h>
-#include "tdb.h"
-#include <gdbm.h>
-
 /* a test program for tdb - the trivial database */
 
+#include "replace.h"
+#include "tdb.h"
+#include "system/filesys.h"
+#include "system/time.h"
+
+#include <gdbm.h>
 
 
 #define DELETE_PROB 7
 #define STORE_PROB 5
 
-static TDB_CONTEXT *db;
+static struct tdb_context *db;
 static GDBM_FILE gdbm;
 
 struct timeval tp1,tp2;
 
-static void start_timer(void)
+static void _start_timer(void)
 {
 	gettimeofday(&tp1,NULL);
 }
 
-static double end_timer(void)
+static double _end_timer(void)
 {
 	gettimeofday(&tp2,NULL);
 	return((tp2.tv_sec - tp1.tv_sec) + 
 	       (tp2.tv_usec - tp1.tv_usec)*1.0e-6);
 }
 
-static void fatal(char *why)
+static void fatal(const char *why)
 {
 	perror(why);
 	exit(1);
 }
 
-static void tdb_log(TDB_CONTEXT *tdb, int level, const char *format, ...)
+#ifdef PRINTF_ATTRIBUTE
+static void tdb_log(struct tdb_context *tdb, int level, const char *format, ...) PRINTF_ATTRIBUTE(3,4);
+#endif
+static void tdb_log(struct tdb_context *tdb, int level, const char *format, ...)
 {
 	va_list ap;
     
@@ -179,7 +174,7 @@ static void addrec_gdbm(void)
 	free(d);
 }
 
-static int traverse_fn(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+static int traverse_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
 {
 #if 0
 	printf("[%s] [%s]\n", key.dptr, dbuf.dptr);
@@ -192,14 +187,15 @@ static void merge_test(void)
 {
 	int i;
 	char keys[5][2];
+	char tdata[] = "test";
 	TDB_DATA key, data;
 	
 	for (i = 0; i < 5; i++) {
-		sprintf(keys[i], "%d", i);
+		snprintf(keys[i],2, "%d", i);
 		key.dptr = keys[i];
 		key.dsize = 2;
 		
-		data.dptr = "test";
+		data.dptr = tdata;
 		data.dsize = 4;
 		
 		if (tdb_store(db, key, data, TDB_REPLACE) != 0) {
@@ -219,16 +215,17 @@ static void merge_test(void)
 	tdb_delete(db, key);
 }
 	
-int main(int argc, char *argv[])
+ int main(int argc, const char *argv[])
 {
 	int i, seed=0;
 	int loops = 10000;
+	char test_gdbm[] = "test.gdbm";
 
 	unlink("test.gdbm");
 
 	db = tdb_open("test.tdb", 0, TDB_CLEAR_IF_FIRST, 
 		      O_RDWR | O_CREAT | O_TRUNC, 0600);
-	gdbm = gdbm_open("test.gdbm", 512, GDBM_WRITER|GDBM_NEWDB|GDBM_FAST, 
+	gdbm = gdbm_open(test_gdbm, 512, GDBM_WRITER|GDBM_NEWDB|GDBM_FAST, 
 			 0600, NULL);
 
 	if (!db || !gdbm) {
@@ -239,17 +236,17 @@ int main(int argc, char *argv[])
 	
 #if 1
 	srand(seed);
-	start_timer();
+	_start_timer();
 	for (i=0;i<loops;i++) addrec_gdbm();
-	printf("gdbm got %.2f ops/sec\n", i/end_timer());
+	printf("gdbm got %.2f ops/sec\n", i/_end_timer());
 #endif
 
 	merge_test();
 
 	srand(seed);
-	start_timer();
+	_start_timer();
 	for (i=0;i<loops;i++) addrec_db();
-	printf("tdb got %.2f ops/sec\n", i/end_timer());
+	printf("tdb got %.2f ops/sec\n", i/_end_timer());
 
 	compare_db();
 
diff --git a/source3/tdb/tools/tdbtorture.c b/source3/tdb/tools/tdbtorture.c
index 2d367b91e1..14a2b48cdc 100644
--- a/source3/tdb/tools/tdbtorture.c
+++ b/source3/tdb/tools/tdbtorture.c
@@ -1,41 +1,43 @@
-#include <stdlib.h>
-#include <time.h>
-#include <stdio.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <string.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/wait.h>
-#include "tdb.h"
-
 /* this tests tdb by doing lots of ops from several simultaneous
-   writers - that stresses the locking code. Build with TDB_DEBUG=1
-   for best effect */
+   writers - that stresses the locking code. 
+*/
 
+#include "replace.h"
+#include "tdb.h"
+#include "system/time.h"
+#include "system/wait.h"
+#include "system/filesys.h"
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#endif
 
 
 #define REOPEN_PROB 30
 #define DELETE_PROB 8
 #define STORE_PROB 4
 #define APPEND_PROB 6
-#define LOCKSTORE_PROB 0
+#define TRANSACTION_PROB 10
+#define LOCKSTORE_PROB 5
 #define TRAVERSE_PROB 20
+#define TRAVERSE_READ_PROB 20
 #define CULL_PROB 100
 #define KEYLEN 3
 #define DATALEN 100
-#define LOCKLEN 20
 
-static TDB_CONTEXT *db;
+static struct tdb_context *db;
+static int in_transaction;
+static int error_count;
 
-static void tdb_log(TDB_CONTEXT *tdb, int level, const char *format, ...)
+#ifdef PRINTF_ATTRIBUTE
+static void tdb_log(struct tdb_context *tdb, enum tdb_debug_level level, const char *format, ...) PRINTF_ATTRIBUTE(3,4);
+#endif
+static void tdb_log(struct tdb_context *tdb, enum tdb_debug_level level, const char *format, ...)
 {
 	va_list ap;
     
+	error_count++;
+
 	va_start(ap, format);
 	vfprintf(stdout, format, ap);
 	va_end(ap);
@@ -50,10 +52,10 @@ static void tdb_log(TDB_CONTEXT *tdb, int level, const char *format, ...)
 #endif	
 }
 
-static void fatal(char *why)
+static void fatal(const char *why)
 {
 	perror(why);
-	exit(1);
+	error_count++;
 }
 
 static char *randbuf(int len)
@@ -69,41 +71,62 @@ static char *randbuf(int len)
 	return buf;
 }
 
-static int cull_traverse(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf,
+static int cull_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
 			 void *state)
 {
+#if CULL_PROB
 	if (random() % CULL_PROB == 0) {
 		tdb_delete(tdb, key);
 	}
+#endif
 	return 0;
 }
 
 static void addrec_db(void)
 {
-	int klen, dlen, slen;
-	char *k, *d, *s;
-	TDB_DATA key, data, lockkey;
+	int klen, dlen;
+	char *k, *d;
+	TDB_DATA key, data;
 
 	klen = 1 + (rand() % KEYLEN);
 	dlen = 1 + (rand() % DATALEN);
-	slen = 1 + (rand() % LOCKLEN);
 
 	k = randbuf(klen);
 	d = randbuf(dlen);
-	s = randbuf(slen);
 
-	key.dptr = k;
+	key.dptr = (unsigned char *)k;
 	key.dsize = klen+1;
 
-	data.dptr = d;
+	data.dptr = (unsigned char *)d;
 	data.dsize = dlen+1;
 
-	lockkey.dptr = s;
-	lockkey.dsize = slen+1;
+#if TRANSACTION_PROB
+	if (in_transaction == 0 && random() % TRANSACTION_PROB == 0) {
+		if (tdb_transaction_start(db) != 0) {
+			fatal("tdb_transaction_start failed");
+		}
+		in_transaction++;
+		goto next;
+	}
+	if (in_transaction && random() % TRANSACTION_PROB == 0) {
+		if (tdb_transaction_commit(db) != 0) {
+			fatal("tdb_transaction_commit failed");
+		}
+		in_transaction--;
+		goto next;
+	}
+	if (in_transaction && random() % TRANSACTION_PROB == 0) {
+		if (tdb_transaction_cancel(db) != 0) {
+			fatal("tdb_transaction_cancel failed");
+		}
+		in_transaction--;
+		goto next;
+	}
+#endif
 
 #if REOPEN_PROB
-	if (random() % REOPEN_PROB == 0) {
-		tdb_reopen_all(1);
+	if (in_transaction == 0 && random() % REOPEN_PROB == 0) {
+		tdb_reopen_all(0);
 		goto next;
 	} 
 #endif
@@ -135,13 +158,13 @@ static void addrec_db(void)
 
 #if LOCKSTORE_PROB
 	if (random() % LOCKSTORE_PROB == 0) {
-		tdb_chainlock(db, lockkey);
+		tdb_chainlock(db, key);
 		data = tdb_fetch(db, key);
 		if (tdb_store(db, key, data, TDB_REPLACE) != 0) {
 			fatal("tdb_store failed");
 		}
 		if (data.dptr) free(data.dptr);
-		tdb_chainunlock(db, lockkey);
+		tdb_chainunlock(db, key);
 		goto next;
 	} 
 #endif
@@ -153,75 +176,143 @@ static void addrec_db(void)
 	}
 #endif
 
+#if TRAVERSE_READ_PROB
+	if (random() % TRAVERSE_READ_PROB == 0) {
+		tdb_traverse_read(db, NULL, NULL);
+		goto next;
+	}
+#endif
+
 	data = tdb_fetch(db, key);
 	if (data.dptr) free(data.dptr);
 
 next:
 	free(k);
 	free(d);
-	free(s);
 }
 
-static int traverse_fn(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf,
+static int traverse_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf,
                        void *state)
 {
 	tdb_delete(tdb, key);
 	return 0;
 }
 
-#ifndef NPROC
-#define NPROC 6
-#endif
-
-#ifndef NLOOPS
-#define NLOOPS 200000
-#endif
+static void usage(void)
+{
+	printf("Usage: tdbtorture [-n NUM_PROCS] [-l NUM_LOOPS] [-s SEED] [-H HASH_SIZE]\n");
+	exit(0);
+}
 
-int main(int argc, char *argv[])
+ int main(int argc, char * const *argv)
 {
-	int i, seed=0;
-	int loops = NLOOPS;
-	pid_t pids[NPROC];
+	int i, seed = -1;
+	int num_procs = 3;
+	int num_loops = 5000;
+	int hash_size = 2;
+	int c;
+	extern char *optarg;
+	pid_t *pids;
+
+	struct tdb_logging_context log_ctx;
+	log_ctx.log_fn = tdb_log;
+
+	while ((c = getopt(argc, argv, "n:l:s:H:h")) != -1) {
+		switch (c) {
+		case 'n':
+			num_procs = strtol(optarg, NULL, 0);
+			break;
+		case 'l':
+			num_loops = strtol(optarg, NULL, 0);
+			break;
+		case 'H':
+			hash_size = strtol(optarg, NULL, 0);
+			break;
+		case 's':
+			seed = strtol(optarg, NULL, 0);
+			break;
+		default:
+			usage();
+		}
+	}
+
+	unlink("torture.tdb");
+
+	pids = calloc(sizeof(pid_t), num_procs);
+	pids[0] = getpid();
 
-	db = tdb_open("torture.tdb", 0, TDB_CLEAR_IF_FIRST, 
-		      O_RDWR | O_CREAT, 0600);
+	for (i=0;i<num_procs-1;i++) {
+		if ((pids[i+1]=fork()) == 0) break;
+	}
+
+	db = tdb_open_ex("torture.tdb", hash_size, TDB_CLEAR_IF_FIRST, 
+			 O_RDWR | O_CREAT, 0600, &log_ctx, NULL);
 	if (!db) {
 		fatal("db open failed");
 	}
 
-	for (i=0;i<NPROC;i++) {
-		pids[i] = fork();
-		if (pids[i] == 0) {
-			tdb_reopen_all(1);
+	if (seed == -1) {
+		seed = (getpid() + time(NULL)) & 0x7FFFFFFF;
+	}
 
-			tdb_logging_function(db, tdb_log);
+	if (i == 0) {
+		printf("testing with %d processes, %d loops, %d hash_size, seed=%d\n", 
+		       num_procs, num_loops, hash_size, seed);
+	}
 
-			srand(seed + getpid());
-			srandom(seed + getpid() + time(NULL));
-			for (i=0;i<loops;i++) addrec_db();
+	srand(seed + i);
+	srandom(seed + i);
 
-			tdb_traverse(db, NULL, NULL);
-			tdb_traverse(db, traverse_fn, NULL);
-			tdb_traverse(db, traverse_fn, NULL);
+	for (i=0;i<num_loops && error_count == 0;i++) {
+		addrec_db();
+	}
 
-			tdb_close(db);
-			exit(0);
-		}
+	if (error_count == 0) {
+		tdb_traverse_read(db, NULL, NULL);
+		tdb_traverse(db, traverse_fn, NULL);
+		tdb_traverse(db, traverse_fn, NULL);
 	}
 
-	for (i=0;i<NPROC;i++) {
-		int status;
-		if (waitpid(pids[i], &status, 0) != pids[i]) {
-			printf("failed to wait for %d\n",
-			       (int)pids[i]);
+	tdb_close(db);
+
+	if (getpid() != pids[0]) {
+		return error_count;
+	}
+
+	for (i=1;i<num_procs;i++) {
+		int status, j;
+		pid_t pid;
+		if (error_count != 0) {
+			/* try and stop the test on any failure */
+			for (j=1;j<num_procs;j++) {
+				if (pids[j] != 0) {
+					kill(pids[j], SIGTERM);
+				}
+			}
+		}
+		pid = waitpid(-1, &status, 0);
+		if (pid == -1) {
+			perror("failed to wait for child\n");
+			exit(1);
+		}
+		for (j=1;j<num_procs;j++) {
+			if (pids[j] == pid) break;
+		}
+		if (j == num_procs) {
+			printf("unknown child %d exited!?\n", (int)pid);
 			exit(1);
 		}
 		if (WEXITSTATUS(status) != 0) {
 			printf("child %d exited with status %d\n",
-			       (int)pids[i], WEXITSTATUS(status));
-			exit(1);
+			       (int)pid, WEXITSTATUS(status));
+			error_count++;
 		}
+		pids[j] = 0;
 	}
-	printf("OK\n");
-	return 0;
+
+	if (error_count == 0) {
+		printf("OK\n");
+	}
+
+	return error_count;
 }