r23806: update Samba4 with the latest ctdb code.

This doesn't get the ctdb code fully working in Samba4, it just gets
it building and not breaking non-clustered use of Samba. It will take
a bit longer to update some of the calling ctdb_cluster.c code to make
it work correctly in Samba4.

Note also that Samba4 now only links to the client portion of
ctdb. For the moment I am leaving the ctdbd as a separate daemon,
which you install separately from http://ctdb.samba.org/.
(This used to be commit b196077cbb55cbecad87065133c2d67198e31066)

26 files changed, 553 insertions, 1152 deletions

diff --git a/source4/cluster/ctdb/tests/1node.txt b/source4/cluster/ctdb/tests/1node.txt
deleted file mode 100644
index db4350c0c0..0000000000
--- a/source4/cluster/ctdb/tests/1node.txt
+++ /dev/null
@@ -1 +0,0 @@
-127.0.0.1:9001
diff --git a/source4/cluster/ctdb/tests/4nodes.txt b/source4/cluster/ctdb/tests/4nodes.txt
deleted file mode 100644
index 880fe914ff..0000000000
--- a/source4/cluster/ctdb/tests/4nodes.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-127.0.0.1:9001
-127.0.0.2:9001
-127.0.0.3:9001
-127.0.0.4:9001
diff --git a/source4/cluster/ctdb/tests/bench-ssh.sh b/source4/cluster/ctdb/tests/bench-ssh.sh
deleted file mode 100755
index 0d11ee9cdd..0000000000
--- a/source4/cluster/ctdb/tests/bench-ssh.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-
-if [ $# -lt 1 ]; then
-    echo "Usage: bench-ssh.sh <NODES> <OPTIONS>"
-    exit 1
-fi
-
-while :; do
-    if [ "`echo $1 | cut -c1`" = "-" -o $# -eq 0 ]; then break; fi
-    nodes="$nodes $1";
-    shift;
-done
-
-options=$*
-dir=`pwd`
-
-echo "Creating nodes-ssh.txt"
-rm -f nodes-ssh.txt
-count=0
-for h in $nodes; do
-    echo "$h:9001" >> nodes-ssh.txt
-    count=`expr $count + 1`
-done
-
-
-echo "Killing old processes"
-for h in $nodes; do
-    scp -q nodes-ssh.txt $h:$dir
-    ssh $h killall -q ctdb_bench
-done
-
-echo "Starting nodes"
-i=0
-for h in $nodes; do
-    if [ $i -eq `expr $count - 1` ]; then
-	ssh $h $dir/bin/ctdb_bench --nlist $dir/nodes-ssh.txt --listen $h:9001 $options
-    else
-	ssh -f $h $dir/bin/ctdb_bench --nlist $dir/nodes-ssh.txt --listen $h:9001 $options
-    fi
-    i=`expr $i + 1`
-done
-
-wait
diff --git a/source4/cluster/ctdb/tests/bench.sh b/source4/cluster/ctdb/tests/bench.sh
index 3d0696f171..48a4277591 100755
--- a/source4/cluster/ctdb/tests/bench.sh
+++ b/source4/cluster/ctdb/tests/bench.sh
@@ -1,9 +1,24 @@
 #!/bin/sh
 
-killall -q ctdb_bench
+killall -q ctdb_bench ctdbd
 
-echo "Trying 2 nodes"
-$VALGRIND bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
-$VALGRIND bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.1:9001 $*
-wait
+NUMNODES=2
+if [ $# -gt 0 ]; then
+    NUMNODES=$1
+fi
+
+rm -f nodes.txt
+for i in `seq 1 $NUMNODES`; do
+  echo 127.0.0.$i >> nodes.txt
+done
+
+tests/start_daemons.sh $NUMNODES nodes.txt || exit 1
 
+killall -9 ctdb_bench
+echo "Trying $NUMNODES nodes"
+for i in `seq 1 $NUMNODES`; do
+  valgrind -q $VALGRIND bin/ctdb_bench --socket sock.$i -n $NUMNODES $*  &
+done
+
+wait
+bin/ctdb shutdown --socket sock.1 -n all
diff --git a/source4/cluster/ctdb/tests/bench1.sh b/source4/cluster/ctdb/tests/bench1.sh
deleted file mode 100755
index 9adcf3198b..0000000000
--- a/source4/cluster/ctdb/tests/bench1.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_bench
-
-echo "Trying 1 nodes"
-bin/ctdb_bench --nlist tests/1node.txt --listen 127.0.0.2:9001 $*
-wait
diff --git a/source4/cluster/ctdb/tests/cmdline.c b/source4/cluster/ctdb/tests/cmdline.c
deleted file mode 100644
index 3c2f5ca19a..0000000000
--- a/source4/cluster/ctdb/tests/cmdline.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* 
-   common commandline code to ctdb test tools
-
-   Copyright (C) Andrew Tridgell  2007
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-
-/* Handle common command line options for ctdb test progs
- */
-
-static struct {
-	const char *nlist;
-	const char *transport;
-	const char *myaddress;
-	int self_connect;
-	int daemon_mode;
-} ctdb_cmdline = {
-	.nlist = NULL,
-	.transport = "tcp",
-	.myaddress = NULL,
-	.self_connect = 0,
-	.daemon_mode = 0
-};
-
-
-struct poptOption popt_ctdb_cmdline[] = {
-	{ "nlist", 0, POPT_ARG_STRING, &ctdb_cmdline.nlist, 0, "node list file", "filename" },
-	{ "listen", 0, POPT_ARG_STRING, &ctdb_cmdline.myaddress, 0, "address to listen on", "address" },
-	{ "transport", 0, POPT_ARG_STRING, &ctdb_cmdline.transport, 0, "protocol transport", NULL },
-	{ "self-connect", 0, POPT_ARG_NONE, &ctdb_cmdline.self_connect, 0, "enable self connect", "boolean" },
-	{ "daemon", 0, POPT_ARG_NONE, &ctdb_cmdline.daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
-	{ NULL }
-};
-
-
-/*
-  startup daemon side of ctdb according to command line options
- */
-struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
-{
-	struct ctdb_context *ctdb;
-	int ret;
-
-	if (ctdb_cmdline.nlist == NULL || ctdb_cmdline.myaddress == NULL) {
-		printf("You must provide a node list with --nlist and an address with --listen\n");
-		exit(1);
-	}
-
-	/* initialise ctdb */
-	ctdb = ctdb_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	if (ctdb_cmdline.self_connect) {
-		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
-	}
-	if (ctdb_cmdline.daemon_mode) {
-		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
-	}
-
-	ret = ctdb_set_transport(ctdb, ctdb_cmdline.transport);
-	if (ret == -1) {
-		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* tell ctdb what address to listen on */
-	ret = ctdb_set_address(ctdb, ctdb_cmdline.myaddress);
-	if (ret == -1) {
-		printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* tell ctdb what nodes are available */
-	ret = ctdb_set_nlist(ctdb, ctdb_cmdline.nlist);
-	if (ret == -1) {
-		printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	return ctdb;
-}
diff --git a/source4/cluster/ctdb/tests/cmdline.h b/source4/cluster/ctdb/tests/cmdline.h
deleted file mode 100644
index 785595ee6c..0000000000
--- a/source4/cluster/ctdb/tests/cmdline.h
+++ /dev/null
@@ -1,7 +0,0 @@
-
-extern struct poptOption popt_ctdb_cmdline[];
-
-#define POPT_CTDB_CMDLINE { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_ctdb_cmdline, 0, "Common ctdb test options:", NULL },
-
-struct ctdb_context *ctdb_cmdline_init(struct event_context *ev);
-
diff --git a/source4/cluster/ctdb/tests/ctdb_bench.c b/source4/cluster/ctdb/tests/ctdb_bench.c
index a9cb2c2372..f1292ee275 100644
--- a/source4/cluster/ctdb/tests/ctdb_bench.c
+++ b/source4/cluster/ctdb/tests/ctdb_bench.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -43,7 +43,7 @@ static double end_timer(void)
 
 static int timelimit = 10;
 static int num_records = 10;
-static int num_msgs = 1;
+static int num_nodes;
 
 enum my_functions {FUNC_INCR=1, FUNC_FETCH=2};
 
@@ -83,14 +83,14 @@ static int msg_plus, msg_minus;
 /*
   handler for messages in bench_ring()
 */
-static void ring_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+static void ring_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 				 TDB_DATA data, void *private_data)
 {
 	int incr = *(int *)data.dptr;
 	int *count = (int *)private_data;
 	int dest;
 	(*count)++;
-	dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
+	dest = (ctdb_get_vnn(ctdb) + incr) % num_nodes;
 	ctdb_send_message(ctdb, dest, srvid, data);
 	if (incr == 1) {
 		msg_plus++;
@@ -107,7 +107,6 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 	int vnn=ctdb_get_vnn(ctdb);
 
 	if (vnn == 0) {
-		int i;
 		/* two messages are injected into the ring, moving
 		   in opposite directions */
 		int dest, incr;
@@ -116,15 +115,13 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 		data.dptr = (uint8_t *)&incr;
 		data.dsize = sizeof(incr);
 
-		for (i=0;i<num_msgs;i++) {
-			incr = 1;
-			dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
-			ctdb_send_message(ctdb, dest, 0, data);
-
-			incr = -1;
-			dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
-			ctdb_send_message(ctdb, dest, 0, data);
-		}
+		incr = 1;
+		dest = (ctdb_get_vnn(ctdb) + incr) % num_nodes;
+		ctdb_send_message(ctdb, dest, 0, data);
+		
+		incr = -1;
+		dest = (ctdb_get_vnn(ctdb) + incr) % num_nodes;
+		ctdb_send_message(ctdb, dest, 0, data);
 	}
 	
 	start_timer();
@@ -142,6 +139,16 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 	       msg_count/end_timer(), msg_plus, msg_minus);
 }
 
+/*
+  handler for reconfigure message
+*/
+static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+				TDB_DATA data, void *private_data)
+{
+	int *ready = (int *)private_data;
+	*ready = 1;
+}
+
 
 /*
   main program
@@ -156,7 +163,7 @@ int main(int argc, const char *argv[])
 		POPT_CTDB_CMDLINE
 		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
 		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
-		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
+		{ NULL, 'n', POPT_ARG_INT, &num_nodes, 0, "num_nodes", "integer" },
 		POPT_TABLEEND
 	};
 	int opt;
@@ -165,6 +172,7 @@ int main(int argc, const char *argv[])
 	int ret;
 	poptContext pc;
 	struct event_context *ev;
+	int cluster_ready=0;
 
 	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
 
@@ -187,10 +195,13 @@ int main(int argc, const char *argv[])
 	ev = event_context_init(NULL);
 
 	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
+	ctdb = ctdb_cmdline_client(ev);
+
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler, 
+				 &cluster_ready);
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	ctdb_db = ctdb_attach(ctdb, "test.tdb");
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
@@ -200,19 +211,19 @@ int main(int argc, const char *argv[])
 	ret = ctdb_set_call(ctdb_db, incr_func,  FUNC_INCR);
 	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
 
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-	ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count);
+	if (ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count))
+		goto error;
 
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
+	printf("Waiting for cluster\n");
+	while (1) {
+		uint32_t recmode=1;
+		ctdb_ctrl_getrecmode(ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode);
+		if (recmode == 0) break;
+		event_loop_once(ev);
+	}
 
 	bench_ring(ctdb, ev);
        
-	/* shut it down */
-	ctdb_shutdown(ctdb);
-
+error:
 	return 0;
 }
diff --git a/source4/cluster/ctdb/tests/ctdb_fetch.c b/source4/cluster/ctdb/tests/ctdb_fetch.c
index 38d652bf5f..f57d05d099 100644
--- a/source4/cluster/ctdb/tests/ctdb_fetch.c
+++ b/source4/cluster/ctdb/tests/ctdb_fetch.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -43,8 +43,7 @@ static double end_timer(void)
 
 static int timelimit = 10;
 static int num_records = 10;
-static int num_msgs = 1;
-
+static int num_nodes;
 static int msg_count;
 
 #define TESTKEY "testkey"
@@ -99,14 +98,14 @@ static void bench_fetch_1node(struct ctdb_context *ctdb)
 	nulldata.dptr = NULL;
 	nulldata.dsize = 0;
 
-	dest = (ctdb_get_vnn(ctdb) + 1) % ctdb_get_num_nodes(ctdb);
+	dest = (ctdb_get_vnn(ctdb) + 1) % num_nodes;
 	ctdb_send_message(ctdb, dest, 0, nulldata);
 }
 
 /*
   handler for messages in bench_ring()
 */
-static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+static void message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 			    TDB_DATA data, void *private_data)
 {
 	msg_count++;
@@ -126,7 +125,7 @@ static void bench_fetch(struct ctdb_context *ctdb, struct event_context *ev)
 {
 	int vnn=ctdb_get_vnn(ctdb);
 
-	if (vnn == ctdb_get_num_nodes(ctdb)-1) {
+	if (vnn == num_nodes - 1) {
 		bench_fetch_1node(ctdb);
 	}
 	
@@ -141,10 +140,6 @@ static void bench_fetch(struct ctdb_context *ctdb, struct event_context *ev)
 			printf("Event loop failed!\n");
 			break;
 		}
-
-		if (LogLevel > 9) {
-			talloc_report_null_full();
-		}
 	}
 
 	printf("Fetch: %.2f msgs/sec\n", msg_count/end_timer());
@@ -162,6 +157,16 @@ static int fetch_func(struct ctdb_call_info *call)
 }
 
 /*
+  handler for reconfigure message
+*/
+static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+				TDB_DATA data, void *private_data)
+{
+	int *ready = (int *)private_data;
+	*ready = 1;
+}
+
+/*
   main program
 */
 int main(int argc, const char *argv[])
@@ -174,7 +179,7 @@ int main(int argc, const char *argv[])
 		POPT_CTDB_CMDLINE
 		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
 		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
-		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
+		{ NULL, 'n', POPT_ARG_INT, &num_nodes, 0, "num_nodes", "integer" },
 		POPT_TABLEEND
 	};
 	int opt;
@@ -184,6 +189,7 @@ int main(int argc, const char *argv[])
 	poptContext pc;
 	struct event_context *ev;
 	struct ctdb_call call;
+	int cluster_ready=0;
 
 	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
 
@@ -207,10 +213,13 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_init(ev);
+	ctdb = ctdb_cmdline_client(ev);
+
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler, 
+				 &cluster_ready);
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	ctdb_db = ctdb_attach(ctdb, "test.tdb");
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
@@ -218,14 +227,15 @@ int main(int argc, const char *argv[])
 
 	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
 
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
 	ctdb_set_message_handler(ctdb, 0, message_handler, &msg_count);
 
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
+	printf("Waiting for cluster\n");
+	while (1) {
+		uint32_t recmode=1;
+		ctdb_ctrl_getrecmode(ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode);
+		if (recmode == 0) break;
+		event_loop_once(ev);
+	}
 
 	bench_fetch(ctdb, ev);
 
@@ -233,6 +243,8 @@ int main(int argc, const char *argv[])
 	call.key.dptr = discard_const(TESTKEY);
 	call.key.dsize = strlen(TESTKEY);
 
+	printf("Fetching final record\n");
+
 	/* fetch the record */
 	call.call_id = FUNC_FETCH;
 	call.call_data.dptr = NULL;
@@ -246,8 +258,5 @@ int main(int argc, const char *argv[])
 
 	printf("DATA:\n%s\n", (char *)call.reply_data.dptr);
 
-	/* go into a wait loop to allow other nodes to complete */
-	ctdb_shutdown(ctdb);
-
 	return 0;
 }
diff --git a/source4/cluster/ctdb/tests/ctdb_fetch1.c b/source4/cluster/ctdb/tests/ctdb_fetch1.c
deleted file mode 100644
index a4cb53dd26..0000000000
--- a/source4/cluster/ctdb/tests/ctdb_fetch1.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/* 
-   simple ctdb fetch test
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "ctdb.h"
-#include "ctdb_private.h"
-#include "cmdline.h"
-#include <sys/time.h>
-
-#define PARENT_SRVID	0
-#define CHILD1_SRVID	1
-#define CHILD2_SRVID	2
-
-int num_msg=0;
-
-static struct timeval tp1,tp2;
-
-static void start_timer(void)
-{
-	gettimeofday(&tp1,NULL);
-}
-
-static double end_timer(void)
-{
-	gettimeofday(&tp2,NULL);
-	return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - 
-		(tp1.tv_sec + (tp1.tv_usec*1.0e-6));
-}
-
-static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-			    TDB_DATA data, void *private_data)
-{
-	num_msg++;
-}
-static void child_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-			    TDB_DATA data, void *private_data)
-{
-	num_msg++;
-}
-
-void test1(struct ctdb_db_context *ctdb_db)
-{
-	TDB_DATA key, data, data2, store_data;
-	int ret;
-	struct ctdb_record_handle *h;
- 
-	/* 
-	   test 1 : write data and read it back.   should all be the same
-	 */
-	printf("Test1: write and verify we can read it back: ");
-	key.dptr  = discard_const("Record");
-	key.dsize = strlen((const char *)key.dptr)+1;
-	h = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data);
-	if (h == NULL) {
-		printf("test1: ctdb_fetch_lock() failed\n");
-		exit(1);
-	}
-
-	store_data.dptr  = discard_const("data to store");
-	store_data.dsize = strlen((const char *)store_data.dptr)+1;
-	ret = ctdb_record_store(h, store_data);
-	talloc_free(h);
-	if (ret!=0) {
-		printf("test1: ctdb_record_store() failed\n");
-		exit(1);
-	}
-
-	h = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data2);
-	if (h == NULL) {
-		printf("test1: ctdb_fetch_lock() failed\n");
-		exit(1);
-	}
-
-	/* hopefully   data2 will now contain the record written above */
-	if (!strcmp("data to store", (const char *)data2.dptr)) {
-		printf("SUCCESS\n");
-	} else {
-		printf("FAILURE\n");
-		exit(10);
-	}
-	
-	/* just write it back to unlock it */
-	ret = ctdb_record_store(h, store_data);
-	talloc_free(h);
-	if (ret!=0) {
-		printf("test1: ctdb_record_store() failed\n");
-		exit(1);
-	}
-}
-
-void child(int srvid, struct event_context *ev, struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db)
-{
-	TDB_DATA data;
-	TDB_DATA key, data2;
-	struct ctdb_record_handle *h;
-
-	data.dptr=discard_const("dummy message");
-	data.dsize=strlen((const char *)data.dptr)+1;
-
-	ctdb_set_message_handler(ctdb, srvid, child_handler, NULL);
-
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), PARENT_SRVID, data);
-	while (num_msg==0) {
-		event_loop_once(ev);
-	}
-
-
-	/* fetch and lock the record */
-	key.dptr  = discard_const("Record");
-	key.dsize = strlen((const char *)key.dptr)+1;
-	printf("client:%d fetching the record\n",srvid);
-	h = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data2);
-	printf("client:%d the record is fetched and locked\n",srvid);
-	if (h == NULL) {
-		printf("client: ctdb_fetch_lock() failed\n");
-		exit(1);
-	}
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), PARENT_SRVID, data);
-
-	/* wait until parent tells us to release the lock */
-	while (num_msg==1) {
-		event_loop_once(ev);
-	}
-
-	printf("child %d terminating\n",srvid);
-	exit(10);
-	   
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-	TDB_DATA data;
-
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int ret;
-	poptContext pc;
-	struct event_context *ev;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
-	if (!ctdb_db) {
-		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-#if 0
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
-#endif
-
-	/*
-	   start two child processes
-	 */
-	if(fork()){
-		/* 
-		   set up a message handler so our child processes can talk to us
-		 */
-		ctdb_set_message_handler(ctdb, PARENT_SRVID, message_handler, NULL);
-	} else {
-		sleep(3);
-		if(!fork()){
-			child(CHILD1_SRVID, ev, ctdb, ctdb_db);
-		} else {
-			child(CHILD2_SRVID, ev, ctdb, ctdb_db);
-		}
-	}
-
-	/* 
-	   test 1 : write data and read it back.
-	 */
-	test1(ctdb_db); 
-
-	/* 
-	   wait until both children have sent us a message they have started
-	 */
-	printf("Wait for both child processes to start: ");
-	while (num_msg!=2) {
-		event_loop_once(ev);
-	}
-	printf("STARTED\n");
-
-
-	/*
-	   send message to child 1 to make it to fetch and lock the record 
-	 */
-	data.dptr=discard_const("dummy message");
-	data.dsize=strlen((const char *)data.dptr)+1;
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD1_SRVID, data);
-
-	/* wait for child 1 to complete fetching and locking the record */
-	while (num_msg!=3) {
-		event_loop_once(ev);
-	}
-
-	/* now tell child 2 to fetch and lock the same record */
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD2_SRVID, data);
-
-	/* wait a while for child 2 to complete fetching and locking the 
-	   record, this should fail since the record is already locked
-	   by the first child */
-	start_timer();
-	while ( (end_timer() < 1.0) && (num_msg!=4) ) {
-		event_loop_once(ev);
-	}
-	if (num_msg!=4) {
-		printf("Child 2 did not get the lock since it is held by client 1:SUCCESS\n");
-	} else {
-		printf("Child 2 did get the lock:FAILURE\n");
-		exit(10);
-	}
-
-	/* send message to child 1 to terminate, which should let child 2
-	   get the lock.
-	 */
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD1_SRVID, data);
-
-
-	/* wait for a final message from child 2 it has received the lock
-	   which indicates success */
-	while (num_msg!=4) {
-		event_loop_once(ev);
-	}
-	printf("child 2 aquired the lock after child 1 terminated:SUCCESS\n");
-
-	/* send a message to child 2 to tell it to terminate too */
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD2_SRVID, data);
-
-
-	printf("Test was SUCCESSFUL\n");
-
-	/* shut it down */
-	talloc_free(ctdb);
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/ctdb_messaging.c b/source4/cluster/ctdb/tests/ctdb_messaging.c
deleted file mode 100644
index 309b9ebd32..0000000000
--- a/source4/cluster/ctdb/tests/ctdb_messaging.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/* 
-   test of messaging
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "cmdline.h"
-
-static int timelimit = 10;
-static int num_records = 10;
-static int num_msgs = 1;
-static int num_clients = 2;
-
-
-/*
-  handler for messages in bench_ring()
-*/
-static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-				 TDB_DATA data, void *private_data)
-{
-	printf("client vnn:%d received a message to srvid:%d [%s]\n",ctdb_get_vnn(ctdb),srvid,data.dptr);
-	fflush(stdout);
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-	char buf[256];
-
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
-		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
-		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
-		{ "num-clients", 0, POPT_ARG_INT, &num_clients, 0, "num_clients", "integer" },
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int ret, i, j;
-	poptContext pc;
-	struct event_context *ev;
-	pid_t pid;
-	int srvid;
-	TDB_DATA data;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
-	if (!ctdb_db) {
-		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-	srvid = -1;
-	for (i=0;i<num_clients-1;i++) {
-		pid=fork();
-		if (pid) {
-			srvid = i;
-			break;
-		}
-	}
-	if (srvid == -1) {
-		srvid = num_clients-1;
-	}
-
-	ctdb_set_message_handler(ctdb, srvid, message_handler, NULL);
-
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
-
-	sleep(3);
-
-	printf("sending message from vnn:%d to vnn:%d/srvid:%d\n",ctdb_get_vnn(ctdb),ctdb_get_vnn(ctdb), 1-srvid);
-	for (i=0;i<ctdb_get_num_nodes(ctdb);i++) {
-		for (j=0;j<num_clients;j++) {
-			printf("sending message to %d:%d\n", i, j);
-			sprintf(buf,"Message from %d to vnn:%d srvid:%d",ctdb_get_vnn(ctdb),i,j);
-			data.dptr = (unsigned char *)buf;
-			data.dsize=strlen(buf)+1;
-			ctdb_send_message(ctdb, i, j, data);
-		}
-	}
-
-	while (1) {
-		event_loop_once(ev);
-	}
-       
-	/* shut it down */
-	ctdb_shutdown(ctdb);
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/ctdb_store.c b/source4/cluster/ctdb/tests/ctdb_store.c
new file mode 100644
index 0000000000..a60e009b91
--- /dev/null
+++ b/source4/cluster/ctdb/tests/ctdb_store.c
@@ -0,0 +1,156 @@
+/* 
+   simple tool to create a lot of records on a tdb and to read them out
+
+   Copyright (C) Andrew Tridgell  2006
+	Ronnie sahlberg 2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "popt.h"
+#include "cmdline.h"
+
+#include <sys/time.h>
+#include <time.h>
+
+static int num_records = 10;
+
+
+static void store_records(struct ctdb_context *ctdb, struct event_context *ev)
+{
+	TDB_DATA key, data;
+	struct ctdb_db_context *ctdb_db;
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+	int ret;
+	struct ctdb_record_handle *h;
+	uint32_t i;
+	
+	ctdb_db = ctdb_db_handle(ctdb, "test.tdb");
+
+	printf("creating %d records\n", num_records);
+	for (i=0;i<num_records;i++) {
+		key.dptr = (uint8_t *)&i;
+		key.dsize = sizeof(uint32_t); 
+
+		h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
+		if (h == NULL) {
+			printf("Failed to fetch record '%s' on node %d\n", 
+			       (const char *)key.dptr, ctdb_get_vnn(ctdb));
+			talloc_free(tmp_ctx);
+			return;
+		}
+
+		data.dptr = (uint8_t *)&i;
+		data.dsize = sizeof(uint32_t);
+
+		ret = ctdb_record_store(h, data);
+		talloc_free(h);
+		if (ret != 0) {
+			printf("Failed to store record\n");
+		}
+		if (i % 1000 == 0) {
+			printf("%u\r", i);
+			fflush(stdout);
+		}
+	}
+
+	printf("fetching all %d records\n", num_records);
+	while (1) {
+		for (i=0;i<num_records;i++) {
+			key.dptr = (uint8_t *)&i;
+			key.dsize = sizeof(uint32_t); 
+
+			h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
+			if (h == NULL) {
+				printf("Failed to fetch record '%s' on node %d\n", 
+				       (const char *)key.dptr, ctdb_get_vnn(ctdb));
+				talloc_free(tmp_ctx);
+				return;
+			}
+			talloc_free(h);
+		}
+		sleep(1);
+		printf(".");
+		fflush(stdout);
+	}
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+  main program
+*/
+int main(int argc, const char *argv[])
+{
+	struct ctdb_context *ctdb;
+	struct ctdb_db_context *ctdb_db;
+
+	struct poptOption popt_options[] = {
+		POPT_AUTOHELP
+		POPT_CTDB_CMDLINE
+		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
+		POPT_TABLEEND
+	};
+	int opt;
+	const char **extra_argv;
+	int extra_argc = 0;
+	poptContext pc;
+	struct event_context *ev;
+
+	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
+
+	while ((opt = poptGetNextOpt(pc)) != -1) {
+		switch (opt) {
+		default:
+			fprintf(stderr, "Invalid option %s: %s\n", 
+				poptBadOption(pc, 0), poptStrerror(opt));
+			exit(1);
+		}
+	}
+
+	/* talloc_enable_leak_report_full(); */
+
+	/* setup the remaining options for the main program to use */
+	extra_argv = poptGetArgs(pc);
+	if (extra_argv) {
+		extra_argv++;
+		while (extra_argv[extra_argc]) extra_argc++;
+	}
+
+	ev = event_context_init(NULL);
+
+	ctdb = ctdb_cmdline_client(ev);
+
+	/* attach to a specific database */
+	ctdb_db = ctdb_attach(ctdb, "test.tdb");
+	if (!ctdb_db) {
+		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	printf("Waiting for cluster\n");
+	while (1) {
+		uint32_t recmode=1;
+		ctdb_ctrl_getrecmode(ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode);
+		if (recmode == 0) break;
+		event_loop_once(ev);
+	}
+
+	store_records(ctdb, ev);
+
+	return 0;
+}
diff --git a/source4/cluster/ctdb/tests/ctdb_test.c b/source4/cluster/ctdb/tests/ctdb_test.c
deleted file mode 100644
index bb888b5c7c..0000000000
--- a/source4/cluster/ctdb/tests/ctdb_test.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/* 
-   ctdb test harness
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "cmdline.h"
-
-enum my_functions {FUNC_SORT=1, FUNC_FETCH=2};
-
-static int int_compare(int *i1, int *i2)
-{
-	return *i1 - *i2;
-}
-
-/*
-  add an integer into a record in sorted order
-*/
-static int sort_func(struct ctdb_call_info *call)
-{
-	if (call->call_data == NULL ||
-	    call->call_data->dsize != sizeof(int)) {
-		return CTDB_ERR_INVALID;
-	}
-	call->new_data = talloc(call, TDB_DATA);
-	if (call->new_data == NULL) {
-		return CTDB_ERR_NOMEM;
-	}
-	call->new_data->dptr = talloc_size(call, 
-					   call->record_data.dsize + 
-					   call->call_data->dsize);
-	if (call->new_data->dptr == NULL) {
-		return CTDB_ERR_NOMEM;
-	}
-	call->new_data->dsize = call->record_data.dsize + call->call_data->dsize;
-	memcpy(call->new_data->dptr,
-	       call->record_data.dptr, call->record_data.dsize);
-	memcpy(call->new_data->dptr+call->record_data.dsize,
-	       call->call_data->dptr, call->call_data->dsize);
-
-	qsort(call->new_data->dptr, call->new_data->dsize / sizeof(int),
-	      sizeof(int), (comparison_fn_t)int_compare);
-
-	return 0;
-}
-
-/*
-  ctdb call function to fetch a record
-*/
-static int fetch_func(struct ctdb_call_info *call)
-{
-	call->reply_data = &call->record_data;
-	return 0;
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int i, ret;
-	poptContext pc;
-	struct event_context *ev;
-	struct ctdb_call call;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
-	if (!ctdb_db) {
-		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* setup a ctdb call function */
-	ret = ctdb_set_call(ctdb_db, sort_func,  FUNC_SORT);
-	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
-
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-	ctdb_connect_wait(ctdb);
-
-	ZERO_STRUCT(call);
-	call.key.dptr = discard_const("test");
-	call.key.dsize = strlen("test")+1;
-
-	/* add some random data */
-	for (i=0;i<10;i++) {
-		int v = random() % 1000;
-
-		call.call_id = FUNC_SORT;
-		call.call_data.dptr = (uint8_t *)&v;
-		call.call_data.dsize = sizeof(v);
-
-		ret = ctdb_call(ctdb_db, &call);
-		if (ret == -1) {
-			printf("ctdb_call FUNC_SORT failed - %s\n", ctdb_errstr(ctdb));
-			exit(1);
-		}
-	}
-
-	/* fetch the record */
-	call.call_id = FUNC_FETCH;
-	call.call_data.dptr = NULL;
-	call.call_data.dsize = 0;
-
-	ret = ctdb_call(ctdb_db, &call);
-	if (ret == -1) {
-		printf("ctdb_call FUNC_FETCH failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	for (i=0;i<call.reply_data.dsize/sizeof(int);i++) {
-		printf("%3d\n", ((int *)call.reply_data.dptr)[i]);
-	}
-	talloc_free(call.reply_data.dptr);
-
-	/* go into a wait loop to allow other nodes to complete */
-	ctdb_shutdown(ctdb);
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/ctdbd.sh b/source4/cluster/ctdb/tests/ctdbd.sh
new file mode 100755
index 0000000000..4a1e9976b2
--- /dev/null
+++ b/source4/cluster/ctdb/tests/ctdbd.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+killall -q ctdbd
+
+tests/start_daemons.sh 2 tests/nodes.txt || exit 1
+
+echo "Testing ping"
+$VALGRIND bin/ctdb ping || exit 1
+
+echo "Testing status"
+$VALGRIND bin/ctdb status || exit 1
+
+echo "Testing statistics"
+$VALGRIND bin/ctdb -n all statistics || exit 1
+
+echo "Testing statisticsreset"
+$VALGRIND bin/ctdb -n all statisticsreset || exit 1
+
+echo "Testing debug"
+$VALGRIND bin/ctdb -n all setdebug 3 || exit 1
+$VALGRIND bin/ctdb -n all getdebug || exit 1
+$VALGRIND bin/ctdb -n all setdebug 0 || exit 1
+$VALGRIND bin/ctdb -n all getdebug || exit 1
+
+echo "Attaching to some databases"
+$VALGRIND bin/ctdb attach test1.tdb || exit 1
+$VALGRIND bin/ctdb attach test2.tdb || exit 1
+
+echo "Testing getdbmap"
+$VALGRIND bin/ctdb getdbmap || exit 1
+
+echo "Testing status"
+$VALGRIND bin/ctdb status || exit 1
+
+echo "Testing variables"
+$VALGRIND bin/ctdb listvars || exit 1
+$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
+$VALGRIND bin/ctdb setvar TraverseTimeout 10 || exit 1
+$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
+
+sleep 1
+
+echo "Testing shutdown"
+$VALGRIND bin/ctdb shutdown -n all || exit 1
+
+sleep 1
+
+echo "All done"
+killall -q ctdbd
+exit 0
diff --git a/source4/cluster/ctdb/tests/events b/source4/cluster/ctdb/tests/events
new file mode 100755
index 0000000000..fb319bc426
--- /dev/null
+++ b/source4/cluster/ctdb/tests/events
@@ -0,0 +1,68 @@
+#!/bin/sh
+# event script for 'make test'
+
+cmd="$1"
+shift
+
+case $cmd in 
+    monitor)
+	echo "`date` monitor event"
+	exit 0
+	;;
+     startup)
+	echo "`date` ctdb startup event"
+	exit 0;	
+	;;
+	
+     takeip)
+	if [ $# != 3 ]; then
+	   echo "must supply interface, IP and maskbits"
+	   exit 1
+	fi
+	iface=$1
+	ip=$2
+	maskbits=$3
+
+	[ `id -u` = 0 ] && {
+	    /sbin/ip addr add $ip/$maskbits dev $iface || {
+		echo "`/bin/date` Failed to add $ip/$maskbits on dev $iface"
+		exit 1
+	    }
+	}
+	exit 0;
+	;;
+
+
+     ##################################################
+     # called when ctdbd wants to release an IP address
+     releaseip)
+	if [ $# != 3 ]; then
+	   echo "`/bin/date` must supply interface, IP and maskbits"
+	   exit 1
+	fi
+	iface=$1
+	ip=$2
+	maskbits=$3
+	[ `id -u` = 0 ] && {
+	    /sbin/ip addr del $ip/$maskbits dev $iface || {
+		echo "`/bin/date` Failed to del $ip on dev $iface"
+		exit 1
+	    }
+	}
+	echo "`date` ctdb takeip event for $1 $2 $3"
+	exit 0
+	;;
+
+     recovered)
+	echo "`date` ctdb recovered event"
+	exit 0
+	;;
+
+     shutdown)
+	echo "`date` ctdb shutdown event"
+	exit 0
+	;;
+esac
+
+echo "`/bin/date` Invalid command $cmd"
+exit 1
diff --git a/source4/cluster/ctdb/tests/fetch.sh b/source4/cluster/ctdb/tests/fetch.sh
index 73192e70ae..325957e63f 100755
--- a/source4/cluster/ctdb/tests/fetch.sh
+++ b/source4/cluster/ctdb/tests/fetch.sh
@@ -1,15 +1,24 @@
 #!/bin/sh
 
-killall -q ctdb_fetch
+NUMNODES=2
+if [ $# -gt 0 ]; then
+    NUMNODES=$1
+fi
 
-echo "Trying 2 nodes"
-$VALGRIND bin/ctdb_fetch --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/nodes.txt --listen 127.0.0.1:9001 $* 
-wait
+rm -f nodes.txt
+for i in `seq 1 $NUMNODES`; do
+  echo 127.0.0.$i >> nodes.txt
+done
+
+tests/start_daemons.sh $NUMNODES nodes.txt || exit 1
 
-echo "Trying 4 nodes"
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.4:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.3:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.2:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.1:9001 $* 
+
+killall -9 -q ctdb_fetch
+for i in `seq 1 $NUMNODES`; do
+  $VALGRIND bin/ctdb_fetch --socket sock.$i -n $NUMNODES $* &
+done
 wait
+
+echo "Shutting down"
+bin/ctdb shutdown -n all --socket=sock.1
+exit 0
diff --git a/source4/cluster/ctdb/tests/fetch1.sh b/source4/cluster/ctdb/tests/fetch1.sh
deleted file mode 100755
index db584ec012..0000000000
--- a/source4/cluster/ctdb/tests/fetch1.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_fetch1
-
-echo "Trying node"
-bin/ctdb_fetch1 --nlist tests/1node.txt --listen 127.0.0.1:9001 $* 
-wait
diff --git a/source4/cluster/ctdb/tests/lockwait.c b/source4/cluster/ctdb/tests/lockwait.c
deleted file mode 100644
index 6cefe5680a..0000000000
--- a/source4/cluster/ctdb/tests/lockwait.c
+++ /dev/null
@@ -1,244 +0,0 @@
-/* 
-   test a lock wait idea
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "system/wait.h"
-#include "popt.h"
-#include "cmdline.h"
-
-
-struct lockwait_handle {
-	struct fd_event *fde;
-	int fd[2];
-	pid_t child;
-	void *private_data;
-	void (*callback)(void *);
-};
-
-static void lockwait_handler(struct event_context *ev, struct fd_event *fde, 
-			     uint16_t flags, void *private_data)
-{
-	struct lockwait_handle *h = talloc_get_type(private_data, 
-						     struct lockwait_handle);
-	void (*callback)(void *) = h->callback;
-	void *p = h->private_data;
-	talloc_set_destructor(h, NULL);
-	close(h->fd[0]);
-	talloc_free(h);	
-	callback(p);
-	waitpid(h->child, NULL, 0);
-}
-
-static int lockwait_destructor(struct lockwait_handle *h)
-{
-	close(h->fd[0]);
-	kill(h->child, SIGKILL);
-	waitpid(h->child, NULL, 0);
-	return 0;
-}
-
-
-static struct lockwait_handle *lockwait(struct event_context *ev, 
-					TALLOC_CTX *mem_ctx,
-					int fd, off_t ofs, size_t len,
-					void (*callback)(void *), void *private_data)
-{
-	struct lockwait_handle *h;
-	int ret;
-
-	h = talloc_zero(mem_ctx, struct lockwait_handle);
-	if (h == NULL) {
-		return NULL;
-	}
-
-	ret = pipe(h->fd);
-	if (ret != 0) {
-		talloc_free(h);
-		return NULL;
-	}
-
-	h->child = fork();
-	if (h->child == (pid_t)-1) {
-		close(h->fd[0]);
-		close(h->fd[1]);
-		talloc_free(h);
-		return NULL;
-	}
-
-	h->callback = callback;
-	h->private_data = private_data;
-
-	if (h->child == 0) {
-		/* in child */
-		struct flock lock;
-		close(h->fd[0]);
-		lock.l_type = F_WRLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start = ofs;
-		lock.l_len = len;
-		lock.l_pid = 0;
-		fcntl(fd,F_SETLKW,&lock);
-		_exit(0);
-	}
-
-	close(h->fd[1]);
-	talloc_set_destructor(h, lockwait_destructor);
-
-	h->fde = event_add_fd(ev, h, h->fd[0], EVENT_FD_READ, lockwait_handler, h);
-	if (h->fde == NULL) {
-		talloc_free(h);
-		return NULL;
-	}
-
-	return h;
-}
-
-
-
-
-static void fcntl_lock_callback(void *p)
-{
-	int *got_lock = (int *)p;
-	*got_lock = 1;
-}
-
-/*
-  get an fcntl lock - waiting if necessary
- */
-static int fcntl_lock(struct event_context *ev,
-		      int fd, int op, off_t offset, off_t count, int type)
-{
-	struct flock lock;
-	int ret;
-	int use_lockwait = (op == F_SETLKW);
-	int got_lock = 0;
-
-	lock.l_type = type;
-	lock.l_whence = SEEK_SET;
-	lock.l_start = offset;
-	lock.l_len = count;
-	lock.l_pid = 0;
-
-	do {
-		ret = fcntl(fd,use_lockwait?F_SETLK:op,&lock);
-		if (ret == 0) {
-			return 0;
-		}
-		if (ret == -1 && 
-		    (errno == EACCES || errno == EAGAIN || errno == EDEADLK)) {
-			struct lockwait_handle *h;
-			h = lockwait(ev, ev, fd, offset, count, 
-				     fcntl_lock_callback, &got_lock);
-			if (h == NULL) {
-				errno = ENOLCK;
-				return -1;
-			}
-			/* in real code we would return to the event loop */
-			while (!got_lock) {
-				event_loop_once(ev);
-			}
-			got_lock = 0;
-		}
-	} while (!got_lock);
-
-	return ret;
-}
-
-static void child(struct event_context *ev, int n)
-{
-	int fd;
-	int count=0;
-	struct timeval tv;
-	fd = open("test.dat", O_CREAT|O_RDWR, 0666);
-	if (fd == -1) {
-		perror("test.dat");
-		exit(1);
-	}
-
-	tv = timeval_current();
-
-	while (timeval_elapsed(&tv) < 10) {
-		int ret;
-		ret = fcntl_lock(ev, fd, F_SETLKW, 0, 1, F_WRLCK);
-		if (ret != 0) {
-			printf("Failed to get lock in child %d!\n", n);
-			break;
-		}
-		fcntl_lock(ev, fd, F_SETLK, 0, 1, F_UNLCK);
-		count++;
-	}
-
-	printf("child %2d %.0f ops/sec\n", n, count/timeval_elapsed(&tv));
-	_exit(0);
-}
-
-static int timelimit = 10;
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	pid_t *pids;
-	int nprogs = 2;
-	int i;
-	struct event_context *ev;
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
-		{ "num-progs", 'n', POPT_ARG_INT, &nprogs, 0, "num_progs", "integer" },
-		POPT_TABLEEND
-	};
-	poptContext pc;
-	int opt;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	ev = event_context_init(NULL);
-
-	pids = talloc_array(ev, pid_t, nprogs);
-
-	/* create N processes fighting over the same lock */
-	for (i=0;i<nprogs;i++) {
-		pids[i] = fork();
-		if (pids[i] == 0) {
-			child(ev, i);
-		}
-	}
-
-	printf("Waiting for %d children ...\n", nprogs);
-
-	/* wait for our kids to finish playing */
-	for (i=0;i<nprogs;i++) {
-		waitpid(pids[i], NULL, 0);
-	}	
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/messaging.sh b/source4/cluster/ctdb/tests/messaging.sh
deleted file mode 100755
index 46f3e4dc77..0000000000
--- a/source4/cluster/ctdb/tests/messaging.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_messaging
-
-echo "Trying 2 nodes"
-bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
-bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.1:9001 $*
-wait
diff --git a/source4/cluster/ctdb/tests/nodes.txt b/source4/cluster/ctdb/tests/nodes.txt
index e1198b59ac..99b07328b3 100644
--- a/source4/cluster/ctdb/tests/nodes.txt
+++ b/source4/cluster/ctdb/tests/nodes.txt
@@ -1,2 +1,4 @@
-127.0.0.1:9001
-127.0.0.2:9001
+127.0.0.1
+127.0.0.2
+127.0.0.3
+127.0.0.4
diff --git a/source4/cluster/ctdb/tests/public_addresses b/source4/cluster/ctdb/tests/public_addresses
new file mode 100644
index 0000000000..97c85af7a8
--- /dev/null
+++ b/source4/cluster/ctdb/tests/public_addresses
@@ -0,0 +1,4 @@
+10.99.99.0/24
+10.99.99.1/24
+10.99.99.2/24
+10.99.99.3/24
diff --git a/source4/cluster/ctdb/tests/recover.sh b/source4/cluster/ctdb/tests/recover.sh
new file mode 100755
index 0000000000..c626441786
--- /dev/null
+++ b/source4/cluster/ctdb/tests/recover.sh
@@ -0,0 +1,107 @@
+#!/bin/sh
+
+killall -q ctdbd
+
+echo "Starting 4 ctdb daemons"
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.2 --socket=/tmp/ctdb.socket.127.0.0.2
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.3 --socket=/tmp/ctdb.socket.127.0.0.3
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.4 --socket=/tmp/ctdb.socket.127.0.0.4
+
+echo
+echo "Attaching to some databases"
+bin/ctdb_control attach test1.tdb || exit 1
+bin/ctdb_control attach test2.tdb || exit 1
+bin/ctdb_control attach test3.tdb || exit 1
+bin/ctdb_control attach test4.tdb || exit 1
+
+echo "Clearing all databases to make sure they are all empty"
+bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^dbid://" -e "s/ .*$//" | while read DB; do
+	seq 0 3 | while read NODE; do
+		bin/ctdb_control cleardb $NODE $DB
+	done
+done
+
+
+echo
+echo
+echo "Printing all databases on all nodes. they should all be empty"
+echo "============================================================="
+bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^.*name://" -e "s/ .*$//" | while read DBNAME; do
+	seq 0 3 | while read NODE; do
+		echo "Content of DBNAME:$DBNAME NODE:$NODE :"
+		bin/ctdb_control catdb $DBNAME $NODE
+	done
+done
+
+echo
+echo
+echo "Populating the databases"
+./bin/ctdb_control writerecord 0 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 0 0x220c2a7b 1
+
+./bin/ctdb_control writerecord 1 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 1 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 1 0x220c2a7b 2
+
+./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 2 0x220c2a7b 3
+
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 3 0x220c2a7b 3
+
+
+echo
+echo
+echo "Printing all databases on all nodes. there should be a record there"
+echo "============================================================="
+bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^.*name://" -e "s/ .*$//" | while read DBNAME; do
+	seq 0 3 | while read NODE; do
+		echo "Content of DBNAME:$DBNAME NODE:$NODE :"
+		bin/ctdb_control catdb $DBNAME $NODE
+	done
+done
+
+echo
+echo
+echo "killing off node #2"
+echo "==================="
+CTDBPID=`./bin/ctdb_control getpid 2 | sed -e "s/Pid://"`
+kill $CTDBPID
+sleep 1
+
+
+echo
+echo
+echo "wait 3 seconds to let the recovery daemon do its job"
+echo "===================================================="
+sleep 3
+
+echo
+echo
+echo "Printing all databases on all nodes."
+echo "The databases should be the same now on all nodes"
+echo "and the record will have been migrated to node 0"
+echo "================================================="
+echo "Node 0:"
+bin/ctdb_control catdb test4.tdb 0
+echo "Node 1:"
+bin/ctdb_control catdb test4.tdb 1
+echo "Node 3:"
+bin/ctdb_control catdb test4.tdb 3
+echo "nodemap:"
+bin/ctdb_control getnodemap 0
+
+echo
+echo
+echo "Traverse the cluster and dump the database"
+bin/ctdb_control catdb test4.tdb
+
+
+#leave the ctdb daemons running   so one can look at the box in more detail
+#killall -q ctdbd
diff --git a/source4/cluster/ctdb/tests/run_tests.sh b/source4/cluster/ctdb/tests/run_tests.sh
new file mode 100755
index 0000000000..356a9b21a0
--- /dev/null
+++ b/source4/cluster/ctdb/tests/run_tests.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+tests/fetch.sh 4 || exit 1
+tests/bench.sh 4 || exit 1
+tests/ctdbd.sh || exit 1
+
+echo "All OK"
+exit 0
diff --git a/source4/cluster/ctdb/tests/start_daemons.sh b/source4/cluster/ctdb/tests/start_daemons.sh
new file mode 100755
index 0000000000..d3c99d075a
--- /dev/null
+++ b/source4/cluster/ctdb/tests/start_daemons.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+NUMNODES="$1"
+NODES=$2
+shift
+shift
+
+killall -q ctdbd
+
+CTDB_OPTIONS="--reclock=rec.lock --nlist $NODES --event-script=tests/events --logfile=-  --dbdir=test.db $*"
+if [ `id -u` -eq 0 ]; then
+    CTDB_OPTIONS="$CTDB_OPTIONS --public-addresses=tests/public_addresses --public-interface=lo"
+fi
+
+echo "Starting $NUMNODES ctdb daemons"
+for i in `seq 1 $NUMNODES`; do
+    $VALGRIND bin/ctdbd --socket=sock.$i $CTDB_OPTIONS || exit 1
+done
+ln -sf $PWD/sock.1 /tmp/ctdb.socket || exit 1
+
+while bin/ctdb status | grep RECOVERY > /dev/null; do
+    echo "`date` Waiting for recovery"
+    sleep 1;
+done
+
+echo "$NUMNODES daemons started"
+
+exit 0
diff --git a/source4/cluster/ctdb/tests/test.sh b/source4/cluster/ctdb/tests/test.sh
deleted file mode 100755
index 23fdb8d6ce..0000000000
--- a/source4/cluster/ctdb/tests/test.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_test
-
-
-echo "Trying 2 nodes ..."
-$VALGRIND bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.1:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.2:9001
-wait
-
-echo "Trying 4 nodes ..."
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.1:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.2:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.3:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.4:9001
-wait
-
diff --git a/source4/cluster/ctdb/tests/test1.sh b/source4/cluster/ctdb/tests/test1.sh
deleted file mode 100755
index 42256f22f9..0000000000
--- a/source4/cluster/ctdb/tests/test1.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-
-echo "Testing daemon mode"
-bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001
-wait
-
-echo "Testing self connect"
-bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001 --self-connect
-wait