r22231: merge from bzr ctdb tree

(This used to be commit 807b959082d3b9a929c9f6597714e636638a940e)

16 files changed, 633 insertions, 12 deletions

diff --git a/source4/cluster/ctdb/tests/1node.txt b/source4/cluster/ctdb/tests/1node.txt
new file mode 100644
index 0000000000..db4350c0c0
--- /dev/null
+++ b/source4/cluster/ctdb/tests/1node.txt
@@ -0,0 +1 @@
+127.0.0.1:9001
diff --git a/source4/cluster/ctdb/tests/4nodes.txt b/source4/cluster/ctdb/tests/4nodes.txt
new file mode 100644
index 0000000000..880fe914ff
--- /dev/null
+++ b/source4/cluster/ctdb/tests/4nodes.txt
@@ -0,0 +1,4 @@
+127.0.0.1:9001
+127.0.0.2:9001
+127.0.0.3:9001
+127.0.0.4:9001
diff --git a/source4/cluster/ctdb/tests/bench-ssh.sh b/source4/cluster/ctdb/tests/bench-ssh.sh
new file mode 100755
index 0000000000..0d11ee9cdd
--- /dev/null
+++ b/source4/cluster/ctdb/tests/bench-ssh.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+if [ $# -lt 1 ]; then
+    echo "Usage: bench-ssh.sh <NODES> <OPTIONS>"
+    exit 1
+fi
+
+while :; do
+    if [ "`echo $1 | cut -c1`" = "-" -o $# -eq 0 ]; then break; fi
+    nodes="$nodes $1";
+    shift;
+done
+
+options=$*
+dir=`pwd`
+
+echo "Creating nodes-ssh.txt"
+rm -f nodes-ssh.txt
+count=0
+for h in $nodes; do
+    echo "$h:9001" >> nodes-ssh.txt
+    count=`expr $count + 1`
+done
+
+
+echo "Killing old processes"
+for h in $nodes; do
+    scp -q nodes-ssh.txt $h:$dir
+    ssh $h killall -q ctdb_bench
+done
+
+echo "Starting nodes"
+i=0
+for h in $nodes; do
+    if [ $i -eq `expr $count - 1` ]; then
+	ssh $h $dir/bin/ctdb_bench --nlist $dir/nodes-ssh.txt --listen $h:9001 $options
+    else
+	ssh -f $h $dir/bin/ctdb_bench --nlist $dir/nodes-ssh.txt --listen $h:9001 $options
+    fi
+    i=`expr $i + 1`
+done
+
+wait
diff --git a/source4/cluster/ctdb/tests/bench.sh b/source4/cluster/ctdb/tests/bench.sh
new file mode 100755
index 0000000000..50e9e08f99
--- /dev/null
+++ b/source4/cluster/ctdb/tests/bench.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+killall -q ctdb_bench
+
+echo "Trying 2 nodes"
+bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
+bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.1:9001 $*
+
+killall -q ctdb_bench
diff --git a/source4/cluster/ctdb/tests/bench1.sh b/source4/cluster/ctdb/tests/bench1.sh
new file mode 100755
index 0000000000..3481d82be2
--- /dev/null
+++ b/source4/cluster/ctdb/tests/bench1.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+killall -q ctdb_bench
+
+echo "Trying 1 nodes"
+bin/ctdb_bench --nlist tests/1node.txt --listen 127.0.0.2:9001 $*
+
+killall -q ctdb_bench
diff --git a/source4/cluster/ctdb/tests/ctdb_bench.c b/source4/cluster/ctdb/tests/ctdb_bench.c
index 023c76e7f9..78d66f6f2c 100644
--- a/source4/cluster/ctdb/tests/ctdb_bench.c
+++ b/source4/cluster/ctdb/tests/ctdb_bench.c
@@ -130,10 +130,10 @@ static int msg_plus, msg_minus;
   handler for messages in bench_ring()
 */
 static void ring_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-				 TDB_DATA data, void *private)
+				 TDB_DATA data, void *private_data)
 {
 	int incr = *(int *)data.dptr;
-	int *count = (int *)private;
+	int *count = (int *)private_data;
 	int dest;
 	(*count)++;
 	dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
@@ -200,6 +200,7 @@ int main(int argc, const char *argv[])
 	const char *transport = "tcp";
 	const char *myaddress = NULL;
 	int self_connect=0;
+	int daemon_mode=0;
 
 	struct poptOption popt_options[] = {
 		POPT_AUTOHELP
@@ -207,6 +208,7 @@ int main(int argc, const char *argv[])
 		{ "listen", 0, POPT_ARG_STRING, &myaddress, 0, "address to listen on", "address" },
 		{ "transport", 0, POPT_ARG_STRING, &transport, 0, "protocol transport", NULL },
 		{ "self-connect", 0, POPT_ARG_NONE, &self_connect, 0, "enable self connect", "boolean" },
+		{ "daemon", 0, POPT_ARG_NONE, &daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
 		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
 		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
 		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
@@ -254,6 +256,9 @@ int main(int argc, const char *argv[])
 	if (self_connect) {
 		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
 	}
+	if (daemon_mode) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
+	}
 
 	ret = ctdb_set_transport(ctdb, transport);
 	if (ret == -1) {
@@ -286,11 +291,11 @@ int main(int argc, const char *argv[])
 	ret = ctdb_set_call(ctdb_db, incr_func,  FUNC_INCR);
 	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
 
-	ctdb_set_message_handler(ctdb, ring_message_handler, &msg_count);
-
 	/* start the protocol running */
 	ret = ctdb_start(ctdb);
 
+	ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count);
+
 	/* wait until all nodes are connected (should not be needed
 	   outside of test code) */
 	ctdb_connect_wait(ctdb);
diff --git a/source4/cluster/ctdb/tests/ctdb_fetch.c b/source4/cluster/ctdb/tests/ctdb_fetch.c
index c0491e9bb5..45e5248980 100644
--- a/source4/cluster/ctdb/tests/ctdb_fetch.c
+++ b/source4/cluster/ctdb/tests/ctdb_fetch.c
@@ -87,7 +87,7 @@ static void bench_fetch_1node(struct ctdb_context *ctdb)
 						      msg_count, ctdb_get_vnn(ctdb));
 	data.dsize = strlen((const char *)data.dptr)+1;
 
-	ret = ctdb_record_store(rec, data);
+	ret = ctdb_store_unlock(rec, data);
 	if (ret != 0) {
 		printf("Failed to store record\n");
 	}
@@ -106,7 +106,7 @@ static void bench_fetch_1node(struct ctdb_context *ctdb)
   handler for messages in bench_ring()
 */
 static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-			    TDB_DATA data, void *private)
+			    TDB_DATA data, void *private_data)
 {
 	msg_count++;
 	bench_fetch_1node(ctdb);
@@ -167,6 +167,7 @@ int main(int argc, const char *argv[])
 	const char *transport = "tcp";
 	const char *myaddress = NULL;
 	int self_connect=0;
+	int daemon_mode=0;
 
 	struct poptOption popt_options[] = {
 		POPT_AUTOHELP
@@ -174,6 +175,7 @@ int main(int argc, const char *argv[])
 		{ "listen", 0, POPT_ARG_STRING, &myaddress, 0, "address to listen on", "address" },
 		{ "transport", 0, POPT_ARG_STRING, &transport, 0, "protocol transport", NULL },
 		{ "self-connect", 0, POPT_ARG_NONE, &self_connect, 0, "enable self connect", "boolean" },
+		{ "daemon", 0, POPT_ARG_NONE, &daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
 		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
 		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
 		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
@@ -222,6 +224,9 @@ int main(int argc, const char *argv[])
 	if (self_connect) {
 		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
 	}
+	if (daemon_mode) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
+	}
 
 	ret = ctdb_set_transport(ctdb, transport);
 	if (ret == -1) {
@@ -252,11 +257,11 @@ int main(int argc, const char *argv[])
 
 	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
 
-	ctdb_set_message_handler(ctdb, message_handler, &msg_count);
-
 	/* start the protocol running */
 	ret = ctdb_start(ctdb);
 
+	ctdb_set_message_handler(ctdb, 0, message_handler, &msg_count);
+
 	/* wait until all nodes are connected (should not be needed
 	   outside of test code) */
 	ctdb_connect_wait(ctdb);
diff --git a/source4/cluster/ctdb/tests/ctdb_fetch1.c b/source4/cluster/ctdb/tests/ctdb_fetch1.c
new file mode 100644
index 0000000000..8071256a43
--- /dev/null
+++ b/source4/cluster/ctdb/tests/ctdb_fetch1.c
@@ -0,0 +1,274 @@
+/* 
+   simple ctdb fetch test
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "popt.h"
+#include "ctdb.h"
+#include "ctdb_private.h"
+
+#define PARENT_SRVID	0
+#define CHILD1_SRVID	1
+#define CHILD2_SRVID	2
+
+int num_msg=0;
+
+static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+			    TDB_DATA data, void *private_data)
+{
+	num_msg++;
+}
+static void child_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+			    TDB_DATA data, void *private_data)
+{
+	num_msg++;
+}
+
+void test1(struct ctdb_db_context *ctdb_db)
+{
+	struct ctdb_record_handle *rh;
+	TDB_DATA key, data, data2, store_data;
+	int ret;
+ 
+	/* 
+	   test 1 : write data and read it back.   should all be the same
+	 */
+	printf("Test1: write and verify we can read it back: ");
+	key.dptr  = discard_const("Record");
+	key.dsize = strlen((const char *)key.dptr)+1;
+	rh = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data);
+
+	store_data.dptr  = discard_const("data to store");
+	store_data.dsize = strlen((const char *)store_data.dptr)+1;
+	ret = ctdb_store_unlock(rh, store_data);
+
+	rh = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data2);
+	/* hopefully   data2 will now contain the record written above */
+	if (!strcmp("data to store", (const char *)data2.dptr)) {
+		printf("SUCCESS\n");
+	} else {
+		printf("FAILURE\n");
+		exit(10);
+	}
+	
+	/* just write it back to unlock it */
+	ret = ctdb_store_unlock(rh, store_data);
+}
+
+void child(int srvid, struct event_context *ev, struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db)
+{
+	TDB_DATA data;
+	struct ctdb_record_handle *rh;
+	TDB_DATA key, data2;
+
+	data.dptr=discard_const("dummy message");
+	data.dsize=strlen((const char *)data.dptr)+1;
+
+	ctdb_set_message_handler(ctdb, srvid, child_handler, NULL);
+
+	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), PARENT_SRVID, data);
+	while (num_msg==0) {
+		event_loop_once(ev);
+	}
+
+
+	/* fetch and lock the record */
+	key.dptr  = discard_const("Record");
+	key.dsize = strlen((const char *)key.dptr)+1;
+	rh = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data2);
+	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), PARENT_SRVID, data);
+
+
+	while (1) {
+		event_loop_once(ev);
+	}
+}
+
+/*
+  main program
+*/
+int main(int argc, const char *argv[])
+{
+	struct ctdb_context *ctdb;
+	struct ctdb_db_context *ctdb_db;
+	const char *nlist = NULL;
+	const char *transport = "tcp";
+	const char *myaddress = NULL;
+	int self_connect=0;
+	int daemon_mode=0;
+	TDB_DATA data;
+
+	struct poptOption popt_options[] = {
+		POPT_AUTOHELP
+		{ "nlist", 0, POPT_ARG_STRING, &nlist, 0, "node list file", "filename" },
+		{ "listen", 0, POPT_ARG_STRING, &myaddress, 0, "address to listen on", "address" },
+		{ "transport", 0, POPT_ARG_STRING, &transport, 0, "protocol transport", NULL },
+		{ "self-connect", 0, POPT_ARG_NONE, &self_connect, 0, "enable self connect", "boolean" },
+		{ "daemon", 0, POPT_ARG_NONE, &daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
+		POPT_TABLEEND
+	};
+	int opt;
+	const char **extra_argv;
+	int extra_argc = 0;
+	int ret;
+	poptContext pc;
+	struct event_context *ev;
+
+	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
+
+	while ((opt = poptGetNextOpt(pc)) != -1) {
+		switch (opt) {
+		default:
+			fprintf(stderr, "Invalid option %s: %s\n", 
+				poptBadOption(pc, 0), poptStrerror(opt));
+			exit(1);
+		}
+	}
+
+	/* setup the remaining options for the main program to use */
+	extra_argv = poptGetArgs(pc);
+	if (extra_argv) {
+		extra_argv++;
+		while (extra_argv[extra_argc]) extra_argc++;
+	}
+
+	if (nlist == NULL || myaddress == NULL) {
+		printf("You must provide a node list with --nlist and an address with --listen\n");
+		exit(1);
+	}
+
+	ev = event_context_init(NULL);
+
+	/* initialise ctdb */
+	ctdb = ctdb_init(ev);
+	if (ctdb == NULL) {
+		printf("Failed to init ctdb\n");
+		exit(1);
+	}
+
+	if (self_connect) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
+	}
+	if (daemon_mode) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
+	}
+
+	ret = ctdb_set_transport(ctdb, transport);
+	if (ret == -1) {
+		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* tell ctdb what address to listen on */
+	ret = ctdb_set_address(ctdb, myaddress);
+	if (ret == -1) {
+		printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* tell ctdb what nodes are available */
+	ret = ctdb_set_nlist(ctdb, nlist);
+	if (ret == -1) {
+		printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* attach to a specific database */
+	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	if (!ctdb_db) {
+		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* start the protocol running */
+	ret = ctdb_start(ctdb);
+
+#if 0
+	/* wait until all nodes are connected (should not be needed
+	   outside of test code) */
+	ctdb_connect_wait(ctdb);
+#endif
+
+	/*
+	   start two child processes
+	 */
+	if(fork()){
+		/* 
+		   set up a message handler so our child processes can talk to us
+		 */
+		ctdb_set_message_handler(ctdb, PARENT_SRVID, message_handler, NULL);
+	} else {
+		sleep(3);
+		if(!fork()){
+			child(CHILD1_SRVID, ev, ctdb, ctdb_db);
+		} else {
+			child(CHILD2_SRVID, ev, ctdb, ctdb_db);
+		}
+	}
+
+	/* 
+	   test 1 : write data and read it back.
+	 */
+	test1(ctdb_db); 
+
+	/* 
+	   wait until both children have sent us a message they have started
+	 */
+	printf("Wait for both child processes to start: ");
+	while (num_msg!=2) {
+		event_loop_once(ev);
+	}
+	printf("STARTED\n");
+
+
+	/*
+	   send message to child 1 to make it to fetch and lock the record 
+	 */
+	data.dptr=discard_const("dummy message");
+	data.dsize=strlen((const char *)data.dptr)+1;
+	printf("Send message to child 1 to fetch_lock the record\n");
+	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD1_SRVID, data);
+
+	/* wait for child 1 to complete fetching and locking the record */
+	while (num_msg!=3) {
+		event_loop_once(ev);
+	}
+	printf("Child 1 has fetched and locked the record\n");
+
+	/* now tell child 2 to fetch and lock the same record */
+	printf("Send message to child 2 to fetch_lock the record\n");
+	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD2_SRVID, data);
+
+	/* wait for child 2 to complete fetching and locking the record */
+	while (num_msg!=4) {
+		event_loop_once(ev);
+	}
+	printf("Child 2 has fetched and locked the record\n");
+
+
+	while (1) {
+		event_loop_once(ev);
+	}
+
+	/* shut it down */
+	talloc_free(ctdb);
+	return 0;
+}
diff --git a/source4/cluster/ctdb/tests/ctdb_messaging.c b/source4/cluster/ctdb/tests/ctdb_messaging.c
new file mode 100644
index 0000000000..050f14de38
--- /dev/null
+++ b/source4/cluster/ctdb/tests/ctdb_messaging.c
@@ -0,0 +1,187 @@
+/* 
+   test of messaging
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "popt.h"
+
+static int timelimit = 10;
+static int num_records = 10;
+static int num_msgs = 1;
+static int num_repeats = 100;
+static int num_clients = 2;
+
+
+/*
+  handler for messages in bench_ring()
+*/
+static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+				 TDB_DATA data, void *private_data)
+{
+	printf("client vnn:%d received a message to srvid:%d [%s]\n",ctdb_get_vnn(ctdb),srvid,data.dptr);
+	fflush(stdout);
+}
+
+/*
+  main program
+*/
+int main(int argc, const char *argv[])
+{
+	struct ctdb_context *ctdb;
+	struct ctdb_db_context *ctdb_db;
+	const char *nlist = NULL;
+	const char *transport = "tcp";
+	const char *myaddress = NULL;
+	int self_connect=0;
+	int daemon_mode=0;
+	char buf[256];
+
+	struct poptOption popt_options[] = {
+		POPT_AUTOHELP
+		{ "nlist", 0, POPT_ARG_STRING, &nlist, 0, "node list file", "filename" },
+		{ "listen", 0, POPT_ARG_STRING, &myaddress, 0, "address to listen on", "address" },
+		{ "transport", 0, POPT_ARG_STRING, &transport, 0, "protocol transport", NULL },
+		{ "self-connect", 0, POPT_ARG_NONE, &self_connect, 0, "enable self connect", "boolean" },
+		{ "daemon", 0, POPT_ARG_NONE, &daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
+		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
+		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
+		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
+		{ "num-clients", 0, POPT_ARG_INT, &num_clients, 0, "num_clients", "integer" },
+		POPT_TABLEEND
+	};
+	int opt;
+	const char **extra_argv;
+	int extra_argc = 0;
+	int ret, i, j;
+	poptContext pc;
+	struct event_context *ev;
+	pid_t pid;
+	int srvid;
+	TDB_DATA data;
+
+	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
+
+	while ((opt = poptGetNextOpt(pc)) != -1) {
+		switch (opt) {
+		default:
+			fprintf(stderr, "Invalid option %s: %s\n", 
+				poptBadOption(pc, 0), poptStrerror(opt));
+			exit(1);
+		}
+	}
+
+	/* setup the remaining options for the main program to use */
+	extra_argv = poptGetArgs(pc);
+	if (extra_argv) {
+		extra_argv++;
+		while (extra_argv[extra_argc]) extra_argc++;
+	}
+
+	if (nlist == NULL || myaddress == NULL) {
+		printf("You must provide a node list with --nlist and an address with --listen\n");
+		exit(1);
+	}
+
+	ev = event_context_init(NULL);
+
+	/* initialise ctdb */
+	ctdb = ctdb_init(ev);
+	if (ctdb == NULL) {
+		printf("Failed to init ctdb\n");
+		exit(1);
+	}
+
+	if (self_connect) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
+	}
+	if (daemon_mode) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
+	}
+
+	ret = ctdb_set_transport(ctdb, transport);
+	if (ret == -1) {
+		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* tell ctdb what address to listen on */
+	ret = ctdb_set_address(ctdb, myaddress);
+	if (ret == -1) {
+		printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* tell ctdb what nodes are available */
+	ret = ctdb_set_nlist(ctdb, nlist);
+	if (ret == -1) {
+		printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* attach to a specific database */
+	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	if (!ctdb_db) {
+		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* start the protocol running */
+	ret = ctdb_start(ctdb);
+
+	srvid = -1;
+	for (i=0;i<num_clients-1;i++) {
+		pid=fork();
+		if (pid) {
+			srvid = i;
+			break;
+		}
+	}
+	if (srvid == -1) {
+		srvid = num_clients-1;
+	}
+
+	ctdb_set_message_handler(ctdb, srvid, message_handler, NULL);
+
+	/* wait until all nodes are connected (should not be needed
+	   outside of test code) */
+	ctdb_connect_wait(ctdb);
+
+	sleep(3);
+
+	printf("sending message from vnn:%d to vnn:%d/srvid:%d\n",ctdb_get_vnn(ctdb),ctdb_get_vnn(ctdb), 1-srvid);
+	for (i=0;i<ctdb_get_num_nodes(ctdb);i++) {
+		for (j=0;j<num_clients;j++) {
+			printf("sending message to %d:%d\n", i, j);
+			sprintf(buf,"Message from %d to vnn:%d srvid:%d",ctdb_get_vnn(ctdb),i,j);
+			data.dptr=buf;
+			data.dsize=strlen(buf)+1;
+			ctdb_send_message(ctdb, i, j, data);
+		}
+	}
+
+	while (1) {
+		event_loop_once(ev);
+	}
+       
+	/* shut it down */
+	talloc_free(ctdb);
+	return 0;
+}
diff --git a/source4/cluster/ctdb/tests/ctdb_test.c b/source4/cluster/ctdb/tests/ctdb_test.c
index 908a2eaac7..62c7d1c552 100644
--- a/source4/cluster/ctdb/tests/ctdb_test.c
+++ b/source4/cluster/ctdb/tests/ctdb_test.c
@@ -81,6 +81,7 @@ int main(int argc, const char *argv[])
 	const char *transport = "tcp";
 	const char *myaddress = NULL;
 	int self_connect=0;
+	int daemon_mode=0;
 
 	struct poptOption popt_options[] = {
 		POPT_AUTOHELP
@@ -88,6 +89,7 @@ int main(int argc, const char *argv[])
 		{ "listen", 0, POPT_ARG_STRING, &myaddress, 0, "address to listen on", "address" },
 		{ "transport", 0, POPT_ARG_STRING, &transport, 0, "protocol transport", NULL },
 		{ "self-connect", 0, POPT_ARG_NONE, &self_connect, 0, "enable self connect", "boolean" },
+		{ "daemon", 0, POPT_ARG_NONE, &daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
 		POPT_TABLEEND
 	};
 	int opt;
@@ -134,6 +136,17 @@ int main(int argc, const char *argv[])
 		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
 	}
 
+	if (daemon_mode) {
+		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
+	}
+
+	/* this flag is only used by test code and it makes ctdb_start() block until all
+	   nodes have connected.
+	   until we do better recovery and cluster rebuild it is probably good to use this flag
+	   in applications.
+	 */
+	ctdb_set_flags(ctdb, CTDB_FLAG_CONNECT_WAIT);
+
 	ret = ctdb_set_transport(ctdb, transport);
 	if (ret == -1) {
 		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
@@ -168,10 +181,6 @@ int main(int argc, const char *argv[])
 	/* start the protocol running */
 	ret = ctdb_start(ctdb);
 
-	/* wait until all nodes are connected (should not be needed
-	   outide of test code) */
-	ctdb_connect_wait(ctdb);
-       
 	ZERO_STRUCT(call);
 	call.key.dptr = discard_const("test");
 	call.key.dsize = strlen("test")+1;
@@ -210,6 +219,11 @@ int main(int argc, const char *argv[])
 	/* go into a wait loop to allow other nodes to complete */
 	ctdb_wait_loop(ctdb);
 
+	/*talloc_report_full(ctdb, stdout);*/
+
+/* sleep for a while so that our daemon will remaining alive for the other nodes in the cluster */
+sleep(10);
+
 	/* shut it down */
 	talloc_free(ctdb);
 	return 0;
diff --git a/source4/cluster/ctdb/tests/fetch.sh b/source4/cluster/ctdb/tests/fetch.sh
new file mode 100755
index 0000000000..ba043e17c5
--- /dev/null
+++ b/source4/cluster/ctdb/tests/fetch.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+killall -q ctdb_fetch
+
+echo "Trying 2 nodes"
+bin/ctdb_fetch --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
+bin/ctdb_fetch --nlist tests/nodes.txt --listen 127.0.0.1:9001 $* 
+
+killall -q ctdb_fetch
diff --git a/source4/cluster/ctdb/tests/fetch1.sh b/source4/cluster/ctdb/tests/fetch1.sh
new file mode 100755
index 0000000000..4331a57d3e
--- /dev/null
+++ b/source4/cluster/ctdb/tests/fetch1.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+killall -q ctdb_fetch1
+
+echo "Trying node"
+bin/ctdb_fetch1 --nlist tests/1node.txt --listen 127.0.0.1:9001 --daemon $* 
+
+killall -q ctdb_fetch1
diff --git a/source4/cluster/ctdb/tests/messaging.sh b/source4/cluster/ctdb/tests/messaging.sh
new file mode 100755
index 0000000000..179a2bef88
--- /dev/null
+++ b/source4/cluster/ctdb/tests/messaging.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+killall -q ctdb_messaging
+
+echo "Trying 2 nodes"
+bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
+bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.1:9001 $*
+
+killall -q ctdb_messaging
diff --git a/source4/cluster/ctdb/tests/nodes.txt b/source4/cluster/ctdb/tests/nodes.txt
new file mode 100644
index 0000000000..e1198b59ac
--- /dev/null
+++ b/source4/cluster/ctdb/tests/nodes.txt
@@ -0,0 +1,2 @@
+127.0.0.1:9001
+127.0.0.2:9001
diff --git a/source4/cluster/ctdb/tests/test.sh b/source4/cluster/ctdb/tests/test.sh
new file mode 100755
index 0000000000..6197ce9616
--- /dev/null
+++ b/source4/cluster/ctdb/tests/test.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+killall -q ctdb_test
+
+
+echo "Trying 2 nodes"
+bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.1:9001 &
+bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.2:9001 &
+
+sleep 3
+killall ctdb_test
+
+echo "Trying 4 nodes"
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.1:9001 &
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.2:9001 &
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.3:9001 &
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.4:9001 &
+sleep 3
+
+killall ctdb_test
+
+echo "Trying 2 nodes in daemon mode"
+bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.1:9001 --daemon &
+bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.2:9001 --daemon &
+
+sleep 3
+killall ctdb_test
+
+echo "Trying 4 nodes in daemon mode"
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.1:9001 --daemon &
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.2:9001 --daemon &
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.3:9001 --daemon &
+bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.4:9001 --daemon &
+sleep 3
+killall ctdb_test
diff --git a/source4/cluster/ctdb/tests/test1.sh b/source4/cluster/ctdb/tests/test1.sh
new file mode 100755
index 0000000000..64d5c9fc87
--- /dev/null
+++ b/source4/cluster/ctdb/tests/test1.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+echo "Testing local send"
+bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001 
+echo "Testing daemon mode"
+bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001 --daemon
+echo "Testing self connect"
+bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001 --self-connect