r23806: update Samba4 with the latest ctdb code.

This doesn't get the ctdb code fully working in Samba4, it just gets
it building and not breaking non-clustered use of Samba. It will take
a bit longer to update some of the calling ctdb_cluster.c code to make
it work correctly in Samba4.

Note also that Samba4 now only links to the client portion of
ctdb. For the moment I am leaving the ctdbd as a separate daemon,
which you install separately from http://ctdb.samba.org/.
(This used to be commit b196077cbb55cbecad87065133c2d67198e31066)

94 files changed, 19141 insertions, 4262 deletions

diff --git a/source4/cluster/ctdb/Makefile.in b/source4/cluster/ctdb/Makefile.in
index adc1f92d0a..2891a2248c 100644
--- a/source4/cluster/ctdb/Makefile.in
+++ b/source4/cluster/ctdb/Makefile.in
@@ -7,34 +7,58 @@ datarootdir = @datarootdir@
 includedir = @includedir@
 libdir = @libdir@
 bindir = @bindir@
-VPATH = @srcdir@:@tdbdir@:@tallocdir@:@libreplacedir@
+sbindir = @sbindir@
+mandir = @mandir@
+localstatedir = @localstatedir@
+VPATH = @srcdir@:@tdbdir@:@tallocdir@:@libreplacedir@:@poptdir@
 srcdir = @srcdir@
+etcdir = @sysconfdir@
 builddir = @builddir@
+DESTDIR = /
 EXTRA_OBJ=@EXTRA_OBJ@
+XSLTPROC = /usr/bin/xsltproc
+INSTALLCMD = @INSTALL@
 
-CFLAGS=-g -I$(srcdir)/include -Iinclude -Ilib/util -I$(srcdir) \
+POPT_LIBS = @POPT_LIBS@
+POPT_CFLAGS = @POPT_CFLAGS@
+POPT_OBJ = @POPT_OBJ@
+
+CFLAGS=-g -I$(srcdir)/include -Iinclude -Ilib -Ilib/util -I$(srcdir) \
        -I@tallocdir@ -I@tdbdir@/include -I@libreplacedir@ \
-	-DLIBDIR=\"$(libdir)\" -DSHLIBEXT=\"@SHLIBEXT@\" -DUSE_MMAP=1 @CFLAGS@
+	-DVARDIR=\"$(localstatedir)\" -DETCDIR=\"$(etcdir)\" \
+	-DUSE_MMAP=1 @CFLAGS@ $(POPT_CFLAGS)
 
-LIB_FLAGS=@LDFLAGS@ -Llib @LIBS@ -lpopt @INFINIBAND_LIBS@
+LIB_FLAGS=@LDFLAGS@ -Llib @LIBS@ $(POPT_LIBS) @INFINIBAND_LIBS@
 
-EVENTS_OBJ = lib/events/events.o lib/events/events_standard.o
+UTIL_OBJ = lib/util/idtree.o lib/util/db_wrap.o lib/util/strlist.o lib/util/util.o \
+	lib/util/util_time.o lib/util/util_file.o
 
-CTDB_COMMON_OBJ = common/ctdb.o common/ctdb_daemon.o common/ctdb_client.o common/ctdb_io.o common/util.o common/ctdb_util.o \
-	common/ctdb_call.o common/ctdb_ltdb.o common/ctdb_lockwait.o common/ctdb_message.o \
-	common/cmdline.o lib/util/idtree.o lib/util/db_wrap.o lib/util/debug.o
+CTDB_COMMON_OBJ =  common/ctdb_io.o common/ctdb_util.o \
+	common/ctdb_ltdb.o common/ctdb_message.o common/cmdline.o  \
+	lib/util/debug.o common/system.o
 
 CTDB_TCP_OBJ = tcp/tcp_connect.o tcp/tcp_io.o tcp/tcp_init.o
 
-CTDB_OBJ = $(CTDB_COMMON_OBJ) $(CTDB_TCP_OBJ)
+CTDB_CLIENT_OBJ = client/ctdb_client.o \
+	$(CTDB_COMMON_OBJ) $(POPT_OBJ) $(UTIL_OBJ) @TALLOC_OBJ@ @TDB_OBJ@ \
+	@LIBREPLACEOBJ@ $(EXTRA_OBJ) @EVENTS_OBJ@ 
 
-OBJS = @TDBOBJ@ @TALLOCOBJ@ @LIBREPLACEOBJ@ @INFINIBAND_WRAPPER_OBJ@ $(EXTRA_OBJ) $(EVENTS_OBJ) $(CTDB_OBJ)
+CTDB_SERVER_OBJ = server/ctdbd.o server/ctdb_daemon.o server/ctdb_lockwait.o \
+	server/ctdb_recoverd.o server/ctdb_recover.o server/ctdb_freeze.o \
+	server/ctdb_tunables.o server/ctdb_monitor.o server/ctdb_server.o \
+	server/ctdb_control.o server/ctdb_call.o server/ctdb_ltdb_server.o \
+	server/ctdb_traverse.o server/eventscript.o server/ctdb_takeover.o \
+	$(CTDB_CLIENT_OBJ) $(CTDB_TCP_OBJ) @INFINIBAND_WRAPPER_OBJ@
 
-BINS = bin/ctdbd bin/ctdbd_test bin/ctdb_test bin/ctdb_bench bin/ctdb_messaging bin/ctdb_fetch bin/ctdb_fetch1 bin/lockwait bin/ctdb_status @INFINIBAND_BINS@
+TEST_BINS=bin/ctdb_bench bin/ctdb_fetch bin/ctdb_store @INFINIBAND_BINS@
+BINS = bin/ctdb bin/scsi_io
+SBINS = bin/ctdbd
 
 DIRS = lib bin
 
-all: showflags dirs $(OBJS) $(BINS)
+.SUFFIXES: .c .o .h .1 .1.xml .1.html
+
+all: showflags dirs doc $(CTDB_SERVER_OBJ) $(CTDB_CLIENT_OBJ) $(BINS) $(SBINS) $(TEST_BINS)
 
 showflags:
 	@echo 'ctdb will be compiled with flags:'
@@ -49,49 +73,47 @@ showflags:
 dirs:
 	@mkdir -p $(DIRS)
 
-bin/ctdb_test: $(OBJS) tests/ctdb_test.o
+bin/ctdbd: $(CTDB_SERVER_OBJ)
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_test.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ $(CTDB_SERVER_OBJ) $(LIB_FLAGS)
 
-bin/ctdbd: $(OBJS) direct/ctdbd.o 
+bin/scsi_io: $(CTDB_CLIENT_OBJ) scsi/scsi_io.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ direct/ctdbd.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ scsi/scsi_io.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_status: $(OBJS) tools/ctdb_status.o 
+bin/ctdb: $(CTDB_CLIENT_OBJ) tools/ctdb.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tools/ctdb_status.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tools/ctdb.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdbd_test: $(OBJS) direct/ctdbd_test.o
+bin/ctdb_bench: $(CTDB_CLIENT_OBJ) tests/ctdb_bench.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ direct/ctdbd_test.o 
+	@$(CC) $(CFLAGS) -o $@ tests/ctdb_bench.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_bench: $(OBJS) tests/ctdb_bench.o 
+bin/ctdb_fetch: $(CTDB_CLIENT_OBJ) tests/ctdb_fetch.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_bench.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_fetch: $(OBJS) tests/ctdb_fetch.o 
+bin/ctdb_store: $(CTDB_CLIENT_OBJ) tests/ctdb_store.o 
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ tests/ctdb_store.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_fetch1: $(OBJS) tests/ctdb_fetch1.o 
+bin/ibwrapper_test: $(CTDB_CLIENT_OBJ) ib/ibwrapper_test.o
 	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_fetch1.o $(OBJS) $(LIB_FLAGS)
+	@$(CC) $(CFLAGS) -o $@ ib/ibwrapper_test.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS)
 
-bin/ctdb_messaging: $(OBJS) tests/ctdb_messaging.o 
-	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/ctdb_messaging.o $(OBJS) $(LIB_FLAGS)
+.1.xml.1:
+	-test -z "$(XSLTPROC)" || $(XSLTPROC) -o $@ http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $<
 
-bin/ibwrapper_test: $(OBJS) ib/ibwrapper_test.o
-	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ ib/ibwrapper_test.o $(OBJS) $(LIB_FLAGS)
+.1.xml.1.html:
+	-test -z "$(XSLTPROC)" || $(XSLTPROC) -o $@ http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl $<
 
-bin/lockwait: $(OBJS) tests/lockwait.o 
-	@echo Linking $@
-	@$(CC) $(CFLAGS) -o $@ tests/lockwait.o $(OBJS) $(LIB_FLAGS)
+doc: doc/ctdb.1 doc/ctdb.1.html \
+	doc/ctdbd.1 doc/ctdbd.1.html \
+	doc/onnode.1 doc/onnode.1.html
 
 clean:
-	rm -f *.o */*.o */*/*.o
-	rm -f $(BINS)
+	rm -f *.o */*.o */*/*.o */*~
+	rm -f $(BINS) $(SBINS) $(TEST_BINS)
 
 distclean: clean
 	rm -f *~ */*~
@@ -99,5 +121,38 @@ distclean: clean
 	rm -f config.log config.status config.cache config.h
 	rm -f Makefile
 
+install: all
+	mkdir -p $(DESTDIR)$(bindir)
+	mkdir -p $(DESTDIR)$(sbindir)
+	mkdir -p $(DESTDIR)$(includedir)
+	mkdir -p $(DESTDIR)$(etcdir)/ctdb
+	mkdir -p $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 bin/ctdb $(DESTDIR)$(bindir)
+	${INSTALLCMD} -m 755 bin/ctdbd $(DESTDIR)$(sbindir)
+	${INSTALLCMD} -m 644 include/ctdb.h $(DESTDIR)$(includedir)
+	${INSTALLCMD} -m 644 include/ctdb_private.h $(DESTDIR)$(includedir) # for samba3
+	${INSTALLCMD} -m 755 config/events $(DESTDIR)$(etcdir)/ctdb
+	${INSTALLCMD} -m 755 config/functions $(DESTDIR)$(etcdir)/ctdb
+	${INSTALLCMD} -m 755 config/statd-callout $(DESTDIR)$(etcdir)/ctdb
+	${INSTALLCMD} -m 755 config/events.d/10.interface $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 config/events.d/40.vsftpd $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 config/events.d/50.samba $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 config/events.d/60.nfs $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 config/events.d/61.nfstickle $(DESTDIR)$(etcdir)/ctdb/events.d
+	${INSTALLCMD} -m 755 tools/onnode.ssh $(DESTDIR)$(bindir)
+	${INSTALLCMD} -m 755 tools/onnode.rsh $(DESTDIR)$(bindir)
+	if [ -f doc/ctdb.1 ];then ${INSTALLCMD} -d $(DESTDIR)$(mandir)/man1; fi
+	if [ -f doc/ctdb.1 ];then ${INSTALLCMD} -m 644 doc/ctdb.1 $(DESTDIR)$(mandir)/man1; fi
+	if [ -f doc/ctdbd.1 ];then ${INSTALLCMD} -m 644 doc/ctdbd.1 $(DESTDIR)$(mandir)/man1; fi
+	if [ -f doc/onnode.1 ];then ${INSTALLCMD} -m 644 doc/onnode.1 $(DESTDIR)$(mandir)/man1; fi
+	cd $(DESTDIR)$(bindir) && ln -sf onnode.ssh onnode
+
+test: all
+	tests/run_tests.sh
+
+valgrindtest: all
+	VALGRIND="valgrind -q --trace-children=yes" tests/run_tests.sh
+
+
 realdistclean: distclean
 	rm -f configure config.h.in
diff --git a/source4/cluster/ctdb/README b/source4/cluster/ctdb/README
deleted file mode 100644
index 094d1d9958..0000000000
--- a/source4/cluster/ctdb/README
+++ /dev/null
@@ -1,3 +0,0 @@
-To build this you need a recent copy of talloc, libreplace and tdb in
-the directory above this directory.
-
diff --git a/source4/cluster/ctdb/aclocal.m4 b/source4/cluster/ctdb/aclocal.m4
new file mode 100644
index 0000000000..5605e476ba
--- /dev/null
+++ b/source4/cluster/ctdb/aclocal.m4
@@ -0,0 +1 @@
+m4_include(libreplace.m4)
diff --git a/source4/cluster/ctdb/client/ctdb_client.c b/source4/cluster/ctdb/client/ctdb_client.c
new file mode 100644
index 0000000000..b72faa83d7
--- /dev/null
+++ b/source4/cluster/ctdb/client/ctdb_client.c
@@ -0,0 +1,2140 @@
+/* 
+   ctdb daemon code
+
+   Copyright (C) Andrew Tridgell  2007
+   Copyright (C) Ronnie Sahlberg  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "db_wrap.h"
+#include "lib/tdb/include/tdb.h"
+#include "lib/util/dlinklist.h"
+#include "lib/events/events.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "../include/ctdb_private.h"
+#include "lib/util/dlinklist.h"
+
+/*
+  allocate a packet for use in client<->daemon communication
+ */
+struct ctdb_req_header *_ctdbd_allocate_pkt(struct ctdb_context *ctdb,
+					    TALLOC_CTX *mem_ctx, 
+					    enum ctdb_operation operation, 
+					    size_t length, size_t slength,
+					    const char *type)
+{
+	int size;
+	struct ctdb_req_header *hdr;
+
+	length = MAX(length, slength);
+	size = (length+(CTDB_DS_ALIGNMENT-1)) & ~(CTDB_DS_ALIGNMENT-1);
+
+	hdr = (struct ctdb_req_header *)talloc_size(mem_ctx, size);
+	if (hdr == NULL) {
+		DEBUG(0,("Unable to allocate packet for operation %u of length %u\n",
+			 operation, (unsigned)length));
+		return NULL;
+	}
+	talloc_set_name_const(hdr, type);
+	memset(hdr, 0, slength);
+	hdr->length       = length;
+	hdr->operation    = operation;
+	hdr->ctdb_magic   = CTDB_MAGIC;
+	hdr->ctdb_version = CTDB_VERSION;
+	hdr->srcnode      = ctdb->vnn;
+	if (ctdb->vnn_map) {
+		hdr->generation = ctdb->vnn_map->generation;
+	}
+
+	return hdr;
+}
+
+/*
+  local version of ctdb_call
+*/
+int ctdb_call_local(struct ctdb_db_context *ctdb_db, struct ctdb_call *call,
+		    struct ctdb_ltdb_header *header, TALLOC_CTX *mem_ctx,
+		    TDB_DATA *data, uint32_t caller)
+{
+	struct ctdb_call_info *c;
+	struct ctdb_registered_call *fn;
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	
+	c = talloc(ctdb, struct ctdb_call_info);
+	CTDB_NO_MEMORY(ctdb, c);
+
+	c->key = call->key;
+	c->call_data = &call->call_data;
+	c->record_data.dptr = talloc_memdup(c, data->dptr, data->dsize);
+	c->record_data.dsize = data->dsize;
+	CTDB_NO_MEMORY(ctdb, c->record_data.dptr);
+	c->new_data = NULL;
+	c->reply_data = NULL;
+	c->status = 0;
+
+	for (fn=ctdb_db->calls;fn;fn=fn->next) {
+		if (fn->id == call->call_id) break;
+	}
+	if (fn == NULL) {
+		ctdb_set_error(ctdb, "Unknown call id %u\n", call->call_id);
+		talloc_free(c);
+		return -1;
+	}
+
+	if (fn->fn(c) != 0) {
+		ctdb_set_error(ctdb, "ctdb_call %u failed\n", call->call_id);
+		talloc_free(c);
+		return -1;
+	}
+
+	if (header->laccessor != caller) {
+		header->lacount = 0;
+	}
+	header->laccessor = caller;
+	header->lacount++;
+
+	/* we need to force the record to be written out if this was a remote access,
+	   so that the lacount is updated */
+	if (c->new_data == NULL && header->laccessor != ctdb->vnn) {
+		c->new_data = &c->record_data;
+	}
+
+	if (c->new_data) {
+		/* XXX check that we always have the lock here? */
+		if (ctdb_ltdb_store(ctdb_db, call->key, header, *c->new_data) != 0) {
+			ctdb_set_error(ctdb, "ctdb_call tdb_store failed\n");
+			talloc_free(c);
+			return -1;
+		}
+	}
+
+	if (c->reply_data) {
+		call->reply_data = *c->reply_data;
+		talloc_steal(ctdb, call->reply_data.dptr);
+		talloc_set_name_const(call->reply_data.dptr, __location__);
+	} else {
+		call->reply_data.dptr = NULL;
+		call->reply_data.dsize = 0;
+	}
+	call->status = c->status;
+
+	talloc_free(c);
+
+	return 0;
+}
+
+
+/*
+  queue a packet for sending from client to daemon
+*/
+static int ctdb_client_queue_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	return ctdb_queue_send(ctdb->daemon.queue, (uint8_t *)hdr, hdr->length);
+}
+
+
+/*
+  state of a in-progress ctdb call in client
+*/
+struct ctdb_client_call_state {
+	enum call_state state;
+	uint32_t reqid;
+	struct ctdb_db_context *ctdb_db;
+	struct ctdb_call call;
+};
+
+/*
+  called when a CTDB_REPLY_CALL packet comes in in the client
+
+  This packet comes in response to a CTDB_REQ_CALL request packet. It
+  contains any reply data from the call
+*/
+static void ctdb_client_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	struct ctdb_reply_call *c = (struct ctdb_reply_call *)hdr;
+	struct ctdb_client_call_state *state;
+
+	state = ctdb_reqid_find(ctdb, hdr->reqid, struct ctdb_client_call_state);
+	if (state == NULL) {
+		DEBUG(0,(__location__ " reqid %u not found\n", hdr->reqid));
+		return;
+	}
+
+	if (hdr->reqid != state->reqid) {
+		/* we found a record  but it was the wrong one */
+		DEBUG(0, ("Dropped client call reply with reqid:%u\n",hdr->reqid));
+		return;
+	}
+
+	state->call.reply_data.dptr = c->data;
+	state->call.reply_data.dsize = c->datalen;
+	state->call.status = c->status;
+
+	talloc_steal(state, c);
+
+	state->state = CTDB_CALL_DONE;
+}
+
+static void ctdb_client_reply_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+
+/*
+  this is called in the client, when data comes in from the daemon
+ */
+static void ctdb_client_read_cb(uint8_t *data, size_t cnt, void *args)
+{
+	struct ctdb_context *ctdb = talloc_get_type(args, struct ctdb_context);
+	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
+	TALLOC_CTX *tmp_ctx;
+
+	/* place the packet as a child of a tmp_ctx. We then use
+	   talloc_free() below to free it. If any of the calls want
+	   to keep it, then they will steal it somewhere else, and the
+	   talloc_free() will be a no-op */
+	tmp_ctx = talloc_new(ctdb);
+	talloc_steal(tmp_ctx, hdr);
+
+	if (cnt == 0) {
+		DEBUG(2,("Daemon has exited - shutting down client\n"));
+		exit(0);
+	}
+
+	if (cnt < sizeof(*hdr)) {
+		DEBUG(0,("Bad packet length %u in client\n", (unsigned)cnt));
+		goto done;
+	}
+	if (cnt != hdr->length) {
+		ctdb_set_error(ctdb, "Bad header length %u expected %u in client\n", 
+			       (unsigned)hdr->length, (unsigned)cnt);
+		goto done;
+	}
+
+	if (hdr->ctdb_magic != CTDB_MAGIC) {
+		ctdb_set_error(ctdb, "Non CTDB packet rejected in client\n");
+		goto done;
+	}
+
+	if (hdr->ctdb_version != CTDB_VERSION) {
+		ctdb_set_error(ctdb, "Bad CTDB version 0x%x rejected in client\n", hdr->ctdb_version);
+		goto done;
+	}
+
+	switch (hdr->operation) {
+	case CTDB_REPLY_CALL:
+		ctdb_client_reply_call(ctdb, hdr);
+		break;
+
+	case CTDB_REQ_MESSAGE:
+		ctdb_request_message(ctdb, hdr);
+		break;
+
+	case CTDB_REPLY_CONTROL:
+		ctdb_client_reply_control(ctdb, hdr);
+		break;
+
+	default:
+		DEBUG(0,("bogus operation code:%u\n",hdr->operation));
+	}
+
+done:
+	talloc_free(tmp_ctx);
+}
+
+/*
+  connect to a unix domain socket
+*/
+int ctdb_socket_connect(struct ctdb_context *ctdb)
+{
+	struct sockaddr_un addr;
+
+	memset(&addr, 0, sizeof(addr));
+	addr.sun_family = AF_UNIX;
+	strncpy(addr.sun_path, ctdb->daemon.name, sizeof(addr.sun_path));
+
+	ctdb->daemon.sd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (ctdb->daemon.sd == -1) {
+		return -1;
+	}
+
+	set_nonblocking(ctdb->daemon.sd);
+	set_close_on_exec(ctdb->daemon.sd);
+	
+	if (connect(ctdb->daemon.sd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
+		close(ctdb->daemon.sd);
+		ctdb->daemon.sd = -1;
+		return -1;
+	}
+
+	ctdb->daemon.queue = ctdb_queue_setup(ctdb, ctdb, ctdb->daemon.sd, 
+					      CTDB_DS_ALIGNMENT, 
+					      ctdb_client_read_cb, ctdb);
+	return 0;
+}
+
+
+struct ctdb_record_handle {
+	struct ctdb_db_context *ctdb_db;
+	TDB_DATA key;
+	TDB_DATA *data;
+	struct ctdb_ltdb_header header;
+};
+
+
+/*
+  make a recv call to the local ctdb daemon - called from client context
+
+  This is called when the program wants to wait for a ctdb_call to complete and get the 
+  results. This call will block unless the call has already completed.
+*/
+int ctdb_call_recv(struct ctdb_client_call_state *state, struct ctdb_call *call)
+{
+	while (state->state < CTDB_CALL_DONE) {
+		event_loop_once(state->ctdb_db->ctdb->ev);
+	}
+	if (state->state != CTDB_CALL_DONE) {
+		DEBUG(0,(__location__ " ctdb_call_recv failed\n"));
+		talloc_free(state);
+		return -1;
+	}
+
+	if (state->call.reply_data.dsize) {
+		call->reply_data.dptr = talloc_memdup(state->ctdb_db,
+						      state->call.reply_data.dptr,
+						      state->call.reply_data.dsize);
+		call->reply_data.dsize = state->call.reply_data.dsize;
+	} else {
+		call->reply_data.dptr = NULL;
+		call->reply_data.dsize = 0;
+	}
+	call->status = state->call.status;
+	talloc_free(state);
+
+	return 0;
+}
+
+
+
+
+/*
+  destroy a ctdb_call in client
+*/
+static int ctdb_client_call_destructor(struct ctdb_client_call_state *state)	
+{
+	ctdb_reqid_remove(state->ctdb_db->ctdb, state->reqid);
+	return 0;
+}
+
+/*
+  construct an event driven local ctdb_call
+
+  this is used so that locally processed ctdb_call requests are processed
+  in an event driven manner
+*/
+static struct ctdb_client_call_state *ctdb_client_call_local_send(struct ctdb_db_context *ctdb_db, 
+								  struct ctdb_call *call,
+								  struct ctdb_ltdb_header *header,
+								  TDB_DATA *data)
+{
+	struct ctdb_client_call_state *state;
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	int ret;
+
+	state = talloc_zero(ctdb_db, struct ctdb_client_call_state);
+	CTDB_NO_MEMORY_NULL(ctdb, state);
+
+	talloc_steal(state, data->dptr);
+
+	state->state = CTDB_CALL_DONE;
+	state->call = *call;
+	state->ctdb_db = ctdb_db;
+
+	ret = ctdb_call_local(ctdb_db, &state->call, header, state, data, ctdb->vnn);
+
+	return state;
+}
+
+/*
+  make a ctdb call to the local daemon - async send. Called from client context.
+
+  This constructs a ctdb_call request and queues it for processing. 
+  This call never blocks.
+*/
+struct ctdb_client_call_state *ctdb_call_send(struct ctdb_db_context *ctdb_db, 
+				       struct ctdb_call *call)
+{
+	struct ctdb_client_call_state *state;
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	struct ctdb_ltdb_header header;
+	TDB_DATA data;
+	int ret;
+	size_t len;
+	struct ctdb_req_call *c;
+
+	/* if the domain socket is not yet open, open it */
+	if (ctdb->daemon.sd==-1) {
+		ctdb_socket_connect(ctdb);
+	}
+
+	ret = ctdb_ltdb_lock(ctdb_db, call->key);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to get chainlock\n"));
+		return NULL;
+	}
+
+	ret = ctdb_ltdb_fetch(ctdb_db, call->key, &header, ctdb_db, &data);
+
+	if (ret == 0 && header.dmaster == ctdb->vnn) {
+		state = ctdb_client_call_local_send(ctdb_db, call, &header, &data);
+		talloc_free(data.dptr);
+		ctdb_ltdb_unlock(ctdb_db, call->key);
+		return state;
+	}
+
+	ctdb_ltdb_unlock(ctdb_db, call->key);
+	talloc_free(data.dptr);
+
+	state = talloc_zero(ctdb_db, struct ctdb_client_call_state);
+	if (state == NULL) {
+		DEBUG(0, (__location__ " failed to allocate state\n"));
+		return NULL;
+	}
+
+	len = offsetof(struct ctdb_req_call, data) + call->key.dsize + call->call_data.dsize;
+	c = ctdbd_allocate_pkt(ctdb, state, CTDB_REQ_CALL, len, struct ctdb_req_call);
+	if (c == NULL) {
+		DEBUG(0, (__location__ " failed to allocate packet\n"));
+		return NULL;
+	}
+
+	state->reqid     = ctdb_reqid_new(ctdb, state);
+	state->ctdb_db = ctdb_db;
+	talloc_set_destructor(state, ctdb_client_call_destructor);
+
+	c->hdr.reqid     = state->reqid;
+	c->flags         = call->flags;
+	c->db_id         = ctdb_db->db_id;
+	c->callid        = call->call_id;
+	c->hopcount      = 0;
+	c->keylen        = call->key.dsize;
+	c->calldatalen   = call->call_data.dsize;
+	memcpy(&c->data[0], call->key.dptr, call->key.dsize);
+	memcpy(&c->data[call->key.dsize], 
+	       call->call_data.dptr, call->call_data.dsize);
+	state->call                = *call;
+	state->call.call_data.dptr = &c->data[call->key.dsize];
+	state->call.key.dptr       = &c->data[0];
+
+	state->state  = CTDB_CALL_WAIT;
+
+
+	ctdb_client_queue_pkt(ctdb, &c->hdr);
+
+	return state;
+}
+
+
+/*
+  full ctdb_call. Equivalent to a ctdb_call_send() followed by a ctdb_call_recv()
+*/
+int ctdb_call(struct ctdb_db_context *ctdb_db, struct ctdb_call *call)
+{
+	struct ctdb_client_call_state *state;
+
+	state = ctdb_call_send(ctdb_db, call);
+	return ctdb_call_recv(state, call);
+}
+
+
+/*
+  tell the daemon what messaging srvid we will use, and register the message
+  handler function in the client
+*/
+int ctdb_set_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			     ctdb_message_fn_t handler,
+			     void *private_data)
+				    
+{
+	int res;
+	int32_t status;
+	
+	res = ctdb_control(ctdb, CTDB_CURRENT_NODE, srvid, CTDB_CONTROL_REGISTER_SRVID, 0, 
+			   tdb_null, NULL, NULL, &status, NULL, NULL);
+	if (res != 0 || status != 0) {
+		DEBUG(0,("Failed to register srvid %llu\n", (unsigned long long)srvid));
+		return -1;
+	}
+
+	/* also need to register the handler with our own ctdb structure */
+	return ctdb_register_message_handler(ctdb, ctdb, srvid, handler, private_data);
+}
+
+/*
+  tell the daemon we no longer want a srvid
+*/
+int ctdb_remove_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data)
+{
+	int res;
+	int32_t status;
+	
+	res = ctdb_control(ctdb, CTDB_CURRENT_NODE, srvid, CTDB_CONTROL_DEREGISTER_SRVID, 0, 
+			   tdb_null, NULL, NULL, &status, NULL, NULL);
+	if (res != 0 || status != 0) {
+		DEBUG(0,("Failed to deregister srvid %llu\n", (unsigned long long)srvid));
+		return -1;
+	}
+
+	/* also need to register the handler with our own ctdb structure */
+	ctdb_deregister_message_handler(ctdb, srvid, private_data);
+	return 0;
+}
+
+
+/*
+  send a message - from client context
+ */
+int ctdb_send_message(struct ctdb_context *ctdb, uint32_t vnn,
+		      uint64_t srvid, TDB_DATA data)
+{
+	struct ctdb_req_message *r;
+	int len, res;
+
+	len = offsetof(struct ctdb_req_message, data) + data.dsize;
+	r = ctdbd_allocate_pkt(ctdb, ctdb, CTDB_REQ_MESSAGE, 
+			       len, struct ctdb_req_message);
+	CTDB_NO_MEMORY(ctdb, r);
+
+	r->hdr.destnode  = vnn;
+	r->srvid         = srvid;
+	r->datalen       = data.dsize;
+	memcpy(&r->data[0], data.dptr, data.dsize);
+	
+	res = ctdb_client_queue_pkt(ctdb, &r->hdr);
+	if (res != 0) {
+		return res;
+	}
+
+	talloc_free(r);
+	return 0;
+}
+
+
+/*
+  cancel a ctdb_fetch_lock operation, releasing the lock
+ */
+static int fetch_lock_destructor(struct ctdb_record_handle *h)
+{
+	ctdb_ltdb_unlock(h->ctdb_db, h->key);
+	return 0;
+}
+
+/*
+  force the migration of a record to this node
+ */
+static int ctdb_client_force_migration(struct ctdb_db_context *ctdb_db, TDB_DATA key)
+{
+	struct ctdb_call call;
+	ZERO_STRUCT(call);
+	call.call_id = CTDB_NULL_FUNC;
+	call.key = key;
+	call.flags = CTDB_IMMEDIATE_MIGRATION;
+	return ctdb_call(ctdb_db, &call);
+}
+
+/*
+  get a lock on a record, and return the records data. Blocks until it gets the lock
+ */
+struct ctdb_record_handle *ctdb_fetch_lock(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx, 
+					   TDB_DATA key, TDB_DATA *data)
+{
+	int ret;
+	struct ctdb_record_handle *h;
+
+	/*
+	  procedure is as follows:
+
+	  1) get the chain lock. 
+	  2) check if we are dmaster
+	  3) if we are the dmaster then return handle 
+	  4) if not dmaster then ask ctdb daemon to make us dmaster, and wait for
+	     reply from ctdbd
+	  5) when we get the reply, goto (1)
+	 */
+
+	h = talloc_zero(mem_ctx, struct ctdb_record_handle);
+	if (h == NULL) {
+		return NULL;
+	}
+
+	h->ctdb_db = ctdb_db;
+	h->key     = key;
+	h->key.dptr = talloc_memdup(h, key.dptr, key.dsize);
+	if (h->key.dptr == NULL) {
+		talloc_free(h);
+		return NULL;
+	}
+	h->data    = data;
+
+	DEBUG(3,("ctdb_fetch_lock: key=%*.*s\n", (int)key.dsize, (int)key.dsize, 
+		 (const char *)key.dptr));
+
+again:
+	/* step 1 - get the chain lock */
+	ret = ctdb_ltdb_lock(ctdb_db, key);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " failed to lock ltdb record\n"));
+		talloc_free(h);
+		return NULL;
+	}
+
+	DEBUG(4,("ctdb_fetch_lock: got chain lock\n"));
+
+	talloc_set_destructor(h, fetch_lock_destructor);
+
+	ret = ctdb_ltdb_fetch(ctdb_db, key, &h->header, h, data);
+
+	/* when torturing, ensure we test the remote path */
+	if ((ctdb_db->ctdb->flags & CTDB_FLAG_TORTURE) &&
+	    random() % 5 == 0) {
+		h->header.dmaster = (uint32_t)-1;
+	}
+
+
+	DEBUG(4,("ctdb_fetch_lock: done local fetch\n"));
+
+	if (ret != 0 || h->header.dmaster != ctdb_db->ctdb->vnn) {
+		ctdb_ltdb_unlock(ctdb_db, key);
+		ret = ctdb_client_force_migration(ctdb_db, key);
+		if (ret != 0) {
+			DEBUG(4,("ctdb_fetch_lock: force_migration failed\n"));
+			talloc_free(h);
+			return NULL;
+		}
+		goto again;
+	}
+
+	DEBUG(4,("ctdb_fetch_lock: we are dmaster - done\n"));
+	return h;
+}
+
+/*
+  store some data to the record that was locked with ctdb_fetch_lock()
+*/
+int ctdb_record_store(struct ctdb_record_handle *h, TDB_DATA data)
+{
+	return ctdb_ltdb_store(h->ctdb_db, h->key, &h->header, data);
+}
+
+struct ctdb_client_control_state {
+	struct ctdb_context *ctdb;
+	uint32_t reqid;
+	int32_t status;
+	TDB_DATA outdata;
+	enum call_state state;
+	char *errormsg;
+};
+
+/*
+  called when a CTDB_REPLY_CONTROL packet comes in in the client
+
+  This packet comes in response to a CTDB_REQ_CONTROL request packet. It
+  contains any reply data from the control
+*/
+static void ctdb_client_reply_control(struct ctdb_context *ctdb, 
+				      struct ctdb_req_header *hdr)
+{
+	struct ctdb_reply_control *c = (struct ctdb_reply_control *)hdr;
+	struct ctdb_client_control_state *state;
+
+	state = ctdb_reqid_find(ctdb, hdr->reqid, struct ctdb_client_control_state);
+	if (state == NULL) {
+		DEBUG(0,(__location__ " reqid %u not found\n", hdr->reqid));
+		return;
+	}
+
+	if (hdr->reqid != state->reqid) {
+		/* we found a record  but it was the wrong one */
+		DEBUG(0, ("Dropped orphaned reply control with reqid:%u\n",hdr->reqid));
+		return;
+	}
+
+	state->outdata.dptr = c->data;
+	state->outdata.dsize = c->datalen;
+	state->status = c->status;
+	if (c->errorlen) {
+		state->errormsg = talloc_strndup(state, 
+						 (char *)&c->data[c->datalen], 
+						 c->errorlen);
+	}
+
+	talloc_steal(state, c);
+
+	state->state = CTDB_CALL_DONE;
+}
+
+
+/* time out handler for ctdb_control */
+static void timeout_func(struct event_context *ev, struct timed_event *te, 
+	struct timeval t, void *private_data)
+{
+	uint32_t *timed_out = (uint32_t *)private_data;
+
+	*timed_out = 1;
+}
+
+/*
+  destroy a ctdb_control in client
+*/
+static int ctdb_control_destructor(struct ctdb_client_control_state *state)	
+{
+	ctdb_reqid_remove(state->ctdb, state->reqid);
+	return 0;
+}
+
+/*
+  send a ctdb control message
+  timeout specifies how long we should wait for a reply.
+  if timeout is NULL we wait indefinitely
+ */
+int ctdb_control(struct ctdb_context *ctdb, uint32_t destnode, uint64_t srvid, 
+		 uint32_t opcode, uint32_t flags, TDB_DATA data, 
+		 TALLOC_CTX *mem_ctx, TDB_DATA *outdata, int32_t *status,
+		 struct timeval *timeout,
+		 char **errormsg)
+{
+	struct ctdb_client_control_state *state;
+	struct ctdb_req_control *c;
+	size_t len;
+	int ret;
+	uint32_t timed_out;
+
+	if (errormsg) {
+		*errormsg = NULL;
+	}
+
+	/* if the domain socket is not yet open, open it */
+	if (ctdb->daemon.sd==-1) {
+		ctdb_socket_connect(ctdb);
+	}
+
+	state = talloc_zero(ctdb, struct ctdb_client_control_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	state->ctdb  = ctdb;
+	state->reqid = ctdb_reqid_new(ctdb, state);
+	state->state = CTDB_CALL_WAIT;
+	state->errormsg = NULL;
+
+	talloc_set_destructor(state, ctdb_control_destructor);
+
+	len = offsetof(struct ctdb_req_control, data) + data.dsize;
+	c = ctdbd_allocate_pkt(ctdb, state, CTDB_REQ_CONTROL, 
+			       len, struct ctdb_req_control);
+	CTDB_NO_MEMORY(ctdb, c);
+	
+	c->hdr.reqid        = state->reqid;
+	c->hdr.destnode     = destnode;
+	c->hdr.reqid        = state->reqid;
+	c->opcode           = opcode;
+	c->client_id        = 0;
+	c->flags            = flags;
+	c->srvid            = srvid;
+	c->datalen          = data.dsize;
+	if (data.dsize) {
+		memcpy(&c->data[0], data.dptr, data.dsize);
+	}
+
+	ret = ctdb_client_queue_pkt(ctdb, &(c->hdr));
+	if (ret != 0) {
+		talloc_free(state);
+		return -1;
+	}
+
+	if (flags & CTDB_CTRL_FLAG_NOREPLY) {
+		talloc_free(state);
+		return 0;
+	}
+
+	/* semi-async operation */
+	timed_out = 0;
+	if (timeout && !timeval_is_zero(timeout)) {
+		event_add_timed(ctdb->ev, state, *timeout, timeout_func, &timed_out);
+	}
+	while ((state->state == CTDB_CALL_WAIT)
+	&&	(timed_out == 0) ){
+		event_loop_once(ctdb->ev);
+	}
+	if (timed_out) {
+		talloc_free(state);
+		if (errormsg) {
+			(*errormsg) = talloc_strdup(mem_ctx, "control timed out");
+		} else {
+			DEBUG(0,("ctdb_control timed out\n"));
+		}
+		return -1;
+	}
+
+	if (outdata) {
+		*outdata = state->outdata;
+		outdata->dptr = talloc_memdup(mem_ctx, outdata->dptr, outdata->dsize);
+	}
+
+	*status = state->status;
+
+	if (!errormsg && state->errormsg) {
+		DEBUG(0,("ctdb_control error: '%s'\n", state->errormsg));
+	}
+
+	if (errormsg && state->errormsg) {
+		(*errormsg) = talloc_move(mem_ctx, &state->errormsg);
+	}
+
+	talloc_free(state);
+
+	return 0;	
+}
+
+
+
+/*
+  a process exists call. Returns 0 if process exists, -1 otherwise
+ */
+int ctdb_ctrl_process_exists(struct ctdb_context *ctdb, uint32_t destnode, pid_t pid)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t status;
+
+	data.dptr = (uint8_t*)&pid;
+	data.dsize = sizeof(pid);
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_PROCESS_EXISTS, 0, data, 
+			   NULL, NULL, &status, NULL, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for process_exists failed\n"));
+		return -1;
+	}
+
+	return status;
+}
+
+/*
+  get remote statistics
+ */
+int ctdb_ctrl_statistics(struct ctdb_context *ctdb, uint32_t destnode, struct ctdb_statistics *status)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_STATISTICS, 0, tdb_null, 
+			   ctdb, &data, &res, NULL, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for statistics failed\n"));
+		return -1;
+	}
+
+	if (data.dsize != sizeof(struct ctdb_statistics)) {
+		DEBUG(0,(__location__ " Wrong statistics size %u - expected %u\n",
+			 (unsigned)data.dsize, (unsigned)sizeof(struct ctdb_statistics)));
+		      return -1;
+	}
+
+	*status = *(struct ctdb_statistics *)data.dptr;
+	talloc_free(data.dptr);
+			
+	return 0;
+}
+
+/*
+  shutdown a remote ctdb node
+ */
+int ctdb_ctrl_shutdown(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_SHUTDOWN, CTDB_CTRL_FLAG_NOREPLY, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for shutdown failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  get vnn map from a remote node
+ */
+int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap)
+{
+	int ret;
+	TDB_DATA outdata;
+	int32_t res;
+	struct ctdb_vnn_map_wire *map;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GETVNNMAP, 0, tdb_null, 
+			   mem_ctx, &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getvnnmap failed\n"));
+		return -1;
+	}
+	
+	map = (struct ctdb_vnn_map_wire *)outdata.dptr;
+	if (outdata.dsize < offsetof(struct ctdb_vnn_map_wire, map) ||
+	    outdata.dsize != map->size*sizeof(uint32_t) + offsetof(struct ctdb_vnn_map_wire, map)) {
+		DEBUG(0,("Bad vnn map size received in ctdb_ctrl_getvnnmap\n"));
+		return -1;
+	}
+
+	(*vnnmap) = talloc(mem_ctx, struct ctdb_vnn_map);
+	CTDB_NO_MEMORY(ctdb, *vnnmap);
+	(*vnnmap)->generation = map->generation;
+	(*vnnmap)->size       = map->size;
+	(*vnnmap)->map        = talloc_array(*vnnmap, uint32_t, map->size);
+
+	CTDB_NO_MEMORY(ctdb, (*vnnmap)->map);
+	memcpy((*vnnmap)->map, map->map, sizeof(uint32_t)*map->size);
+	talloc_free(outdata.dptr);
+		    
+	return 0;
+}
+
+/*
+  get the recovery mode of a remote node
+ */
+int ctdb_ctrl_getrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *recmode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_RECMODE, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getrecmode failed\n"));
+		return -1;
+	}
+
+	*recmode = res;
+
+	return 0;
+}
+
+/*
+  set the recovery mode of a remote node
+ */
+int ctdb_ctrl_setrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmode)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t res;
+
+	data.dsize = sizeof(uint32_t);
+	data.dptr = (unsigned char *)&recmode;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_SET_RECMODE, 0, data, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for setrecmode failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  get the recovery master of a remote node
+ */
+int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *recmaster)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_RECMASTER, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getrecmaster failed\n"));
+		return -1;
+	}
+
+	*recmaster = res;
+
+	return 0;
+}
+
+/*
+  set the recovery master of a remote node
+ */
+int ctdb_ctrl_setrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmaster)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t res;
+
+	ZERO_STRUCT(data);
+	data.dsize = sizeof(uint32_t);
+	data.dptr = (unsigned char *)&recmaster;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_SET_RECMASTER, 0, data, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for setrecmaster failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/*
+  get a list of databases off a remote node
+ */
+int ctdb_ctrl_getdbmap(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, 
+		       TALLOC_CTX *mem_ctx, struct ctdb_dbid_map **dbmap)
+{
+	int ret;
+	TDB_DATA outdata;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_DBMAP, 0, tdb_null, 
+			   mem_ctx, &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getdbmap failed\n"));
+		return -1;
+	}
+
+	*dbmap = (struct ctdb_dbid_map *)talloc_memdup(mem_ctx, outdata.dptr, outdata.dsize);
+	talloc_free(outdata.dptr);
+		    
+	return 0;
+}
+
+
+/*
+  get a list of nodes (vnn and flags ) from a remote node
+ */
+int ctdb_ctrl_getnodemap(struct ctdb_context *ctdb, 
+		struct timeval timeout, uint32_t destnode, 
+		TALLOC_CTX *mem_ctx, struct ctdb_node_map **nodemap)
+{
+	int ret;
+	TDB_DATA outdata;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_NODEMAP, 0, tdb_null, 
+			   mem_ctx, &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getnodes failed\n"));
+		return -1;
+	}
+
+	*nodemap = (struct ctdb_node_map *)talloc_memdup(mem_ctx, outdata.dptr, outdata.dsize);
+	talloc_free(outdata.dptr);
+		    
+	return 0;
+}
+
+/*
+  set vnn map on a node
+ */
+int ctdb_ctrl_setvnnmap(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, 
+			TALLOC_CTX *mem_ctx, struct ctdb_vnn_map *vnnmap)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t res;
+	struct ctdb_vnn_map_wire *map;
+	size_t len;
+
+	len = offsetof(struct ctdb_vnn_map_wire, map) + sizeof(uint32_t)*vnnmap->size;
+	map = talloc_size(mem_ctx, len);
+	CTDB_NO_MEMORY_VOID(ctdb, map);
+
+	map->generation = vnnmap->generation;
+	map->size = vnnmap->size;
+	memcpy(map->map, vnnmap->map, sizeof(uint32_t)*map->size);
+	
+	data.dsize = len;
+	data.dptr  = (uint8_t *)map;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_SETVNNMAP, 0, data, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for setvnnmap failed\n"));
+		return -1;
+	}
+
+	talloc_free(map);
+
+	return 0;
+}
+
+/*
+  get all keys and records for a specific database
+ */
+int ctdb_ctrl_pulldb(struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid, uint32_t lmaster, 
+		     TALLOC_CTX *mem_ctx, struct ctdb_key_list *keys)
+{
+	int i, ret;
+	TDB_DATA indata, outdata;
+	struct ctdb_control_pulldb pull;
+	struct ctdb_control_pulldb_reply *reply;
+	struct ctdb_rec_data *rec;
+	int32_t res;
+
+	pull.db_id   = dbid;
+	pull.lmaster = lmaster;
+
+	indata.dsize = sizeof(struct ctdb_control_pulldb);
+	indata.dptr  = (unsigned char *)&pull;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_PULL_DB, 0, indata, 
+			   mem_ctx, &outdata, &res, NULL, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for pulldb failed\n"));
+		return -1;
+	}
+
+
+	reply = (struct ctdb_control_pulldb_reply *)outdata.dptr;
+	keys->dbid     = reply->db_id;
+	keys->num      = reply->count;
+	
+	keys->keys     = talloc_array(mem_ctx, TDB_DATA, keys->num);
+	keys->headers  = talloc_array(mem_ctx, struct ctdb_ltdb_header, keys->num);
+	keys->data     = talloc_array(mem_ctx, TDB_DATA, keys->num);
+
+	rec = (struct ctdb_rec_data *)&reply->data[0];
+
+	for (i=0;i<reply->count;i++) {
+		keys->keys[i].dptr = talloc_memdup(mem_ctx, &rec->data[0], rec->keylen);
+		keys->keys[i].dsize = rec->keylen;
+		
+		keys->data[i].dptr = talloc_memdup(mem_ctx, &rec->data[keys->keys[i].dsize], rec->datalen);
+		keys->data[i].dsize = rec->datalen;
+
+		if (keys->data[i].dsize < sizeof(struct ctdb_ltdb_header)) {
+			DEBUG(0,(__location__ " bad ltdb record\n"));
+			return -1;
+		}
+		memcpy(&keys->headers[i], keys->data[i].dptr, sizeof(struct ctdb_ltdb_header));
+		keys->data[i].dptr += sizeof(struct ctdb_ltdb_header);
+		keys->data[i].dsize -= sizeof(struct ctdb_ltdb_header);
+
+		rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec);
+	}	    
+
+	talloc_free(outdata.dptr);
+
+	return 0;
+}
+
+/*
+  copy a tdb from one node to another node
+ */
+int ctdb_ctrl_copydb(struct ctdb_context *ctdb, struct timeval timeout, uint32_t sourcenode, 
+		     uint32_t destnode, uint32_t dbid, uint32_t lmaster, TALLOC_CTX *mem_ctx)
+{
+	int ret;
+	TDB_DATA indata, outdata;
+	int32_t res;
+
+	indata.dsize = 2*sizeof(uint32_t);
+	indata.dptr  = (unsigned char *)talloc_array(mem_ctx, uint32_t, 2);
+
+	((uint32_t *)(&indata.dptr[0]))[0] = dbid;
+	((uint32_t *)(&indata.dptr[0]))[1] = lmaster;
+
+	DEBUG(3,("pulling dbid 0x%x from %u\n", dbid, sourcenode));
+
+	ret = ctdb_control(ctdb, sourcenode, 0, 
+			   CTDB_CONTROL_PULL_DB, 0, indata, 
+			   mem_ctx, &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for pulldb failed\n"));
+		return -1;
+	}
+
+	DEBUG(3,("pushing dbid 0x%x to %u\n", dbid, destnode));
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_PUSH_DB, 0, outdata, 
+			   mem_ctx, NULL, &res, &timeout, NULL);
+	talloc_free(outdata.dptr);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for pushdb failed\n"));
+		return -1;
+	}
+
+	DEBUG(3,("copydb for dbid 0x%x done for %u to %u\n", 
+		 dbid, sourcenode, destnode));
+
+	return 0;
+}
+
+/*
+  change dmaster for all keys in the database to the new value
+ */
+int ctdb_ctrl_setdmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, 
+			 TALLOC_CTX *mem_ctx, uint32_t dbid, uint32_t dmaster)
+{
+	int ret;
+	TDB_DATA indata;
+	int32_t res;
+
+	indata.dsize = 2*sizeof(uint32_t);
+	indata.dptr = (unsigned char *)talloc_array(mem_ctx, uint32_t, 2);
+
+	((uint32_t *)(&indata.dptr[0]))[0] = dbid;
+	((uint32_t *)(&indata.dptr[0]))[1] = dmaster;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_SET_DMASTER, 0, indata, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for setdmaster failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  ping a node, return number of clients connected
+ */
+int ctdb_ctrl_ping(struct ctdb_context *ctdb, uint32_t destnode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_PING, 0, 
+			   tdb_null, NULL, NULL, &res, NULL, NULL);
+	if (ret != 0) {
+		return -1;
+	}
+	return res;
+}
+
+/*
+  find the real path to a ltdb 
+ */
+int ctdb_ctrl_getdbpath(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t dbid, TALLOC_CTX *mem_ctx, 
+		   const char **path)
+{
+	int ret;
+	int32_t res;
+	TDB_DATA data;
+
+	data.dptr = (uint8_t *)&dbid;
+	data.dsize = sizeof(dbid);
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GETDBPATH, 0, data, 
+			   mem_ctx, &data, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		return -1;
+	}
+
+	(*path) = talloc_strndup(mem_ctx, (const char *)data.dptr, data.dsize);
+	if ((*path) == NULL) {
+		return -1;
+	}
+
+	talloc_free(data.dptr);
+
+	return 0;
+}
+
+/*
+  find the name of a db 
+ */
+int ctdb_ctrl_getdbname(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t dbid, TALLOC_CTX *mem_ctx, 
+		   const char **name)
+{
+	int ret;
+	int32_t res;
+	TDB_DATA data;
+
+	data.dptr = (uint8_t *)&dbid;
+	data.dsize = sizeof(dbid);
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_DBNAME, 0, data, 
+			   mem_ctx, &data, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		return -1;
+	}
+
+	(*name) = talloc_strndup(mem_ctx, (const char *)data.dptr, data.dsize);
+	if ((*name) == NULL) {
+		return -1;
+	}
+
+	talloc_free(data.dptr);
+
+	return 0;
+}
+
+/*
+  create a database
+ */
+int ctdb_ctrl_createdb(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, const char *name)
+{
+	int ret;
+	int32_t res;
+	TDB_DATA data;
+
+	data.dptr = discard_const(name);
+	data.dsize = strlen(name)+1;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_DB_ATTACH, 0, data, 
+			   mem_ctx, &data, &res, &timeout, NULL);
+
+	if (ret != 0 || res != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  get debug level on a node
+ */
+int ctdb_ctrl_get_debuglevel(struct ctdb_context *ctdb, uint32_t destnode, uint32_t *level)
+{
+	int ret;
+	int32_t res;
+	TDB_DATA data;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_GET_DEBUG, 0, tdb_null, 
+			   ctdb, &data, &res, NULL, NULL);
+	if (ret != 0 || res != 0) {
+		return -1;
+	}
+	if (data.dsize != sizeof(uint32_t)) {
+		DEBUG(0,("Bad control reply size in ctdb_get_debuglevel (got %u)\n",
+			 (unsigned)data.dsize));
+		return -1;
+	}
+	*level = *(uint32_t *)data.dptr;
+	talloc_free(data.dptr);
+	return 0;
+}
+
+/*
+  set debug level on a node
+ */
+int ctdb_ctrl_set_debuglevel(struct ctdb_context *ctdb, uint32_t destnode, uint32_t level)
+{
+	int ret;
+	int32_t res;
+	TDB_DATA data;
+
+	data.dptr = (uint8_t *)&level;
+	data.dsize = sizeof(level);
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_SET_DEBUG, 0, data, 
+			   NULL, NULL, &res, NULL, NULL);
+	if (ret != 0 || res != 0) {
+		return -1;
+	}
+	return 0;
+}
+
+
+/*
+  get a list of connected nodes
+ */
+uint32_t *ctdb_get_connected_nodes(struct ctdb_context *ctdb, 
+				struct timeval timeout,
+				TALLOC_CTX *mem_ctx,
+				uint32_t *num_nodes)
+{
+	struct ctdb_node_map *map=NULL;
+	int ret, i;
+	uint32_t *nodes;
+
+	*num_nodes = 0;
+
+	ret = ctdb_ctrl_getnodemap(ctdb, timeout, CTDB_CURRENT_NODE, mem_ctx, &map);
+	if (ret != 0) {
+		return NULL;
+	}
+
+	nodes = talloc_array(mem_ctx, uint32_t, map->num);
+	if (nodes == NULL) {
+		return NULL;
+	}
+
+	for (i=0;i<map->num;i++) {
+		if (!(map->nodes[i].flags & NODE_FLAGS_DISCONNECTED)) {
+			nodes[*num_nodes] = map->nodes[i].vnn;
+			(*num_nodes)++;
+		}
+	}
+
+	return nodes;
+}
+
+
+/*
+  reset remote status
+ */
+int ctdb_statistics_reset(struct ctdb_context *ctdb, uint32_t destnode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_STATISTICS_RESET, 0, tdb_null, 
+			   NULL, NULL, &res, NULL, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for reset statistics failed\n"));
+		return -1;
+	}
+	return 0;
+}
+
+
+/*
+  attach to a specific database - client call
+*/
+struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name)
+{
+	struct ctdb_db_context *ctdb_db;
+	TDB_DATA data;
+	int ret;
+	int32_t res;
+
+	ctdb_db = talloc_zero(ctdb, struct ctdb_db_context);
+	CTDB_NO_MEMORY_NULL(ctdb, ctdb_db);
+
+	ctdb_db->ctdb = ctdb;
+	ctdb_db->db_name = talloc_strdup(ctdb_db, name);
+	CTDB_NO_MEMORY_NULL(ctdb, ctdb_db->db_name);
+
+	data.dptr = discard_const(name);
+	data.dsize = strlen(name)+1;
+
+	/* tell ctdb daemon to attach */
+	ret = ctdb_control(ctdb, CTDB_CURRENT_NODE, 0, CTDB_CONTROL_DB_ATTACH,
+			   0, data, ctdb_db, &data, &res, NULL, NULL);
+	if (ret != 0 || res != 0 || data.dsize != sizeof(uint32_t)) {
+		DEBUG(0,("Failed to attach to database '%s'\n", name));
+		talloc_free(ctdb_db);
+		return NULL;
+	}
+	
+	ctdb_db->db_id = *(uint32_t *)data.dptr;
+	talloc_free(data.dptr);
+
+	ret = ctdb_ctrl_getdbpath(ctdb, timeval_current_ofs(2, 0), CTDB_CURRENT_NODE, ctdb_db->db_id, ctdb_db, &ctdb_db->db_path);
+	if (ret != 0) {
+		DEBUG(0,("Failed to get dbpath for database '%s'\n", name));
+		talloc_free(ctdb_db);
+		return NULL;
+	}
+
+	ctdb_db->ltdb = tdb_wrap_open(ctdb, ctdb_db->db_path, 0, 0, O_RDWR, 0);
+	if (ctdb_db->ltdb == NULL) {
+		ctdb_set_error(ctdb, "Failed to open tdb '%s'\n", ctdb_db->db_path);
+		talloc_free(ctdb_db);
+		return NULL;
+	}
+
+	DLIST_ADD(ctdb->db_list, ctdb_db);
+
+	return ctdb_db;
+}
+
+
+/*
+  setup a call for a database
+ */
+int ctdb_set_call(struct ctdb_db_context *ctdb_db, ctdb_fn_t fn, uint32_t id)
+{
+	TDB_DATA data;
+	int32_t status;
+	struct ctdb_control_set_call c;
+	int ret;
+	struct ctdb_registered_call *call;
+
+	c.db_id = ctdb_db->db_id;
+	c.fn    = fn;
+	c.id    = id;
+
+	data.dptr = (uint8_t *)&c;
+	data.dsize = sizeof(c);
+
+	ret = ctdb_control(ctdb_db->ctdb, CTDB_CURRENT_NODE, 0, CTDB_CONTROL_SET_CALL, 0,
+			   data, NULL, NULL, &status, NULL, NULL);
+	if (ret != 0 || status != 0) {
+		DEBUG(0,("ctdb_set_call failed for call %u\n", id));
+		return -1;
+	}
+
+	/* also register locally */
+	call = talloc(ctdb_db, struct ctdb_registered_call);
+	call->fn = fn;
+	call->id = id;
+
+	DLIST_ADD(ctdb_db->calls, call);	
+	return 0;
+}
+
+
+struct traverse_state {
+	bool done;
+	uint32_t count;
+	ctdb_traverse_func fn;
+	void *private_data;
+};
+
+/*
+  called on each key during a ctdb_traverse
+ */
+static void traverse_handler(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA data, void *p)
+{
+	struct traverse_state *state = (struct traverse_state *)p;
+	struct ctdb_rec_data *d = (struct ctdb_rec_data *)data.dptr;
+	TDB_DATA key;
+
+	if (data.dsize < sizeof(uint32_t) ||
+	    d->length != data.dsize) {
+		DEBUG(0,("Bad data size %u in traverse_handler\n", (unsigned)data.dsize));
+		state->done = True;
+		return;
+	}
+
+	key.dsize = d->keylen;
+	key.dptr  = &d->data[0];
+	data.dsize = d->datalen;
+	data.dptr = &d->data[d->keylen];
+
+	if (key.dsize == 0 && data.dsize == 0) {
+		/* end of traverse */
+		state->done = True;
+		return;
+	}
+
+	if (state->fn(ctdb, key, data, state->private_data) != 0) {
+		state->done = True;
+	}
+
+	state->count++;
+}
+
+
+/*
+  start a cluster wide traverse, calling the supplied fn on each record
+  return the number of records traversed, or -1 on error
+ */
+int ctdb_traverse(struct ctdb_db_context *ctdb_db, ctdb_traverse_func fn, void *private_data)
+{
+	TDB_DATA data;
+	struct ctdb_traverse_start t;
+	int32_t status;
+	int ret;
+	uint64_t srvid = (getpid() | 0xFLL<<60);
+	struct traverse_state state;
+
+	state.done = False;
+	state.count = 0;
+	state.private_data = private_data;
+	state.fn = fn;
+
+	ret = ctdb_set_message_handler(ctdb_db->ctdb, srvid, traverse_handler, &state);
+	if (ret != 0) {
+		DEBUG(0,("Failed to setup traverse handler\n"));
+		return -1;
+	}
+
+	t.db_id = ctdb_db->db_id;
+	t.srvid = srvid;
+	t.reqid = 0;
+
+	data.dptr = (uint8_t *)&t;
+	data.dsize = sizeof(t);
+
+	ret = ctdb_control(ctdb_db->ctdb, CTDB_CURRENT_NODE, 0, CTDB_CONTROL_TRAVERSE_START, 0,
+			   data, NULL, NULL, &status, NULL, NULL);
+	if (ret != 0 || status != 0) {
+		DEBUG(0,("ctdb_traverse_all failed\n"));
+		ctdb_remove_message_handler(ctdb_db->ctdb, srvid, &state);
+		return -1;
+	}
+
+	while (!state.done) {
+		event_loop_once(ctdb_db->ctdb->ev);
+	}
+
+	ret = ctdb_remove_message_handler(ctdb_db->ctdb, srvid, &state);
+	if (ret != 0) {
+		DEBUG(0,("Failed to remove ctdb_traverse handler\n"));
+		return -1;
+	}
+
+	return state.count;
+}
+
+/*
+  called on each key during a catdb
+ */
+static int dumpdb_fn(struct ctdb_context *ctdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	FILE *f = (FILE *)p;
+	char *keystr, *datastr;
+	struct ctdb_ltdb_header *h = (struct ctdb_ltdb_header *)data.dptr;
+
+	keystr  = hex_encode_talloc(ctdb, key.dptr, key.dsize);
+	datastr = hex_encode_talloc(ctdb, data.dptr+sizeof(*h), data.dsize-sizeof(*h));
+
+	fprintf(f, "dmaster: %u\n", h->dmaster);
+	fprintf(f, "rsn: %llu\n", (unsigned long long)h->rsn);
+	fprintf(f, "key: %s\ndata: %s\n", keystr, datastr);
+
+	talloc_free(keystr);
+	talloc_free(datastr);
+	return 0;
+}
+
+/*
+  convenience function to list all keys to stdout
+ */
+int ctdb_dump_db(struct ctdb_db_context *ctdb_db, FILE *f)
+{
+	return ctdb_traverse(ctdb_db, dumpdb_fn, f);
+}
+
+/*
+  get the pid of a ctdb daemon
+ */
+int ctdb_ctrl_getpid(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *pid)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_PID, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getpid failed\n"));
+		return -1;
+	}
+
+	*pid = res;
+
+	return 0;
+}
+
+
+/*
+  freeze a node
+ */
+int ctdb_ctrl_freeze(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_FREEZE, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control freeze failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  thaw a node
+ */
+int ctdb_ctrl_thaw(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_THAW, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control thaw failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  get vnn of a node, or -1
+ */
+int ctdb_ctrl_getvnn(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_VNN, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getvnn failed\n"));
+		return -1;
+	}
+
+	return res;
+}
+
+/*
+  set the monitoring mode of a remote node
+ */
+int ctdb_ctrl_setmonmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t monmode)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t res;
+
+	data.dsize = sizeof(uint32_t);
+	data.dptr = (uint8_t *)&monmode;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_SET_MONMODE, 0, data, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for setmonmode failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  get the monitoring mode of a remote node
+ */
+int ctdb_ctrl_getmonmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *monmode)
+{
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_MONMODE, 0, tdb_null, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getrecmode failed\n"));
+		return -1;
+	}
+
+	*monmode = res;
+
+	return 0;
+}
+
+
+/*
+  get maximum rsn for a db on a node
+ */
+int ctdb_ctrl_get_max_rsn(struct ctdb_context *ctdb, struct timeval timeout, 
+			  uint32_t destnode, uint32_t db_id, uint64_t *max_rsn)
+{
+	TDB_DATA data, outdata;
+	int ret;
+	int32_t res;
+
+	data.dptr = (uint8_t *)&db_id;
+	data.dsize = sizeof(db_id);
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_MAX_RSN, 0, data, ctdb,
+			   &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0 || outdata.dsize != sizeof(uint64_t)) {
+		DEBUG(0,(__location__ " ctdb_control for get_max_rsn failed\n"));
+		return -1;
+	}
+
+	*max_rsn = *(uint64_t *)outdata.dptr;
+	talloc_free(outdata.dptr);
+
+	return 0;	
+}
+
+/*
+  set the rsn on non-empty records to the given rsn
+ */
+int ctdb_ctrl_set_rsn_nonempty(struct ctdb_context *ctdb, struct timeval timeout, 
+			       uint32_t destnode, uint32_t db_id, uint64_t rsn)
+{
+	TDB_DATA data;
+	int ret;
+	int32_t res;
+	struct ctdb_control_set_rsn_nonempty p;
+
+	p.db_id = db_id;
+	p.rsn = rsn;
+
+	data.dptr = (uint8_t *)&p;
+	data.dsize = sizeof(p);
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_SET_RSN_NONEMPTY, 0, data, NULL,
+			   NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for set_rsn_nonempty failed\n"));
+		return -1;
+	}
+
+	return 0;	
+}
+
+/*
+  delete records which have a rsn below the given rsn
+ */
+int ctdb_ctrl_delete_low_rsn(struct ctdb_context *ctdb, struct timeval timeout, 
+			     uint32_t destnode, uint32_t db_id, uint64_t rsn)
+{
+	TDB_DATA data;
+	int ret;
+	int32_t res;
+	struct ctdb_control_delete_low_rsn p;
+
+	p.db_id = db_id;
+	p.rsn = rsn;
+
+	data.dptr = (uint8_t *)&p;
+	data.dsize = sizeof(p);
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_DELETE_LOW_RSN, 0, data, NULL,
+			   NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for delete_low_rsn failed\n"));
+		return -1;
+	}
+
+	return 0;	
+}
+
+/* 
+  sent to a node to make it take over an ip address
+*/
+int ctdb_ctrl_takeover_ip(struct ctdb_context *ctdb, struct timeval timeout, 
+			  uint32_t destnode, struct ctdb_public_ip *ip)
+{
+	TDB_DATA data;
+	int ret;
+	int32_t res;
+
+	data.dsize = sizeof(*ip);
+	data.dptr  = (uint8_t *)ip;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_TAKEOVER_IP, 0, data, NULL,
+			   NULL, &res, &timeout, NULL);
+
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for takeover_ip failed\n"));
+		return -1;
+	}
+
+	return 0;	
+}
+
+
+/* 
+  sent to a node to make it release an ip address
+*/
+int ctdb_ctrl_release_ip(struct ctdb_context *ctdb, struct timeval timeout, 
+			 uint32_t destnode, struct ctdb_public_ip *ip)
+{
+	TDB_DATA data;
+	int ret;
+	int32_t res;
+
+	data.dsize = sizeof(*ip);
+	data.dptr  = (uint8_t *)ip;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_RELEASE_IP, 0, data, NULL,
+			   NULL, &res, &timeout, NULL);
+
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for release_ip failed\n"));
+		return -1;
+	}
+
+	return 0;	
+}
+
+
+/*
+  get a tunable
+ */
+int ctdb_ctrl_get_tunable(struct ctdb_context *ctdb, 
+			  struct timeval timeout, 
+			  uint32_t destnode,
+			  const char *name, uint32_t *value)
+{
+	struct ctdb_control_get_tunable *t;
+	TDB_DATA data, outdata;
+	int32_t res;
+	int ret;
+
+	data.dsize = offsetof(struct ctdb_control_get_tunable, name) + strlen(name) + 1;
+	data.dptr  = talloc_size(ctdb, data.dsize);
+	CTDB_NO_MEMORY(ctdb, data.dptr);
+
+	t = (struct ctdb_control_get_tunable *)data.dptr;
+	t->length = strlen(name)+1;
+	memcpy(t->name, name, t->length);
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_GET_TUNABLE, 0, data, ctdb,
+			   &outdata, &res, &timeout, NULL);
+	talloc_free(data.dptr);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for get_tunable failed\n"));
+		return -1;
+	}
+
+	if (outdata.dsize != sizeof(uint32_t)) {
+		DEBUG(0,("Invalid return data in get_tunable\n"));
+		talloc_free(outdata.dptr);
+		return -1;
+	}
+	
+	*value = *(uint32_t *)outdata.dptr;
+	talloc_free(outdata.dptr);
+
+	return 0;
+}
+
+/*
+  set a tunable
+ */
+int ctdb_ctrl_set_tunable(struct ctdb_context *ctdb, 
+			  struct timeval timeout, 
+			  uint32_t destnode,
+			  const char *name, uint32_t value)
+{
+	struct ctdb_control_set_tunable *t;
+	TDB_DATA data;
+	int32_t res;
+	int ret;
+
+	data.dsize = offsetof(struct ctdb_control_set_tunable, name) + strlen(name) + 1;
+	data.dptr  = talloc_size(ctdb, data.dsize);
+	CTDB_NO_MEMORY(ctdb, data.dptr);
+
+	t = (struct ctdb_control_set_tunable *)data.dptr;
+	t->length = strlen(name)+1;
+	memcpy(t->name, name, t->length);
+	t->value = value;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_SET_TUNABLE, 0, data, NULL,
+			   NULL, &res, &timeout, NULL);
+	talloc_free(data.dptr);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for set_tunable failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  list tunables
+ */
+int ctdb_ctrl_list_tunables(struct ctdb_context *ctdb, 
+			    struct timeval timeout, 
+			    uint32_t destnode,
+			    TALLOC_CTX *mem_ctx,
+			    const char ***list, uint32_t *count)
+{
+	TDB_DATA outdata;
+	int32_t res;
+	int ret;
+	struct ctdb_control_list_tunable *t;
+	char *p, *s, *ptr;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_LIST_TUNABLES, 0, tdb_null, 
+			   mem_ctx, &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for list_tunables failed\n"));
+		return -1;
+	}
+
+	t = (struct ctdb_control_list_tunable *)outdata.dptr;
+	if (outdata.dsize < offsetof(struct ctdb_control_list_tunable, data) ||
+	    t->length > outdata.dsize-offsetof(struct ctdb_control_list_tunable, data)) {
+		DEBUG(0,("Invalid data in list_tunables reply\n"));
+		talloc_free(outdata.dptr);
+		return -1;		
+	}
+	
+	p = talloc_strndup(mem_ctx, (char *)t->data, t->length);
+	CTDB_NO_MEMORY(ctdb, p);
+
+	talloc_free(outdata.dptr);
+	
+	(*list) = NULL;
+	(*count) = 0;
+
+	for (s=strtok_r(p, ":", &ptr); s; s=strtok_r(NULL, ":", &ptr)) {
+		(*list) = talloc_realloc(mem_ctx, *list, const char *, 1+(*count));
+		CTDB_NO_MEMORY(ctdb, *list);
+		(*list)[*count] = talloc_strdup(*list, s);
+		CTDB_NO_MEMORY(ctdb, (*list)[*count]);
+		(*count)++;
+	}
+
+	talloc_free(p);
+
+	return 0;
+}
+
+
+int ctdb_ctrl_get_public_ips(struct ctdb_context *ctdb, 
+			struct timeval timeout, uint32_t destnode, 
+			TALLOC_CTX *mem_ctx, struct ctdb_all_public_ips **ips)
+{
+	int ret;
+	TDB_DATA outdata;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_PUBLIC_IPS, 0, tdb_null, 
+			   mem_ctx, &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for getpublicips failed\n"));
+		return -1;
+	}
+
+	*ips = (struct ctdb_all_public_ips *)talloc_memdup(mem_ctx, outdata.dptr, outdata.dsize);
+	talloc_free(outdata.dptr);
+		    
+	return 0;
+}
+
+/*
+  set/clear the permanent disabled bit on a remote node
+ */
+int ctdb_ctrl_modflags(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, 
+		       uint32_t set, uint32_t clear)
+{
+	int ret;
+	TDB_DATA data;
+	struct ctdb_node_modflags m;
+	int32_t res;
+
+	m.set = set;
+	m.clear = clear;
+
+	data.dsize = sizeof(m);
+	data.dptr = (unsigned char *)&m;
+
+	ret = ctdb_control(ctdb, destnode, 0, 
+			   CTDB_CONTROL_MODIFY_FLAGS, 0, data, 
+			   NULL, NULL, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for modflags failed\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/*
+  get all tunables
+ */
+int ctdb_ctrl_get_all_tunables(struct ctdb_context *ctdb, 
+			       struct timeval timeout, 
+			       uint32_t destnode,
+			       struct ctdb_tunable *tunables)
+{
+	TDB_DATA outdata;
+	int ret;
+	int32_t res;
+
+	ret = ctdb_control(ctdb, destnode, 0, CTDB_CONTROL_GET_ALL_TUNABLES, 0, tdb_null, ctdb,
+			   &outdata, &res, &timeout, NULL);
+	if (ret != 0 || res != 0) {
+		DEBUG(0,(__location__ " ctdb_control for get all tunables failed\n"));
+		return -1;
+	}
+
+	if (outdata.dsize != sizeof(*tunables)) {
+		DEBUG(0,(__location__ " bad data size %u in ctdb_ctrl_get_all_tunables should be %u\n",
+			 (unsigned)outdata.dsize, (unsigned)sizeof(*tunables)));
+		return -1;		
+	}
+
+	*tunables = *(struct ctdb_tunable *)outdata.dptr;
+	talloc_free(outdata.dptr);
+	return 0;
+}
+
+
+/*
+  initialise the ctdb daemon for client applications
+
+  NOTE: In current code the daemon does not fork. This is for testing purposes only
+  and to simplify the code.
+*/
+struct ctdb_context *ctdb_init(struct event_context *ev)
+{
+	struct ctdb_context *ctdb;
+
+	ctdb = talloc_zero(ev, struct ctdb_context);
+	ctdb->ev  = ev;
+	ctdb->idr = idr_init(ctdb);
+	CTDB_NO_MEMORY_NULL(ctdb, ctdb->idr);
+
+	return ctdb;
+}
+
+
+/*
+  set some ctdb flags
+*/
+void ctdb_set_flags(struct ctdb_context *ctdb, unsigned flags)
+{
+	ctdb->flags |= flags;
+}
+
+/*
+  setup the local socket name
+*/
+int ctdb_set_socketname(struct ctdb_context *ctdb, const char *socketname)
+{
+	ctdb->daemon.name = talloc_strdup(ctdb, socketname);
+	return 0;
+}
+
+/*
+  return the vnn of this node
+*/
+uint32_t ctdb_get_vnn(struct ctdb_context *ctdb)
+{
+	return ctdb->vnn;
+}
+
diff --git a/source4/cluster/ctdb/common/cmdline.c b/source4/cluster/ctdb/common/cmdline.c
index 821e1fe38f..f8fca5e4a0 100644
--- a/source4/cluster/ctdb/common/cmdline.c
+++ b/source4/cluster/ctdb/common/cmdline.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2007
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -28,30 +28,35 @@
  */
 
 static struct {
-	const char *nlist;
-	const char *transport;
-	const char *myaddress;
-	int self_connect;
-	const char *db_dir;
+	const char *socketname;
 	int torture;
+	const char *events;
 } ctdb_cmdline = {
-	.nlist = NULL,
-	.transport = "tcp",
-	.myaddress = NULL,
-	.self_connect = 0,
-	.db_dir = NULL,
-	.torture = 0
+	.socketname = CTDB_PATH,
+	.torture = 0,
 };
 
+enum {OPT_EVENTSYSTEM=1};
+
+static void ctdb_cmdline_callback(poptContext con, 
+				  enum poptCallbackReason reason,
+				  const struct poptOption *opt,
+				  const char *arg, const void *data)
+{
+	switch (opt->val) {
+	case OPT_EVENTSYSTEM:
+		event_set_default_backend(arg);
+		break;
+	}
+}
+
 
 struct poptOption popt_ctdb_cmdline[] = {
-	{ "nlist", 0, POPT_ARG_STRING, &ctdb_cmdline.nlist, 0, "node list file", "filename" },
-	{ "listen", 0, POPT_ARG_STRING, &ctdb_cmdline.myaddress, 0, "address to listen on", "address" },
-	{ "transport", 0, POPT_ARG_STRING, &ctdb_cmdline.transport, 0, "protocol transport", NULL },
-	{ "self-connect", 0, POPT_ARG_NONE, &ctdb_cmdline.self_connect, 0, "enable self connect", "boolean" },
+	{ NULL, 0, POPT_ARG_CALLBACK, (void *)ctdb_cmdline_callback },	
+	{ "socket", 0, POPT_ARG_STRING, &ctdb_cmdline.socketname, 0, "local socket name", "filename" },
 	{ "debug", 'd', POPT_ARG_INT, &LogLevel, 0, "debug level"},
-	{ "dbdir", 0, POPT_ARG_STRING, &ctdb_cmdline.db_dir, 0, "directory for the tdb files", NULL },
 	{ "torture", 0, POPT_ARG_NONE, &ctdb_cmdline.torture, 0, "enable nastiness in library", NULL },
+	{ "events", 0, POPT_ARG_STRING, NULL, OPT_EVENTSYSTEM, "event system", NULL },
 	{ NULL }
 };
 
@@ -64,11 +69,6 @@ struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
 	struct ctdb_context *ctdb;
 	int ret;
 
-	if (ctdb_cmdline.nlist == NULL || ctdb_cmdline.myaddress == NULL) {
-		printf("You must provide a node list with --nlist and an address with --listen\n");
-		exit(1);
-	}
-
 	/* initialise ctdb */
 	ctdb = ctdb_init(ev);
 	if (ctdb == NULL) {
@@ -76,36 +76,14 @@ struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
 		exit(1);
 	}
 
-	if (ctdb_cmdline.self_connect) {
-		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
-	}
 	if (ctdb_cmdline.torture) {
 		ctdb_set_flags(ctdb, CTDB_FLAG_TORTURE);
 	}
 
-	ret = ctdb_set_transport(ctdb, ctdb_cmdline.transport);
-	if (ret == -1) {
-		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* tell ctdb what address to listen on */
-	ret = ctdb_set_address(ctdb, ctdb_cmdline.myaddress);
+	/* tell ctdb the socket address */
+	ret = ctdb_set_socketname(ctdb, ctdb_cmdline.socketname);
 	if (ret == -1) {
-		printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* tell ctdb what nodes are available */
-	ret = ctdb_set_nlist(ctdb, ctdb_cmdline.nlist);
-	if (ret == -1) {
-		printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	ret = ctdb_set_tdb_dir(ctdb, ctdb_cmdline.db_dir);
-	if (ret == -1) {
-		printf("ctdb_set_tdb_dir failed - %s\n", ctdb_errstr(ctdb));
+		printf("ctdb_set_socketname failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
 	}
 
@@ -116,7 +94,7 @@ struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
 /*
   startup a client only ctdb context
  */
-struct ctdb_context *ctdb_cmdline_client(struct event_context *ev, const char *ctdb_socket)
+struct ctdb_context *ctdb_cmdline_client(struct event_context *ev)
 {
 	struct ctdb_context *ctdb;
 	int ret;
@@ -128,7 +106,12 @@ struct ctdb_context *ctdb_cmdline_client(struct event_context *ev, const char *c
 		exit(1);
 	}
 
-	ctdb->daemon.name = talloc_strdup(ctdb, ctdb_socket);
+	/* tell ctdb the socket address */
+	ret = ctdb_set_socketname(ctdb, ctdb_cmdline.socketname);
+	if (ret == -1) {
+		printf("ctdb_set_socketname failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
 
 	ret = ctdb_socket_connect(ctdb);
 	if (ret != 0) {
@@ -137,5 +120,13 @@ struct ctdb_context *ctdb_cmdline_client(struct event_context *ev, const char *c
 		return NULL;
 	}
 
+	/* get our vnn */
+	ctdb->vnn = ctdb_ctrl_getvnn(ctdb, timeval_zero(), CTDB_CURRENT_NODE);
+	if (ctdb->vnn == (uint32_t)-1) {
+		DEBUG(0,(__location__ " Failed to get ctdb vnn\n"));
+		talloc_free(ctdb);
+		return NULL;
+	}
+
 	return ctdb;
 }
diff --git a/source4/cluster/ctdb/common/ctdb.c b/source4/cluster/ctdb/common/ctdb.c
deleted file mode 100644
index a802daa7b5..0000000000
--- a/source4/cluster/ctdb/common/ctdb.c
+++ /dev/null
@@ -1,421 +0,0 @@
-/* 
-   ctdb main protocol code
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/tdb/include/tdb.h"
-#include "lib/events/events.h"
-#include "lib/util/dlinklist.h"
-#include "system/network.h"
-#include "system/filesys.h"
-#include "../include/ctdb_private.h"
-
-/*
-  choose the transport we will use
-*/
-int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
-{
-	int ctdb_tcp_init(struct ctdb_context *ctdb);
-#ifdef USE_INFINIBAND
-	int ctdb_ibw_init(struct ctdb_context *ctdb);
-#endif /* USE_INFINIBAND */
-
-	if (strcmp(transport, "tcp") == 0) {
-		return ctdb_tcp_init(ctdb);
-	}
-#ifdef USE_INFINIBAND
-	if (strcmp(transport, "ib") == 0) {
-		return ctdb_ibw_init(ctdb);
-	}
-#endif /* USE_INFINIBAND */
-
-	ctdb_set_error(ctdb, "Unknown transport '%s'\n", transport);
-	return -1;
-}
-
-/*
-  set some ctdb flags
-*/
-void ctdb_set_flags(struct ctdb_context *ctdb, unsigned flags)
-{
-	ctdb->flags |= flags;
-}
-
-/*
-  clear some ctdb flags
-*/
-void ctdb_clear_flags(struct ctdb_context *ctdb, unsigned flags)
-{
-	ctdb->flags &= ~flags;
-}
-
-/*
-  set max acess count before a dmaster migration
-*/
-void ctdb_set_max_lacount(struct ctdb_context *ctdb, unsigned count)
-{
-	ctdb->max_lacount = count;
-}
-
-/*
-  set the directory for the local databases
-*/
-int ctdb_set_tdb_dir(struct ctdb_context *ctdb, const char *dir)
-{
-	if (dir == NULL) {
-		ctdb->db_directory = talloc_asprintf(ctdb, "ctdb-%u", ctdb_get_vnn(ctdb));
-	} else {
-		ctdb->db_directory = talloc_strdup(ctdb, dir);
-	}
-	if (ctdb->db_directory == NULL) {
-		return -1;
-	}
-	return 0;
-}
-
-/*
-  add a node to the list of active nodes
-*/
-static int ctdb_add_node(struct ctdb_context *ctdb, char *nstr)
-{
-	struct ctdb_node *node, **nodep;
-
-	nodep = talloc_realloc(ctdb, ctdb->nodes, struct ctdb_node *, ctdb->num_nodes+1);
-	CTDB_NO_MEMORY(ctdb, nodep);
-
-	ctdb->nodes = nodep;
-	nodep = &ctdb->nodes[ctdb->num_nodes];
-	(*nodep) = talloc_zero(ctdb->nodes, struct ctdb_node);
-	CTDB_NO_MEMORY(ctdb, *nodep);
-	node = *nodep;
-
-	if (ctdb_parse_address(ctdb, node, nstr, &node->address) != 0) {
-		return -1;
-	}
-	node->ctdb = ctdb;
-	node->name = talloc_asprintf(node, "%s:%u", 
-				     node->address.address, 
-				     node->address.port);
-	/* for now we just set the vnn to the line in the file - this
-	   will change! */
-	node->vnn = ctdb->num_nodes;
-
-	if (ctdb->methods->add_node(node) != 0) {
-		talloc_free(node);
-		return -1;
-	}
-
-	if (ctdb_same_address(&ctdb->address, &node->address)) {
-		ctdb->vnn = node->vnn;
-	}
-
-	ctdb->num_nodes++;
-
-	return 0;
-}
-
-/*
-  setup the node list from a file
-*/
-int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist)
-{
-	char **lines;
-	int nlines;
-	int i;
-
-	lines = file_lines_load(nlist, &nlines, ctdb);
-	if (lines == NULL) {
-		ctdb_set_error(ctdb, "Failed to load nlist '%s'\n", nlist);
-		return -1;
-	}
-
-	for (i=0;i<nlines;i++) {
-		if (ctdb_add_node(ctdb, lines[i]) != 0) {
-			talloc_free(lines);
-			return -1;
-		}
-	}
-	
-	talloc_free(lines);
-	return 0;
-}
-
-/*
-  setup the local node address
-*/
-int ctdb_set_address(struct ctdb_context *ctdb, const char *address)
-{
-	if (ctdb_parse_address(ctdb, ctdb, address, &ctdb->address) != 0) {
-		return -1;
-	}
-	
-	ctdb->name = talloc_asprintf(ctdb, "%s:%u", 
-				     ctdb->address.address, 
-				     ctdb->address.port);
-	return 0;
-}
-
-/*
-  add a node to the list of active nodes
-*/
-int ctdb_set_call(struct ctdb_db_context *ctdb_db, ctdb_fn_t fn, int id)
-{
-	struct ctdb_registered_call *call;
-
-	call = talloc(ctdb_db, struct ctdb_registered_call);
-	call->fn = fn;
-	call->id = id;
-
-	DLIST_ADD(ctdb_db->calls, call);	
-	return 0;
-}
-
-/*
-  return the vnn of this node
-*/
-uint32_t ctdb_get_vnn(struct ctdb_context *ctdb)
-{
-	return ctdb->vnn;
-}
-
-/*
-  return the number of nodes
-*/
-uint32_t ctdb_get_num_nodes(struct ctdb_context *ctdb)
-{
-	return ctdb->num_nodes;
-}
-
-
-/*
-  called by the transport layer when a packet comes in
-*/
-void ctdb_recv_pkt(struct ctdb_context *ctdb, uint8_t *data, uint32_t length)
-{
-	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
-	TALLOC_CTX *tmp_ctx;
-
-	ctdb->status.node_packets_recv++;
-
-	/* place the packet as a child of the tmp_ctx. We then use
-	   talloc_free() below to free it. If any of the calls want
-	   to keep it, then they will steal it somewhere else, and the
-	   talloc_free() will only free the tmp_ctx */
-	tmp_ctx = talloc_new(ctdb);
-	talloc_steal(tmp_ctx, hdr);
-
-	if (length < sizeof(*hdr)) {
-		ctdb_set_error(ctdb, "Bad packet length %d\n", length);
-		goto done;
-	}
-	if (length != hdr->length) {
-		ctdb_set_error(ctdb, "Bad header length %d expected %d\n", 
-			       hdr->length, length);
-		goto done;
-	}
-
-	if (hdr->ctdb_magic != CTDB_MAGIC) {
-		ctdb_set_error(ctdb, "Non CTDB packet rejected\n");
-		goto done;
-	}
-
-	if (hdr->ctdb_version != CTDB_VERSION) {
-		ctdb_set_error(ctdb, "Bad CTDB version 0x%x rejected\n", hdr->ctdb_version);
-		goto done;
-	}
-
-	DEBUG(3,(__location__ " ctdb request %d of type %d length %d from "
-		 "node %d to %d\n", hdr->reqid, hdr->operation, hdr->length,
-		 hdr->srcnode, hdr->destnode));
-
-	switch (hdr->operation) {
-	case CTDB_REQ_CALL:
-		ctdb->status.count.req_call++;
-		ctdb_request_call(ctdb, hdr);
-		break;
-
-	case CTDB_REPLY_CALL:
-		ctdb->status.count.reply_call++;
-		ctdb_reply_call(ctdb, hdr);
-		break;
-
-	case CTDB_REPLY_ERROR:
-		ctdb->status.count.reply_error++;
-		ctdb_reply_error(ctdb, hdr);
-		break;
-
-	case CTDB_REPLY_REDIRECT:
-		ctdb->status.count.reply_redirect++;
-		ctdb_reply_redirect(ctdb, hdr);
-		break;
-
-	case CTDB_REQ_DMASTER:
-		ctdb->status.count.req_dmaster++;
-		ctdb_request_dmaster(ctdb, hdr);
-		break;
-
-	case CTDB_REPLY_DMASTER:
-		ctdb->status.count.reply_dmaster++;
-		ctdb_reply_dmaster(ctdb, hdr);
-		break;
-
-	case CTDB_REQ_MESSAGE:
-		ctdb->status.count.req_message++;
-		ctdb_request_message(ctdb, hdr);
-		break;
-
-	case CTDB_REQ_FINISHED:
-		ctdb->status.count.req_finished++;
-		ctdb_request_finished(ctdb, hdr);
-		break;
-
-	default:
-		DEBUG(0,("%s: Packet with unknown operation %d\n", 
-			 __location__, hdr->operation));
-		break;
-	}
-
-done:
-	talloc_free(tmp_ctx);
-}
-
-/*
-  called by the transport layer when a packet comes in
-*/
-void ctdb_recv_raw_pkt(void *p, uint8_t *data, uint32_t length)
-{
-	struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
-	ctdb_recv_pkt(ctdb, data, length);
-}
-
-/*
-  called by the transport layer when a node is dead
-*/
-static void ctdb_node_dead(struct ctdb_node *node)
-{
-	node->ctdb->num_connected--;
-	DEBUG(1,("%s: node %s is dead: %d connected\n", 
-		 node->ctdb->name, node->name, node->ctdb->num_connected));
-}
-
-/*
-  called by the transport layer when a node is connected
-*/
-static void ctdb_node_connected(struct ctdb_node *node)
-{
-	node->ctdb->num_connected++;
-	DEBUG(1,("%s: connected to %s - %d connected\n", 
-		 node->ctdb->name, node->name, node->ctdb->num_connected));
-}
-
-/*
-  wait for all nodes to be connected
-*/
-void ctdb_daemon_connect_wait(struct ctdb_context *ctdb)
-{
-	int expected = ctdb->num_nodes - 1;
-	if (ctdb->flags & CTDB_FLAG_SELF_CONNECT) {
-		expected++;
-	}
-	while (ctdb->num_connected != expected) {
-		DEBUG(3,("ctdb_connect_wait: waiting for %d nodes (have %d)\n", 
-			 expected, ctdb->num_connected));
-		event_loop_once(ctdb->ev);
-	}
-	DEBUG(3,("ctdb_connect_wait: got all %d nodes\n", expected));
-}
-
-struct queue_next {
-	struct ctdb_context *ctdb;
-	struct ctdb_req_header *hdr;
-};
-
-
-/*
-  trigered when a deferred packet is due
- */
-static void queue_next_trigger(struct event_context *ev, struct timed_event *te, 
-			       struct timeval t, void *private_data)
-{
-	struct queue_next *q = talloc_get_type(private_data, struct queue_next);
-	ctdb_recv_pkt(q->ctdb, (uint8_t *)q->hdr, q->hdr->length);
-	talloc_free(q);
-}	
-
-/*
-  defer a packet, so it is processed on the next event loop
-  this is used for sending packets to ourselves
- */
-static void ctdb_defer_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
-{
-	struct queue_next *q;
-	q = talloc(ctdb, struct queue_next);
-	if (q == NULL) {
-		DEBUG(0,(__location__ " Failed to allocate deferred packet\n"));
-		return;
-	}
-	q->ctdb = ctdb;
-	q->hdr = talloc_memdup(ctdb, hdr, hdr->length);
-	if (q->hdr == NULL) {
-		DEBUG(0,("Error copying deferred packet to self\n"));
-		return;
-	}
-	event_add_timed(ctdb->ev, q, timeval_zero(), queue_next_trigger, q);
-}
-
-/*
-  queue a packet or die
-*/
-void ctdb_queue_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
-{
-	struct ctdb_node *node;
-	ctdb->status.node_packets_sent++;
-	node = ctdb->nodes[hdr->destnode];
-	if (hdr->destnode == ctdb->vnn && !(ctdb->flags & CTDB_FLAG_SELF_CONNECT)) {
-		ctdb_defer_packet(ctdb, hdr);
-	} else if (ctdb->methods->queue_pkt(node, (uint8_t *)hdr, hdr->length) != 0) {
-		ctdb_fatal(ctdb, "Unable to queue packet\n");
-	}
-}
-
-
-static const struct ctdb_upcalls ctdb_upcalls = {
-	.recv_pkt       = ctdb_recv_pkt,
-	.node_dead      = ctdb_node_dead,
-	.node_connected = ctdb_node_connected
-};
-
-/*
-  initialise the ctdb daemon. 
-
-  NOTE: In current code the daemon does not fork. This is for testing purposes only
-  and to simplify the code.
-*/
-struct ctdb_context *ctdb_init(struct event_context *ev)
-{
-	struct ctdb_context *ctdb;
-
-	ctdb = talloc_zero(ev, struct ctdb_context);
-	ctdb->ev = ev;
-	ctdb->upcalls = &ctdb_upcalls;
-	ctdb->idr = idr_init(ctdb);
-	ctdb->max_lacount = CTDB_DEFAULT_MAX_LACOUNT;
-
-	return ctdb;
-}
-
diff --git a/source4/cluster/ctdb/common/ctdb_client.c b/source4/cluster/ctdb/common/ctdb_client.c
deleted file mode 100644
index e3806f517d..0000000000
--- a/source4/cluster/ctdb/common/ctdb_client.c
+++ /dev/null
@@ -1,681 +0,0 @@
-/* 
-   ctdb daemon code
-
-   Copyright (C) Andrew Tridgell  2007
-   Copyright (C) Ronnie Sahlberg  2007
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "db_wrap.h"
-#include "lib/tdb/include/tdb.h"
-#include "lib/events/events.h"
-#include "lib/util/dlinklist.h"
-#include "system/network.h"
-#include "system/filesys.h"
-#include "../include/ctdb.h"
-#include "../include/ctdb_private.h"
-
-/*
-  queue a packet for sending from client to daemon
-*/
-static int ctdb_client_queue_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
-{
-	return ctdb_queue_send(ctdb->daemon.queue, (uint8_t *)hdr, hdr->length);
-}
-
-
-/*
-  handle a connect wait reply packet
- */
-static void ctdb_reply_connect_wait(struct ctdb_context *ctdb, 
-				    struct ctdb_req_header *hdr)
-{
-	struct ctdb_reply_connect_wait *r = (struct ctdb_reply_connect_wait *)hdr;
-	ctdb->num_connected = r->num_connected;
-}
-
-/*
-  state of a in-progress ctdb call in client
-*/
-struct ctdb_client_call_state {
-	enum call_state state;
-	uint32_t reqid;
-	struct ctdb_db_context *ctdb_db;
-	struct ctdb_call call;
-};
-
-/*
-  called when a CTDB_REPLY_CALL packet comes in in the client
-
-  This packet comes in response to a CTDB_REQ_CALL request packet. It
-  contains any reply data from the call
-*/
-static void ctdb_client_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
-{
-	struct ctdb_reply_call *c = (struct ctdb_reply_call *)hdr;
-	struct ctdb_client_call_state *state;
-
-	state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_client_call_state);
-	if (state == NULL) {
-		DEBUG(0, ("reqid %d not found\n", hdr->reqid));
-		return;
-	}
-
-	state->call.reply_data.dptr = c->data;
-	state->call.reply_data.dsize = c->datalen;
-	state->call.status = c->status;
-
-	talloc_steal(state, c);
-
-	state->state = CTDB_CALL_DONE;
-}
-
-static void ctdb_reply_status(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
-
-/*
-  this is called in the client, when data comes in from the daemon
- */
-static void ctdb_client_read_cb(uint8_t *data, size_t cnt, void *args)
-{
-	struct ctdb_context *ctdb = talloc_get_type(args, struct ctdb_context);
-	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
-	TALLOC_CTX *tmp_ctx;
-
-	/* place the packet as a child of a tmp_ctx. We then use
-	   talloc_free() below to free it. If any of the calls want
-	   to keep it, then they will steal it somewhere else, and the
-	   talloc_free() will be a no-op */
-	tmp_ctx = talloc_new(ctdb);
-	talloc_steal(tmp_ctx, hdr);
-
-	if (cnt == 0) {
-		DEBUG(2,("Daemon has exited - shutting down client\n"));
-		exit(0);
-	}
-
-	if (cnt < sizeof(*hdr)) {
-		DEBUG(0,("Bad packet length %d in client\n", cnt));
-		goto done;
-	}
-	if (cnt != hdr->length) {
-		ctdb_set_error(ctdb, "Bad header length %d expected %d in client\n", 
-			       hdr->length, cnt);
-		goto done;
-	}
-
-	if (hdr->ctdb_magic != CTDB_MAGIC) {
-		ctdb_set_error(ctdb, "Non CTDB packet rejected in client\n");
-		goto done;
-	}
-
-	if (hdr->ctdb_version != CTDB_VERSION) {
-		ctdb_set_error(ctdb, "Bad CTDB version 0x%x rejected in client\n", hdr->ctdb_version);
-		goto done;
-	}
-
-	switch (hdr->operation) {
-	case CTDB_REPLY_CALL:
-		ctdb_client_reply_call(ctdb, hdr);
-		break;
-
-	case CTDB_REQ_MESSAGE:
-		ctdb_request_message(ctdb, hdr);
-		break;
-
-	case CTDB_REPLY_CONNECT_WAIT:
-		ctdb_reply_connect_wait(ctdb, hdr);
-		break;
-
-	case CTDB_REPLY_STATUS:
-		ctdb_reply_status(ctdb, hdr);
-		break;
-
-	default:
-		DEBUG(0,("bogus operation code:%d\n",hdr->operation));
-	}
-
-done:
-	talloc_free(tmp_ctx);
-}
-
-/*
-  connect to a unix domain socket
-*/
-int ctdb_socket_connect(struct ctdb_context *ctdb)
-{
-	struct sockaddr_un addr;
-
-	memset(&addr, 0, sizeof(addr));
-	addr.sun_family = AF_UNIX;
-	strncpy(addr.sun_path, ctdb->daemon.name, sizeof(addr.sun_path));
-
-	ctdb->daemon.sd = socket(AF_UNIX, SOCK_STREAM, 0);
-	if (ctdb->daemon.sd == -1) {
-		return -1;
-	}
-	
-	if (connect(ctdb->daemon.sd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
-		close(ctdb->daemon.sd);
-		ctdb->daemon.sd = -1;
-		return -1;
-	}
-
-	ctdb->daemon.queue = ctdb_queue_setup(ctdb, ctdb, ctdb->daemon.sd, 
-					      CTDB_DS_ALIGNMENT, 
-					      ctdb_client_read_cb, ctdb);
-	return 0;
-}
-
-
-struct ctdb_record_handle {
-	struct ctdb_db_context *ctdb_db;
-	TDB_DATA key;
-	TDB_DATA *data;
-	struct ctdb_ltdb_header header;
-};
-
-
-/*
-  make a recv call to the local ctdb daemon - called from client context
-
-  This is called when the program wants to wait for a ctdb_call to complete and get the 
-  results. This call will block unless the call has already completed.
-*/
-int ctdb_call_recv(struct ctdb_client_call_state *state, struct ctdb_call *call)
-{
-	while (state->state < CTDB_CALL_DONE) {
-		event_loop_once(state->ctdb_db->ctdb->ev);
-	}
-	if (state->state != CTDB_CALL_DONE) {
-		DEBUG(0,(__location__ " ctdb_call_recv failed\n"));
-		talloc_free(state);
-		return -1;
-	}
-
-	if (state->call.reply_data.dsize) {
-		call->reply_data.dptr = talloc_memdup(state->ctdb_db,
-						      state->call.reply_data.dptr,
-						      state->call.reply_data.dsize);
-		call->reply_data.dsize = state->call.reply_data.dsize;
-	} else {
-		call->reply_data.dptr = NULL;
-		call->reply_data.dsize = 0;
-	}
-	call->status = state->call.status;
-	talloc_free(state);
-
-	return 0;
-}
-
-
-
-
-/*
-  destroy a ctdb_call in client
-*/
-static int ctdb_client_call_destructor(struct ctdb_client_call_state *state)	
-{
-	idr_remove(state->ctdb_db->ctdb->idr, state->reqid);
-	return 0;
-}
-
-/*
-  construct an event driven local ctdb_call
-
-  this is used so that locally processed ctdb_call requests are processed
-  in an event driven manner
-*/
-static struct ctdb_client_call_state *ctdb_client_call_local_send(struct ctdb_db_context *ctdb_db, 
-								  struct ctdb_call *call,
-								  struct ctdb_ltdb_header *header,
-								  TDB_DATA *data)
-{
-	struct ctdb_client_call_state *state;
-	struct ctdb_context *ctdb = ctdb_db->ctdb;
-	int ret;
-
-	state = talloc_zero(ctdb_db, struct ctdb_client_call_state);
-	CTDB_NO_MEMORY_NULL(ctdb, state);
-
-	talloc_steal(state, data->dptr);
-
-	state->state = CTDB_CALL_DONE;
-	state->call = *call;
-	state->ctdb_db = ctdb_db;
-
-	ret = ctdb_call_local(ctdb_db, &state->call, header, data, ctdb->vnn);
-	talloc_steal(state, state->call.reply_data.dptr);
-
-	return state;
-}
-
-/*
-  make a ctdb call to the local daemon - async send. Called from client context.
-
-  This constructs a ctdb_call request and queues it for processing. 
-  This call never blocks.
-*/
-struct ctdb_client_call_state *ctdb_call_send(struct ctdb_db_context *ctdb_db, 
-				       struct ctdb_call *call)
-{
-	struct ctdb_client_call_state *state;
-	struct ctdb_context *ctdb = ctdb_db->ctdb;
-	struct ctdb_ltdb_header header;
-	TDB_DATA data;
-	int ret;
-	size_t len;
-	struct ctdb_req_call *c;
-
-	/* if the domain socket is not yet open, open it */
-	if (ctdb->daemon.sd==-1) {
-		ctdb_socket_connect(ctdb);
-	}
-
-	ret = ctdb_ltdb_lock(ctdb_db, call->key);
-	if (ret != 0) {
-		DEBUG(0,(__location__ " Failed to get chainlock\n"));
-		return NULL;
-	}
-
-	ret = ctdb_ltdb_fetch(ctdb_db, call->key, &header, ctdb_db, &data);
-	if (ret != 0) {
-		ctdb_ltdb_unlock(ctdb_db, call->key);
-		DEBUG(0,(__location__ " Failed to fetch record\n"));
-		return NULL;
-	}
-
-	if (header.dmaster == ctdb->vnn && !(ctdb->flags & CTDB_FLAG_SELF_CONNECT)) {
-		state = ctdb_client_call_local_send(ctdb_db, call, &header, &data);
-		talloc_free(data.dptr);
-		ctdb_ltdb_unlock(ctdb_db, call->key);
-		return state;
-	}
-
-	ctdb_ltdb_unlock(ctdb_db, call->key);
-	talloc_free(data.dptr);
-
-	state = talloc_zero(ctdb_db, struct ctdb_client_call_state);
-	if (state == NULL) {
-		DEBUG(0, (__location__ " failed to allocate state\n"));
-		return NULL;
-	}
-
-	len = offsetof(struct ctdb_req_call, data) + call->key.dsize + call->call_data.dsize;
-	c = ctdbd_allocate_pkt(state, len);
-	if (c == NULL) {
-		DEBUG(0, (__location__ " failed to allocate packet\n"));
-		return NULL;
-	}
-	talloc_set_name_const(c, "ctdb client req_call packet");
-	memset(c, 0, offsetof(struct ctdb_req_call, data));
-
-	c->hdr.length    = len;
-	c->hdr.ctdb_magic = CTDB_MAGIC;
-	c->hdr.ctdb_version = CTDB_VERSION;
-	c->hdr.operation = CTDB_REQ_CALL;
-	/* this limits us to 16k outstanding messages - not unreasonable */
-	c->hdr.reqid     = idr_get_new(ctdb->idr, state, 0xFFFF);
-	c->flags         = call->flags;
-	c->db_id         = ctdb_db->db_id;
-	c->callid        = call->call_id;
-	c->keylen        = call->key.dsize;
-	c->calldatalen   = call->call_data.dsize;
-	memcpy(&c->data[0], call->key.dptr, call->key.dsize);
-	memcpy(&c->data[call->key.dsize], 
-	       call->call_data.dptr, call->call_data.dsize);
-	state->call                = *call;
-	state->call.call_data.dptr = &c->data[call->key.dsize];
-	state->call.key.dptr       = &c->data[0];
-
-	state->state  = CTDB_CALL_WAIT;
-	state->ctdb_db = ctdb_db;
-	state->reqid = c->hdr.reqid;
-
-	talloc_set_destructor(state, ctdb_client_call_destructor);
-
-	ctdb_client_queue_pkt(ctdb, &c->hdr);
-
-	return state;
-}
-
-
-/*
-  full ctdb_call. Equivalent to a ctdb_call_send() followed by a ctdb_call_recv()
-*/
-int ctdb_call(struct ctdb_db_context *ctdb_db, struct ctdb_call *call)
-{
-	struct ctdb_client_call_state *state;
-
-	state = ctdb_call_send(ctdb_db, call);
-	return ctdb_call_recv(state, call);
-}
-
-
-/*
-  tell the daemon what messaging srvid we will use, and register the message
-  handler function in the client
-*/
-int ctdb_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-			     ctdb_message_fn_t handler,
-			     void *private_data)
-				    
-{
-	struct ctdb_req_register c;
-	int res;
-
-	/* if the domain socket is not yet open, open it */
-	if (ctdb->daemon.sd==-1) {
-		ctdb_socket_connect(ctdb);
-	}
-
-	ZERO_STRUCT(c);
-
-	c.hdr.length       = sizeof(c);
-	c.hdr.ctdb_magic   = CTDB_MAGIC;
-	c.hdr.ctdb_version = CTDB_VERSION;
-	c.hdr.operation    = CTDB_REQ_REGISTER;
-	c.srvid            = srvid;
-
-	res = ctdb_client_queue_pkt(ctdb, &c.hdr);
-	if (res != 0) {
-		return res;
-	}
-
-	/* also need to register the handler with our ctdb structure */
-	return ctdb_register_message_handler(ctdb, ctdb, srvid, handler, private_data);
-}
-
-
-/*
-  send a message - from client context
- */
-int ctdb_send_message(struct ctdb_context *ctdb, uint32_t vnn,
-		      uint32_t srvid, TDB_DATA data)
-{
-	struct ctdb_req_message *r;
-	int len, res;
-
-	len = offsetof(struct ctdb_req_message, data) + data.dsize;
-	r = ctdb->methods->allocate_pkt(ctdb, len);
-	CTDB_NO_MEMORY(ctdb, r);
-	talloc_set_name_const(r, "req_message packet");
-
-	r->hdr.length    = len;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REQ_MESSAGE;
-	r->hdr.destnode  = vnn;
-	r->hdr.srcnode   = ctdb->vnn;
-	r->hdr.reqid     = 0;
-	r->srvid         = srvid;
-	r->datalen       = data.dsize;
-	memcpy(&r->data[0], data.dptr, data.dsize);
-	
-	res = ctdb_client_queue_pkt(ctdb, &r->hdr);
-	if (res != 0) {
-		return res;
-	}
-
-	talloc_free(r);
-	return 0;
-}
-
-/*
-  wait for all nodes to be connected - from client
- */
-void ctdb_connect_wait(struct ctdb_context *ctdb)
-{
-	struct ctdb_req_connect_wait r;
-	int res;
-
-	ZERO_STRUCT(r);
-
-	r.hdr.length     = sizeof(r);
-	r.hdr.ctdb_magic = CTDB_MAGIC;
-	r.hdr.ctdb_version = CTDB_VERSION;
-	r.hdr.operation = CTDB_REQ_CONNECT_WAIT;
-
-	DEBUG(3,("ctdb_connect_wait: sending to ctdbd\n"));
-
-	/* if the domain socket is not yet open, open it */
-	if (ctdb->daemon.sd==-1) {
-		ctdb_socket_connect(ctdb);
-	}
-	
-	res = ctdb_queue_send(ctdb->daemon.queue, (uint8_t *)&r.hdr, r.hdr.length);
-	if (res != 0) {
-		DEBUG(0,(__location__ " Failed to queue a connect wait request\n"));
-		return;
-	}
-
-	DEBUG(3,("ctdb_connect_wait: waiting\n"));
-
-	/* now we can go into the normal wait routine, as the reply packet
-	   will update the ctdb->num_connected variable */
-	ctdb_daemon_connect_wait(ctdb);
-}
-
-/*
-  cancel a ctdb_fetch_lock operation, releasing the lock
- */
-static int fetch_lock_destructor(struct ctdb_record_handle *h)
-{
-	ctdb_ltdb_unlock(h->ctdb_db, h->key);
-	return 0;
-}
-
-/*
-  force the migration of a record to this node
- */
-static int ctdb_client_force_migration(struct ctdb_db_context *ctdb_db, TDB_DATA key)
-{
-	struct ctdb_call call;
-	ZERO_STRUCT(call);
-	call.call_id = CTDB_NULL_FUNC;
-	call.key = key;
-	call.flags = CTDB_IMMEDIATE_MIGRATION;
-	return ctdb_call(ctdb_db, &call);
-}
-
-/*
-  get a lock on a record, and return the records data. Blocks until it gets the lock
- */
-struct ctdb_record_handle *ctdb_fetch_lock(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx, 
-					   TDB_DATA key, TDB_DATA *data)
-{
-	int ret;
-	struct ctdb_record_handle *h;
-
-	/*
-	  procedure is as follows:
-
-	  1) get the chain lock. 
-	  2) check if we are dmaster
-	  3) if we are the dmaster then return handle 
-	  4) if not dmaster then ask ctdb daemon to make us dmaster, and wait for
-	     reply from ctdbd
-	  5) when we get the reply, goto (1)
-	 */
-
-	h = talloc_zero(mem_ctx, struct ctdb_record_handle);
-	if (h == NULL) {
-		return NULL;
-	}
-
-	h->ctdb_db = ctdb_db;
-	h->key     = key;
-	h->key.dptr = talloc_memdup(h, key.dptr, key.dsize);
-	if (h->key.dptr == NULL) {
-		talloc_free(h);
-		return NULL;
-	}
-	h->data    = data;
-
-	DEBUG(3,("ctdb_fetch_lock: key=%*.*s\n", key.dsize, key.dsize, 
-		 (const char *)key.dptr));
-
-again:
-	/* step 1 - get the chain lock */
-	ret = ctdb_ltdb_lock(ctdb_db, key);
-	if (ret != 0) {
-		DEBUG(0, (__location__ " failed to lock ltdb record\n"));
-		talloc_free(h);
-		return NULL;
-	}
-
-	DEBUG(4,("ctdb_fetch_lock: got chain lock\n"));
-
-	talloc_set_destructor(h, fetch_lock_destructor);
-
-	ret = ctdb_ltdb_fetch(ctdb_db, key, &h->header, h, data);
-	if (ret != 0) {
-		ctdb_ltdb_unlock(ctdb_db, key);
-		talloc_free(h);
-		return NULL;
-	}
-
-	/* when torturing, ensure we test the remote path */
-	if ((ctdb_db->ctdb->flags & CTDB_FLAG_TORTURE) &&
-	    random() % 5 == 0) {
-		h->header.dmaster = (uint32_t)-1;
-	}
-
-
-	DEBUG(4,("ctdb_fetch_lock: done local fetch\n"));
-
-	if (h->header.dmaster != ctdb_db->ctdb->vnn) {
-		ctdb_ltdb_unlock(ctdb_db, key);
-		ret = ctdb_client_force_migration(ctdb_db, key);
-		if (ret != 0) {
-			DEBUG(4,("ctdb_fetch_lock: force_migration failed\n"));
-			talloc_free(h);
-			return NULL;
-		}
-		goto again;
-	}
-
-	DEBUG(4,("ctdb_fetch_lock: we are dmaster - done\n"));
-	return h;
-}
-
-/*
-  store some data to the record that was locked with ctdb_fetch_lock()
-*/
-int ctdb_record_store(struct ctdb_record_handle *h, TDB_DATA data)
-{
-	return ctdb_ltdb_store(h->ctdb_db, h->key, &h->header, data);
-}
-
-/*
-  wait until we're the only node left.
-  this function never returns
-*/
-void ctdb_shutdown(struct ctdb_context *ctdb)
-{
-	struct ctdb_req_shutdown r;
-	int len;
-
-	/* if the domain socket is not yet open, open it */
-	if (ctdb->daemon.sd==-1) {
-		ctdb_socket_connect(ctdb);
-	}
-
-	len = sizeof(struct ctdb_req_shutdown);
-	ZERO_STRUCT(r);
-	r.hdr.length       = len;
-	r.hdr.ctdb_magic   = CTDB_MAGIC;
-	r.hdr.ctdb_version = CTDB_VERSION;
-	r.hdr.operation    = CTDB_REQ_SHUTDOWN;
-	r.hdr.reqid        = 0;
-
-	ctdb_client_queue_pkt(ctdb, &(r.hdr));
-
-	/* this event loop will terminate once we receive the reply */
-	while (1) {
-		event_loop_once(ctdb->ev);
-	}
-}
-
-enum ctdb_status_states {CTDB_STATUS_WAIT, CTDB_STATUS_DONE};
-
-struct ctdb_status_state {
-	uint32_t reqid;
-	struct ctdb_status *status;
-	enum ctdb_status_states state;
-};
-
-/*
-  handle a ctdb_reply_status reply
- */
-static void ctdb_reply_status(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
-{
-	struct ctdb_reply_status *r = (struct ctdb_reply_status *)hdr;
-	struct ctdb_status_state *state;
-
-	state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_status_state);
-	if (state == NULL) {
-		DEBUG(0, ("reqid %d not found\n", hdr->reqid));
-		return;
-	}
-
-	*state->status = r->status;
-	state->state = CTDB_STATUS_DONE;
-}
-
-/*
-  wait until we're the only node left.
-  this function never returns
-*/
-int ctdb_status(struct ctdb_context *ctdb, struct ctdb_status *status)
-{
-	struct ctdb_req_status r;
-	int ret;
-	struct ctdb_status_state *state;
-
-	/* if the domain socket is not yet open, open it */
-	if (ctdb->daemon.sd==-1) {
-		ctdb_socket_connect(ctdb);
-	}
-
-	state = talloc(ctdb, struct ctdb_status_state);
-	CTDB_NO_MEMORY(ctdb, state);
-
-	state->reqid = idr_get_new(ctdb->idr, state, 0xFFFF);
-	state->status = status;
-	state->state = CTDB_STATUS_WAIT;
-	
-	ZERO_STRUCT(r);
-	r.hdr.length       = sizeof(r);
-	r.hdr.ctdb_magic   = CTDB_MAGIC;
-	r.hdr.ctdb_version = CTDB_VERSION;
-	r.hdr.operation    = CTDB_REQ_STATUS;
-	r.hdr.reqid        = state->reqid;
-
-	ret = ctdb_client_queue_pkt(ctdb, &(r.hdr));
-	if (ret != 0) {
-		talloc_free(state);
-		return -1;
-	}
-	
-	while (state->state == CTDB_STATUS_WAIT) {
-		event_loop_once(ctdb->ev);
-	}
-
-	talloc_free(state);
-
-	return 0;
-}
-
diff --git a/source4/cluster/ctdb/common/ctdb_daemon.c b/source4/cluster/ctdb/common/ctdb_daemon.c
deleted file mode 100644
index f1de3933e7..0000000000
--- a/source4/cluster/ctdb/common/ctdb_daemon.c
+++ /dev/null
@@ -1,710 +0,0 @@
-/* 
-   ctdb daemon code
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "db_wrap.h"
-#include "lib/tdb/include/tdb.h"
-#include "lib/events/events.h"
-#include "lib/util/dlinklist.h"
-#include "system/network.h"
-#include "system/filesys.h"
-#include "system/wait.h"
-#include "../include/ctdb.h"
-#include "../include/ctdb_private.h"
-
-/*
-  structure describing a connected client in the daemon
- */
-struct ctdb_client {
-	struct ctdb_context *ctdb;
-	int fd;
-	struct ctdb_queue *queue;
-};
-
-
-
-static void daemon_incoming_packet(void *, uint8_t *, uint32_t );
-
-static void ctdb_main_loop(struct ctdb_context *ctdb)
-{
-	ctdb->methods->start(ctdb);
-
-	/* go into a wait loop to allow other nodes to complete */
-	event_loop_wait(ctdb->ev);
-
-	DEBUG(0,("event_loop_wait() returned. this should not happen\n"));
-	exit(1);
-}
-
-
-static void set_non_blocking(int fd)
-{
-	unsigned v;
-	v = fcntl(fd, F_GETFL, 0);
-        fcntl(fd, F_SETFL, v | O_NONBLOCK);
-}
-
-static void block_signal(int signum)
-{
-	struct sigaction act;
-
-	memset(&act, 0, sizeof(act));
-
-	act.sa_handler = SIG_IGN;
-	sigemptyset(&act.sa_mask);
-	sigaddset(&act.sa_mask, signum);
-	sigaction(signum, &act, NULL);
-}
-
-
-/*
-  send a packet to a client
- */
-static int daemon_queue_send(struct ctdb_client *client, struct ctdb_req_header *hdr)
-{
-	client->ctdb->status.client_packets_sent++;
-	return ctdb_queue_send(client->queue, (uint8_t *)hdr, hdr->length);
-}
-
-/*
-  message handler for when we are in daemon mode. This redirects the message
-  to the right client
- */
-static void daemon_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-				    TDB_DATA data, void *private_data)
-{
-	struct ctdb_client *client = talloc_get_type(private_data, struct ctdb_client);
-	struct ctdb_req_message *r;
-	int len;
-
-	/* construct a message to send to the client containing the data */
-	len = offsetof(struct ctdb_req_message, data) + data.dsize;
-	r = ctdbd_allocate_pkt(ctdb, len);
-
-	talloc_set_name_const(r, "req_message packet");
-
-	memset(r, 0, offsetof(struct ctdb_req_message, data));
-
-	r->hdr.length    = len;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REQ_MESSAGE;
-	r->srvid         = srvid;
-	r->datalen       = data.dsize;
-	memcpy(&r->data[0], data.dptr, data.dsize);
-
-	daemon_queue_send(client, &r->hdr);
-
-	talloc_free(r);
-}
-					   
-
-/*
-  this is called when the ctdb daemon received a ctdb request to 
-  set the srvid from the client
- */
-static void daemon_request_register_message_handler(struct ctdb_client *client, 
-						    struct ctdb_req_register *c)
-{
-	int res;
-	res = ctdb_register_message_handler(client->ctdb, client, 
-					    c->srvid, daemon_message_handler, 
-					    client);
-	if (res != 0) {
-		DEBUG(0,(__location__ " Failed to register handler %u in daemon\n", 
-			 c->srvid));
-	} else {
-		DEBUG(2,(__location__ " Registered message handler for srvid=%u\n", 
-			 c->srvid));
-	}
-}
-
-
-/*
-  called when the daemon gets a shutdown request from a client
- */
-static void daemon_request_shutdown(struct ctdb_client *client, 
-				      struct ctdb_req_shutdown *f)
-{
-	struct ctdb_context *ctdb = talloc_get_type(client->ctdb, struct ctdb_context);
-	int len;
-	uint32_t node;
-
-	/* we dont send to ourself so we can already count one daemon as
-	   exiting */
-	ctdb->num_finished++;
-
-
-	/* loop over all nodes of the cluster */
-	for (node=0; node<ctdb->num_nodes;node++) {
-		struct ctdb_req_finished *rf;
-
-		/* dont send a message to ourself */
-		if (ctdb->vnn == node) {
-			continue;
-		}
-
-		len = sizeof(struct ctdb_req_finished);
-		rf = ctdb->methods->allocate_pkt(ctdb, len);
-		CTDB_NO_MEMORY_FATAL(ctdb, rf);
-		talloc_set_name_const(rf, "ctdb_req_finished packet");
-
-		ZERO_STRUCT(*rf);
-		rf->hdr.length    = len;
-		rf->hdr.ctdb_magic = CTDB_MAGIC;
-		rf->hdr.ctdb_version = CTDB_VERSION;
-		rf->hdr.operation = CTDB_REQ_FINISHED;
-		rf->hdr.destnode  = node;
-		rf->hdr.srcnode   = ctdb->vnn;
-		rf->hdr.reqid     = 0;
-
-		ctdb_queue_packet(ctdb, &(rf->hdr));
-
-		talloc_free(rf);
-	}
-
-	/* wait until all nodes have are prepared to shutdown */
-	while (ctdb->num_finished != ctdb->num_nodes) {
-		event_loop_once(ctdb->ev);
-	}
-
-	/* all daemons have requested to finish - we now exit */
-	DEBUG(1,("All daemons finished - exiting\n"));
-	_exit(0);
-}
-
-
-
-/*
-  called when the daemon gets a connect wait request from a client
- */
-static void daemon_request_connect_wait(struct ctdb_client *client, 
-					struct ctdb_req_connect_wait *c)
-{
-	struct ctdb_reply_connect_wait r;
-	int res;
-
-	/* first wait - in the daemon */
-	ctdb_daemon_connect_wait(client->ctdb);
-
-	/* now send the reply */
-	ZERO_STRUCT(r);
-
-	r.hdr.length     = sizeof(r);
-	r.hdr.ctdb_magic = CTDB_MAGIC;
-	r.hdr.ctdb_version = CTDB_VERSION;
-	r.hdr.operation = CTDB_REPLY_CONNECT_WAIT;
-	r.vnn           = ctdb_get_vnn(client->ctdb);
-	r.num_connected = client->ctdb->num_connected;
-	
-	res = daemon_queue_send(client, &r.hdr);
-	if (res != 0) {
-		DEBUG(0,(__location__ " Failed to queue a connect wait response\n"));
-		return;
-	}
-}
-
-
-/*
-  called when the daemon gets a status request from a client
- */
-static void daemon_request_status(struct ctdb_client *client, 
-				  struct ctdb_req_status *c)
-{
-	struct ctdb_reply_status r;
-	int res;
-
-	/* now send the reply */
-	ZERO_STRUCT(r);
-
-	r.hdr.length     = sizeof(r);
-	r.hdr.ctdb_magic = CTDB_MAGIC;
-	r.hdr.ctdb_version = CTDB_VERSION;
-	r.hdr.operation = CTDB_REPLY_STATUS;
-	r.hdr.reqid = c->hdr.reqid;
-	r.status = client->ctdb->status;
-	
-	res = daemon_queue_send(client, &r.hdr);
-	if (res != 0) {
-		DEBUG(0,(__location__ " Failed to queue a connect wait response\n"));
-		return;
-	}
-}
-
-/*
-  destroy a ctdb_client
-*/
-static int ctdb_client_destructor(struct ctdb_client *client)
-{
-	close(client->fd);
-	client->fd = -1;
-	return 0;
-}
-
-
-/*
-  this is called when the ctdb daemon received a ctdb request message
-  from a local client over the unix domain socket
- */
-static void daemon_request_message_from_client(struct ctdb_client *client, 
-					       struct ctdb_req_message *c)
-{
-	TDB_DATA data;
-	int res;
-
-	/* maybe the message is for another client on this node */
-	if (ctdb_get_vnn(client->ctdb)==c->hdr.destnode) {
-		ctdb_request_message(client->ctdb, (struct ctdb_req_header *)c);
-		return;
-	}
-	
-	/* its for a remote node */
-	data.dptr = &c->data[0];
-	data.dsize = c->datalen;
-	res = ctdb_daemon_send_message(client->ctdb, c->hdr.destnode,
-				       c->srvid, data);
-	if (res != 0) {
-		DEBUG(0,(__location__ " Failed to send message to remote node %u\n",
-			 c->hdr.destnode));
-	}
-}
-
-
-struct daemon_call_state {
-	struct ctdb_client *client;
-	uint32_t reqid;
-	struct ctdb_call *call;
-	struct timeval start_time;
-};
-
-/* 
-   complete a call from a client 
-*/
-static void daemon_call_from_client_callback(struct ctdb_call_state *state)
-{
-	struct daemon_call_state *dstate = talloc_get_type(state->async.private_data, 
-							   struct daemon_call_state);
-	struct ctdb_reply_call *r;
-	int res;
-	uint32_t length;
-	struct ctdb_client *client = dstate->client;
-
-	talloc_steal(client, dstate);
-	talloc_steal(dstate, dstate->call);
-
-	res = ctdb_daemon_call_recv(state, dstate->call);
-	if (res != 0) {
-		DEBUG(0, (__location__ " ctdbd_call_recv() returned error\n"));
-		client->ctdb->status.pending_calls--;
-		ctdb_latency(&client->ctdb->status.max_call_latency, dstate->start_time);
-		return;
-	}
-
-	length = offsetof(struct ctdb_reply_call, data) + dstate->call->reply_data.dsize;
-	r = ctdbd_allocate_pkt(dstate, length);
-	if (r == NULL) {
-		DEBUG(0, (__location__ " Failed to allocate reply_call in ctdb daemon\n"));
-		client->ctdb->status.pending_calls--;
-		ctdb_latency(&client->ctdb->status.max_call_latency, dstate->start_time);
-		return;
-	}
-	memset(r, 0, offsetof(struct ctdb_reply_call, data));
-	r->hdr.length       = length;
-	r->hdr.ctdb_magic   = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation    = CTDB_REPLY_CALL;
-	r->hdr.reqid        = dstate->reqid;
-	r->datalen          = dstate->call->reply_data.dsize;
-	memcpy(&r->data[0], dstate->call->reply_data.dptr, r->datalen);
-
-	res = daemon_queue_send(client, &r->hdr);
-	if (res != 0) {
-		DEBUG(0, (__location__ "Failed to queue packet from daemon to client\n"));
-	}
-	ctdb_latency(&client->ctdb->status.max_call_latency, dstate->start_time);
-	talloc_free(dstate);
-	client->ctdb->status.pending_calls--;
-}
-
-
-/*
-  this is called when the ctdb daemon received a ctdb request call
-  from a local client over the unix domain socket
- */
-static void daemon_request_call_from_client(struct ctdb_client *client, 
-					    struct ctdb_req_call *c)
-{
-	struct ctdb_call_state *state;
-	struct ctdb_db_context *ctdb_db;
-	struct daemon_call_state *dstate;
-	struct ctdb_call *call;
-	struct ctdb_ltdb_header header;
-	TDB_DATA key, data;
-	int ret;
-	struct ctdb_context *ctdb = client->ctdb;
-
-	ctdb->status.total_calls++;
-	ctdb->status.pending_calls++;
-
-	ctdb_db = find_ctdb_db(client->ctdb, c->db_id);
-	if (!ctdb_db) {
-		DEBUG(0, (__location__ " Unknown database in request. db_id==0x%08x",
-			  c->db_id));
-		ctdb->status.pending_calls--;
-		return;
-	}
-
-	key.dptr = c->data;
-	key.dsize = c->keylen;
-
-	ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, key, &header, 
-					   (struct ctdb_req_header *)c, &data,
-					   daemon_incoming_packet, client);
-	if (ret == -2) {
-		/* will retry later */
-		ctdb->status.pending_calls--;
-		return;
-	}
-
-	if (ret != 0) {
-		DEBUG(0,(__location__ " Unable to fetch record\n"));
-		ctdb->status.pending_calls--;
-		return;
-	}
-
-	dstate = talloc(client, struct daemon_call_state);
-	if (dstate == NULL) {
-		ctdb_ltdb_unlock(ctdb_db, key);
-		DEBUG(0,(__location__ " Unable to allocate dstate\n"));
-		ctdb->status.pending_calls--;
-		return;
-	}
-	dstate->start_time = timeval_current();
-	dstate->client = client;
-	dstate->reqid  = c->hdr.reqid;
-	talloc_steal(dstate, data.dptr);
-
-	call = dstate->call = talloc_zero(dstate, struct ctdb_call);
-	if (call == NULL) {
-		ctdb_ltdb_unlock(ctdb_db, key);
-		DEBUG(0,(__location__ " Unable to allocate call\n"));
-		ctdb->status.pending_calls--;
-		ctdb_latency(&ctdb->status.max_call_latency, dstate->start_time);
-		return;
-	}
-
-	call->call_id = c->callid;
-	call->key = key;
-	call->call_data.dptr = c->data + c->keylen;
-	call->call_data.dsize = c->calldatalen;
-	call->flags = c->flags;
-
-	if (header.dmaster == ctdb->vnn && !(ctdb->flags & CTDB_FLAG_SELF_CONNECT)) {
-		state = ctdb_call_local_send(ctdb_db, call, &header, &data);
-	} else {
-		state = ctdb_daemon_call_send_remote(ctdb_db, call, &header);
-	}
-
-	ctdb_ltdb_unlock(ctdb_db, key);
-
-	if (state == NULL) {
-		DEBUG(0,(__location__ " Unable to setup call send\n"));
-		ctdb->status.pending_calls--;
-		ctdb_latency(&ctdb->status.max_call_latency, dstate->start_time);
-		return;
-	}
-	talloc_steal(state, dstate);
-	talloc_steal(client, state);
-
-	state->async.fn = daemon_call_from_client_callback;
-	state->async.private_data = dstate;
-}
-
-/* data contains a packet from the client */
-static void daemon_incoming_packet(void *p, uint8_t *data, uint32_t nread)
-{
-	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
-	struct ctdb_client *client = talloc_get_type(p, struct ctdb_client);
-	TALLOC_CTX *tmp_ctx;
-	struct ctdb_context *ctdb = client->ctdb;
-
-	/* place the packet as a child of a tmp_ctx. We then use
-	   talloc_free() below to free it. If any of the calls want
-	   to keep it, then they will steal it somewhere else, and the
-	   talloc_free() will be a no-op */
-	tmp_ctx = talloc_new(client);
-	talloc_steal(tmp_ctx, hdr);
-
-	if (hdr->ctdb_magic != CTDB_MAGIC) {
-		ctdb_set_error(client->ctdb, "Non CTDB packet rejected in daemon\n");
-		goto done;
-	}
-
-	if (hdr->ctdb_version != CTDB_VERSION) {
-		ctdb_set_error(client->ctdb, "Bad CTDB version 0x%x rejected in daemon\n", hdr->ctdb_version);
-		goto done;
-	}
-
-	switch (hdr->operation) {
-	case CTDB_REQ_CALL:
-		ctdb->status.client.req_call++;
-		daemon_request_call_from_client(client, (struct ctdb_req_call *)hdr);
-		break;
-
-	case CTDB_REQ_REGISTER:
-		ctdb->status.client.req_register++;
-		daemon_request_register_message_handler(client, 
-							(struct ctdb_req_register *)hdr);
-		break;
-	case CTDB_REQ_MESSAGE:
-		ctdb->status.client.req_message++;
-		daemon_request_message_from_client(client, (struct ctdb_req_message *)hdr);
-		break;
-
-	case CTDB_REQ_CONNECT_WAIT:
-		ctdb->status.client.req_connect_wait++;
-		daemon_request_connect_wait(client, (struct ctdb_req_connect_wait *)hdr);
-		break;
-
-	case CTDB_REQ_SHUTDOWN:
-		ctdb->status.client.req_shutdown++;
-		daemon_request_shutdown(client, (struct ctdb_req_shutdown *)hdr);
-		break;
-
-	case CTDB_REQ_STATUS:
-		ctdb->status.client.req_status++;
-		daemon_request_status(client, (struct ctdb_req_status *)hdr);
-		break;
-
-	default:
-		DEBUG(0,(__location__ " daemon: unrecognized operation %d\n",
-			 hdr->operation));
-	}
-
-done:
-	talloc_free(tmp_ctx);
-}
-
-/*
-  called when the daemon gets a incoming packet
- */
-static void ctdb_daemon_read_cb(uint8_t *data, size_t cnt, void *args)
-{
-	struct ctdb_client *client = talloc_get_type(args, struct ctdb_client);
-	struct ctdb_req_header *hdr;
-
-	if (cnt == 0) {
-		talloc_free(client);
-		return;
-	}
-
-	client->ctdb->status.client_packets_recv++;
-
-	if (cnt < sizeof(*hdr)) {
-		ctdb_set_error(client->ctdb, "Bad packet length %d in daemon\n", cnt);
-		return;
-	}
-	hdr = (struct ctdb_req_header *)data;
-	if (cnt != hdr->length) {
-		ctdb_set_error(client->ctdb, "Bad header length %d expected %d\n in daemon", 
-			       hdr->length, cnt);
-		return;
-	}
-
-	if (hdr->ctdb_magic != CTDB_MAGIC) {
-		ctdb_set_error(client->ctdb, "Non CTDB packet rejected\n");
-		return;
-	}
-
-	if (hdr->ctdb_version != CTDB_VERSION) {
-		ctdb_set_error(client->ctdb, "Bad CTDB version 0x%x rejected in daemon\n", hdr->ctdb_version);
-		return;
-	}
-
-	DEBUG(3,(__location__ " client request %d of type %d length %d from "
-		 "node %d to %d\n", hdr->reqid, hdr->operation, hdr->length,
-		 hdr->srcnode, hdr->destnode));
-
-	/* it is the responsibility of the incoming packet function to free 'data' */
-	daemon_incoming_packet(client, data, cnt);
-}
-
-static void ctdb_accept_client(struct event_context *ev, struct fd_event *fde, 
-			 uint16_t flags, void *private_data)
-{
-	struct sockaddr_in addr;
-	socklen_t len;
-	int fd;
-	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
-	struct ctdb_client *client;
-
-	memset(&addr, 0, sizeof(addr));
-	len = sizeof(addr);
-	fd = accept(ctdb->daemon.sd, (struct sockaddr *)&addr, &len);
-	if (fd == -1) {
-		return;
-	}
-	set_non_blocking(fd);
-
-	client = talloc_zero(ctdb, struct ctdb_client);
-	client->ctdb = ctdb;
-	client->fd = fd;
-
-	client->queue = ctdb_queue_setup(ctdb, client, fd, CTDB_DS_ALIGNMENT, 
-					 ctdb_daemon_read_cb, client);
-
-	talloc_set_destructor(client, ctdb_client_destructor);
-}
-
-
-
-static void ctdb_read_from_parent(struct event_context *ev, struct fd_event *fde, 
-			 uint16_t flags, void *private_data)
-{
-	int *fd = private_data;
-	int cnt;
-	char buf;
-
-	/* XXX this is a good place to try doing some cleaning up before exiting */
-	cnt = read(*fd, &buf, 1);
-	if (cnt==0) {
-		DEBUG(2,(__location__ " parent process exited. filedescriptor dissappeared\n"));
-		exit(1);
-	} else {
-		DEBUG(0,(__location__ " ctdb: did not expect data from parent process\n"));
-		exit(1);
-	}
-}
-
-
-
-/*
-  create a unix domain socket and bind it
-  return a file descriptor open on the socket 
-*/
-static int ux_socket_bind(struct ctdb_context *ctdb)
-{
-	struct sockaddr_un addr;
-
-	ctdb->daemon.sd = socket(AF_UNIX, SOCK_STREAM, 0);
-	if (ctdb->daemon.sd == -1) {
-		ctdb->daemon.sd = -1;
-		return -1;
-	}
-
-	set_non_blocking(ctdb->daemon.sd);
-
-	memset(&addr, 0, sizeof(addr));
-	addr.sun_family = AF_UNIX;
-	strncpy(addr.sun_path, ctdb->daemon.name, sizeof(addr.sun_path));
-
-	if (bind(ctdb->daemon.sd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
-		close(ctdb->daemon.sd);
-		ctdb->daemon.sd = -1;
-		return -1;
-	}	
-	listen(ctdb->daemon.sd, 1);
-
-	return 0;
-}
-
-/*
-  delete the socket on exit - called on destruction of autofree context
- */
-static int unlink_destructor(const char *name)
-{
-	unlink(name);
-	return 0;
-}
-
-/*
-  start the protocol going
-*/
-int ctdb_start(struct ctdb_context *ctdb)
-{
-	pid_t pid;
-	static int fd[2];
-	int res;
-	struct fd_event *fde;
-	const char *domain_socket_name;
-
-	/* generate a name to use for our local socket */
-	ctdb->daemon.name = talloc_asprintf(ctdb, "%s.%s", CTDB_PATH, ctdb->address.address);
-	/* get rid of any old sockets */
-	unlink(ctdb->daemon.name);
-
-	/* create a unix domain stream socket to listen to */
-	res = ux_socket_bind(ctdb);
-	if (res!=0) {
-		DEBUG(0,(__location__ " Failed to open CTDB unix domain socket\n"));
-		exit(10);
-	}
-
-	res = pipe(&fd[0]);
-	if (res) {
-		DEBUG(0,(__location__ " Failed to open pipe for CTDB\n"));
-		exit(1);
-	}
-	pid = fork();
-	if (pid==-1) {
-		DEBUG(0,(__location__ " Failed to fork CTDB daemon\n"));
-		exit(1);
-	}
-
-	if (pid) {
-		close(fd[0]);
-		close(ctdb->daemon.sd);
-		ctdb->daemon.sd = -1;
-		return 0;
-	}
-
-	block_signal(SIGPIPE);
-
-	/* ensure the socket is deleted on exit of the daemon */
-	domain_socket_name = talloc_strdup(talloc_autofree_context(), ctdb->daemon.name);
-	talloc_set_destructor(domain_socket_name, unlink_destructor);	
-	
-	close(fd[1]);
-
-	ctdb->ev = event_context_init(NULL);
-	fde = event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ, ctdb_read_from_parent, &fd[0]);
-	fde = event_add_fd(ctdb->ev, ctdb, ctdb->daemon.sd, EVENT_FD_READ, ctdb_accept_client, ctdb);
-	ctdb_main_loop(ctdb);
-
-	return 0;
-}
-
-/*
-  allocate a packet for use in client<->daemon communication
- */
-void *ctdbd_allocate_pkt(TALLOC_CTX *mem_ctx, size_t len)
-{
-	int size;
-
-	size = (len+(CTDB_DS_ALIGNMENT-1)) & ~(CTDB_DS_ALIGNMENT-1);
-	return talloc_size(mem_ctx, size);
-}
-
-/*
-  called when a CTDB_REQ_FINISHED packet comes in
-*/
-void ctdb_request_finished(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
-{
-	ctdb->num_finished++;
-}
diff --git a/source4/cluster/ctdb/common/ctdb_io.c b/source4/cluster/ctdb/common/ctdb_io.c
index 517fbbd842..3cc522b58a 100644
--- a/source4/cluster/ctdb/common/ctdb_io.c
+++ b/source4/cluster/ctdb/common/ctdb_io.c
@@ -6,18 +6,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -39,6 +39,7 @@ struct ctdb_queue_pkt {
 	struct ctdb_queue_pkt *next, *prev;
 	uint8_t *data;
 	uint32_t length;
+	uint32_t full_length;
 };
 
 struct ctdb_queue {
@@ -63,8 +64,10 @@ static void queue_io_read(struct ctdb_queue *queue)
 	ssize_t nread;
 	uint8_t *data, *data_base;
 
-	if (ioctl(queue->fd, FIONREAD, &num_ready) != 0 ||
-	    num_ready == 0) {
+	if (ioctl(queue->fd, FIONREAD, &num_ready) != 0) {
+		return;
+	}
+	if (num_ready == 0) {
 		/* the descriptor has been closed */
 		goto failed;
 	}
@@ -74,11 +77,14 @@ static void queue_io_read(struct ctdb_queue *queue)
 						  num_ready + queue->partial.length);
 
 	if (queue->partial.data == NULL) {
+		DEBUG(0,("read error alloc failed for %u\n", 
+			 num_ready + queue->partial.length));
 		goto failed;
 	}
 
 	nread = read(queue->fd, queue->partial.data + queue->partial.length, num_ready);
 	if (nread <= 0) {
+		DEBUG(0,("read error nread=%d\n", (int)nread));
 		goto failed;
 	}
 
@@ -103,8 +109,14 @@ static void queue_io_read(struct ctdb_queue *queue)
 		uint8_t *d2;
 		uint32_t len;
 		len = *(uint32_t *)data;
+		if (len == 0) {
+			/* bad packet! treat as EOF */
+			DEBUG(0,("Invalid packet of length 0\n"));
+			goto failed;
+		}
 		d2 = talloc_memdup(queue, data, len);
 		if (d2 == NULL) {
+			DEBUG(0,("read error memdup failed for %u\n", len));
 			/* sigh */
 			goto failed;
 		}
@@ -121,6 +133,8 @@ static void queue_io_read(struct ctdb_queue *queue)
 		} else {
 			queue->partial.data = talloc_memdup(queue, data, nread);
 			if (queue->partial.data == NULL) {
+				DEBUG(0,("read error memdup partial failed for %u\n", 
+					 (unsigned)nread));
 				goto failed;
 			}
 			queue->partial.length = nread;
@@ -154,13 +168,23 @@ static void queue_io_write(struct ctdb_queue *queue)
 	while (queue->out_queue) {
 		struct ctdb_queue_pkt *pkt = queue->out_queue;
 		ssize_t n;
-
-		n = write(queue->fd, pkt->data, pkt->length);
+		if (queue->ctdb->flags & CTDB_FLAG_TORTURE) {
+			n = write(queue->fd, pkt->data, 1);
+		} else {
+			n = write(queue->fd, pkt->data, pkt->length);
+		}
 
 		if (n == -1 && errno != EAGAIN && errno != EWOULDBLOCK) {
+			if (pkt->length != pkt->full_length) {
+				/* partial packet sent - we have to drop it */
+				DLIST_REMOVE(queue->out_queue, pkt);
+				talloc_free(pkt);
+			}
+			talloc_free(queue->fde);
+			queue->fde = NULL;
+			queue->fd = -1;
 			event_add_timed(queue->ctdb->ev, queue, timeval_zero(), 
 					queue_dead, queue);
-			EVENT_FD_NOT_WRITEABLE(queue->fde);
 			return;
 		}
 		if (n <= 0) return;
@@ -200,21 +224,31 @@ static void queue_io_handler(struct event_context *ev, struct fd_event *fde,
 int ctdb_queue_send(struct ctdb_queue *queue, uint8_t *data, uint32_t length)
 {
 	struct ctdb_queue_pkt *pkt;
-	uint32_t length2;
+	uint32_t length2, full_length;
 
-	/* enforce the length and alignment rules from the tcp packet allocator */
-	length2 = (length+(queue->alignment-1)) & ~(queue->alignment-1);
-	*(uint32_t *)data = length2;
+	if (queue->alignment) {
+		/* enforce the length and alignment rules from the tcp packet allocator */
+		length2 = (length+(queue->alignment-1)) & ~(queue->alignment-1);
+		*(uint32_t *)data = length2;
+	} else {
+		length2 = length;
+	}
 
 	if (length2 != length) {
 		memset(data+length, 0, length2-length);
 	}
+
+	full_length = length2;
 	
 	/* if the queue is empty then try an immediate write, avoiding
 	   queue overhead. This relies on non-blocking sockets */
-	if (queue->out_queue == NULL && queue->fd != -1) {
+	if (queue->out_queue == NULL && queue->fd != -1 &&
+	    !(queue->ctdb->flags & CTDB_FLAG_TORTURE)) {
 		ssize_t n = write(queue->fd, data, length2);
 		if (n == -1 && errno != EAGAIN && errno != EWOULDBLOCK) {
+			talloc_free(queue->fde);
+			queue->fde = NULL;
+			queue->fd = -1;
 			event_add_timed(queue->ctdb->ev, queue, timeval_zero(), 
 					queue_dead, queue);
 			/* yes, we report success, as the dead node is 
@@ -235,6 +269,7 @@ int ctdb_queue_send(struct ctdb_queue *queue, uint8_t *data, uint32_t length)
 	CTDB_NO_MEMORY(queue->ctdb, pkt->data);
 
 	pkt->length = length2;
+	pkt->full_length = full_length;
 
 	if (queue->out_queue == NULL && queue->fd != -1) {
 		EVENT_FD_WRITEABLE(queue->fde);
@@ -256,7 +291,7 @@ int ctdb_queue_set_fd(struct ctdb_queue *queue, int fd)
 	queue->fde = NULL;
 
 	if (fd != -1) {
-		queue->fde = event_add_fd(queue->ctdb->ev, queue, fd, EVENT_FD_READ, 
+		queue->fde = event_add_fd(queue->ctdb->ev, queue, fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
 					  queue_io_handler, queue);
 		if (queue->fde == NULL) {
 			return -1;
diff --git a/source4/cluster/ctdb/common/ctdb_ltdb.c b/source4/cluster/ctdb/common/ctdb_ltdb.c
index feb7207fcc..7db1523ca1 100644
--- a/source4/cluster/ctdb/common/ctdb_ltdb.c
+++ b/source4/cluster/ctdb/common/ctdb_ltdb.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -42,87 +42,16 @@ struct ctdb_db_context *ctdb_db_handle(struct ctdb_context *ctdb, const char *na
 
 
 /*
-  this is the dummy null procedure that all databases support
-*/
-static int ctdb_null_func(struct ctdb_call_info *call)
-{
-	return 0;
-}
-
-
-/*
-  attach to a specific database
-*/
-struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name, int tdb_flags, 
-				    int open_flags, mode_t mode)
-{
-	struct ctdb_db_context *ctdb_db, *tmp_db;
-	TDB_DATA data;
-	int ret;
-
-	ctdb_db = talloc_zero(ctdb, struct ctdb_db_context);
-	CTDB_NO_MEMORY_NULL(ctdb, ctdb_db);
-
-	ctdb_db->ctdb = ctdb;
-	ctdb_db->db_name = talloc_strdup(ctdb_db, name);
-	CTDB_NO_MEMORY_NULL(ctdb, ctdb_db->db_name);
-
-	data.dptr = discard_const(name);
-	data.dsize = strlen(name);
-	ctdb_db->db_id = ctdb_hash(&data);
-
-	for (tmp_db=ctdb->db_list;tmp_db;tmp_db=tmp_db->next) {
-		if (tmp_db->db_id == ctdb_db->db_id) {
-			ctdb_set_error(ctdb, "CTDB database hash collission '%s' : '%s'",
-					name, tmp_db->db_name);
-			talloc_free(ctdb_db);
-			return NULL;
-		}
-	}
-
-	if (mkdir(ctdb->db_directory, 0700) == -1 && errno != EEXIST) {
-		DEBUG(0,(__location__ " Unable to create ctdb directory '%s'\n", 
-			 ctdb->db_directory));
-		talloc_free(ctdb_db);
-		return NULL;
-	}
-
-	/* add the node id to the database name, so when we run on loopback
-	   we don't conflict in the local filesystem */
-	name = talloc_asprintf(ctdb_db, "%s/%s", ctdb->db_directory, name);
-
-	/* when we have a separate daemon this will need to be a real
-	   file, not a TDB_INTERNAL, so the parent can access it to
-	   for ltdb bypass */
-	ctdb_db->ltdb = tdb_wrap_open(ctdb, name, 0, TDB_CLEAR_IF_FIRST, open_flags, mode);
-	if (ctdb_db->ltdb == NULL) {
-		ctdb_set_error(ctdb, "Failed to open tdb %s\n", name);
-		talloc_free(ctdb_db);
-		return NULL;
-	}
-
-
-	/* 
-	   all databases support the "null" function. we need this in
-	   order to do forced migration of records
-	 */
-	ret = ctdb_set_call(ctdb_db, ctdb_null_func, CTDB_NULL_FUNC);
-	if (ret != 0) {
-		talloc_free(ctdb_db);
-		return NULL;
-	}
-
-	DLIST_ADD(ctdb->db_list, ctdb_db);
-
-	return ctdb_db;
-}
-
-/*
   return the lmaster given a key
 */
 uint32_t ctdb_lmaster(struct ctdb_context *ctdb, const TDB_DATA *key)
 {
-	return ctdb_hash(key) % ctdb->num_nodes;
+	uint32_t idx, lmaster;
+
+	idx = ctdb_hash(key) % ctdb->vnn_map->size;
+	lmaster = ctdb->vnn_map->map[idx];
+
+	return lmaster;
 }
 
 
@@ -158,6 +87,12 @@ int ctdb_ltdb_fetch(struct ctdb_db_context *ctdb_db,
 		TDB_DATA d2;
 		/* return an initial header */
 		if (rec.dptr) free(rec.dptr);
+		if (ctdb->vnn_map == NULL) {
+			/* called from the client */
+			ZERO_STRUCTP(data);
+			header->dmaster = (uint32_t)-1;
+			return -1;
+		}
 		ltdb_initial_header(ctdb_db, key, header);
 		ZERO_STRUCT(d2);
 		if (data) {
@@ -197,6 +132,17 @@ int ctdb_ltdb_store(struct ctdb_db_context *ctdb_db, TDB_DATA key,
 	TDB_DATA rec;
 	int ret;
 
+	if (ctdb->flags & CTDB_FLAG_TORTURE) {
+		struct ctdb_ltdb_header *h2;
+		rec = tdb_fetch(ctdb_db->ltdb->tdb, key);
+		h2 = (struct ctdb_ltdb_header *)rec.dptr;
+		if (rec.dptr && rec.dsize >= sizeof(h2) && h2->rsn > header->rsn) {
+			DEBUG(0,("RSN regression! %llu %llu\n",
+				 (unsigned long long)h2->rsn, (unsigned long long)header->rsn));
+		}
+		if (rec.dptr) free(rec.dptr);
+	}
+
 	rec.dsize = sizeof(*header) + data.dsize;
 	rec.dptr = talloc_size(ctdb, rec.dsize);
 	CTDB_NO_MEMORY(ctdb, rec.dptr);
@@ -224,121 +170,9 @@ int ctdb_ltdb_lock(struct ctdb_db_context *ctdb_db, TDB_DATA key)
  */
 int ctdb_ltdb_unlock(struct ctdb_db_context *ctdb_db, TDB_DATA key)
 {
-	return tdb_chainunlock(ctdb_db->ltdb->tdb, key);
-}
-
-struct lock_fetch_state {
-	struct ctdb_context *ctdb;
-	void (*recv_pkt)(void *, uint8_t *, uint32_t);
-	void *recv_context;
-	struct ctdb_req_header *hdr;
-};
-
-/*
-  called when we should retry the operation
- */
-static void lock_fetch_callback(void *p)
-{
-	struct lock_fetch_state *state = talloc_get_type(p, struct lock_fetch_state);
-	state->recv_pkt(state->recv_context, (uint8_t *)state->hdr, state->hdr->length);
-	talloc_free(state);
-	DEBUG(2,(__location__ " PACKET REQUEUED\n"));
-}
-
-
-/*
-  do a non-blocking ltdb_lock, deferring this ctdb request until we
-  have the chainlock
-
-  It does the following:
-
-   1) tries to get the chainlock. If it succeeds, then it returns 0
-
-   2) if it fails to get a chainlock immediately then it sets up a
-   non-blocking chainlock via ctdb_lockwait, and when it gets the
-   chainlock it re-submits this ctdb request to the main packet
-   receive function
-
-   This effectively queues all ctdb requests that cannot be
-   immediately satisfied until it can get the lock. This means that
-   the main ctdb daemon will not block waiting for a chainlock held by
-   a client
-
-   There are 3 possible return values:
-
-       0:    means that it got the lock immediately.
-      -1:    means that it failed to get the lock, and won't retry
-      -2:    means that it failed to get the lock immediately, but will retry
- */
-int ctdb_ltdb_lock_requeue(struct ctdb_db_context *ctdb_db, 
-			   TDB_DATA key, struct ctdb_req_header *hdr,
-			   void (*recv_pkt)(void *, uint8_t *, uint32_t ),
-			   void *recv_context)
-{
-	int ret;
-	struct tdb_context *tdb = ctdb_db->ltdb->tdb;
-	struct lockwait_handle *h;
-	struct lock_fetch_state *state;
-	
-	ret = tdb_chainlock_nonblock(tdb, key);
-
-	if (ret != 0 &&
-	    !(errno == EACCES || errno == EAGAIN || errno == EDEADLK)) {
-		/* a hard failure - don't try again */
-		return -1;
-	}
-
-	/* when torturing, ensure we test the contended path */
-	if ((ctdb_db->ctdb->flags & CTDB_FLAG_TORTURE) &&
-	    random() % 5 == 0) {
-		ret = -1;
-		tdb_chainunlock(tdb, key);
-	}
-
-	/* first the non-contended path */
-	if (ret == 0) {
-		return 0;
-	}
-
-	state = talloc(ctdb_db, struct lock_fetch_state);
-	state->ctdb = ctdb_db->ctdb;
-	state->hdr = hdr;
-	state->recv_pkt = recv_pkt;
-	state->recv_context = recv_context;
-
-	/* now the contended path */
-	h = ctdb_lockwait(ctdb_db, key, lock_fetch_callback, state);
-	if (h == NULL) {
-		tdb_chainunlock(tdb, key);
-		return -1;
-	}
-
-	/* we need to move the packet off the temporary context in ctdb_recv_pkt(),
-	   so it won't be freed yet */
-	talloc_steal(state, hdr);
-	talloc_steal(state, h);
-
-	/* now tell the caller than we will retry asynchronously */
-	return -2;
-}
-
-/*
-  a varient of ctdb_ltdb_lock_requeue that also fetches the record
- */
-int ctdb_ltdb_lock_fetch_requeue(struct ctdb_db_context *ctdb_db, 
-				 TDB_DATA key, struct ctdb_ltdb_header *header, 
-				 struct ctdb_req_header *hdr, TDB_DATA *data,
-				 void (*recv_pkt)(void *, uint8_t *, uint32_t ),
-				 void *recv_context)
-{
-	int ret;
-
-	ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr, recv_pkt, recv_context);
-	if (ret == 0) {
-		ret = ctdb_ltdb_fetch(ctdb_db, key, header, hdr, data);
-		if (ret != 0) {
-			ctdb_ltdb_unlock(ctdb_db, key);
-		}
+	int ret = tdb_chainunlock(ctdb_db->ltdb->tdb, key);
+	if (ret != 0) {
+		DEBUG(0,("tdb_chainunlock failed\n"));
 	}
 	return ret;
 }
diff --git a/source4/cluster/ctdb/common/ctdb_message.c b/source4/cluster/ctdb/common/ctdb_message.c
index a477cc59b6..1aea28fd35 100644
--- a/source4/cluster/ctdb/common/ctdb_message.c
+++ b/source4/cluster/ctdb/common/ctdb_message.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2007
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 /*
   see http://wiki.samba.org/index.php/Samba_%26_Clustering for
@@ -31,27 +31,19 @@
 /*
   this dispatches the messages to the registered ctdb message handler
 */
-static int ctdb_dispatch_message(struct ctdb_context *ctdb, uint32_t srvid, TDB_DATA data)
+int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA data)
 {
 	struct ctdb_message_list *ml;
 
-	/* XXX we need a must faster way of finding the matching srvid
-	   - maybe a tree? */
 	for (ml=ctdb->message_list;ml;ml=ml->next) {
-		if (ml->srvid == srvid || ml->srvid == CTDB_SRVID_ALL) break;
-	}
-	if (ml == NULL) {
-		DEBUG(1,(__location__ " daemon vnn:%d  no msg handler for srvid=%u\n", 
-			 ctdb_get_vnn(ctdb), srvid));
-		/* no registered message handler */
-		return -1;
+		if (ml->srvid == srvid || ml->srvid == CTDB_SRVID_ALL) {
+			ml->message_handler(ctdb, srvid, data, ml->message_private);
+		}
 	}
 
-	ml->message_handler(ctdb, srvid, data, ml->message_private);
 	return 0;
 }
 
-
 /*
   called when a CTDB_REQ_MESSAGE packet comes in
 */
@@ -66,87 +58,6 @@ void ctdb_request_message(struct ctdb_context *ctdb, struct ctdb_req_header *hdr
 	ctdb_dispatch_message(ctdb, c->srvid, data);
 }
 
-/*
-  this local messaging handler is ugly, but is needed to prevent
-  recursion in ctdb_send_message() when the destination node is the
-  same as the source node
- */
-struct ctdb_local_message {
-	struct ctdb_context *ctdb;
-	uint32_t srvid;
-	TDB_DATA data;
-};
-
-static void ctdb_local_message_trigger(struct event_context *ev, struct timed_event *te, 
-				       struct timeval t, void *private_data)
-{
-	struct ctdb_local_message *m = talloc_get_type(private_data, 
-						       struct ctdb_local_message);
-	int res;
-
-	res = ctdb_dispatch_message(m->ctdb, m->srvid, m->data);
-	if (res != 0) {
-		DEBUG(0, (__location__ " Failed to dispatch message for srvid=%u\n", m->srvid));
-	}
-	talloc_free(m);
-}
-
-static int ctdb_local_message(struct ctdb_context *ctdb, uint32_t srvid, TDB_DATA data)
-{
-	struct ctdb_local_message *m;
-	m = talloc(ctdb, struct ctdb_local_message);
-	CTDB_NO_MEMORY(ctdb, m);
-
-	m->ctdb = ctdb;
-	m->srvid = srvid;
-	m->data  = data;
-	m->data.dptr = talloc_memdup(m, m->data.dptr, m->data.dsize);
-	if (m->data.dptr == NULL) {
-		talloc_free(m);
-		return -1;
-	}
-
-	/* this needs to be done as an event to prevent recursion */
-	event_add_timed(ctdb->ev, m, timeval_zero(), ctdb_local_message_trigger, m);
-	return 0;
-}
-
-/*
-  send a ctdb message
-*/
-int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t vnn,
-			     uint32_t srvid, TDB_DATA data)
-{
-	struct ctdb_req_message *r;
-	int len;
-
-	/* see if this is a message to ourselves */
-	if (vnn == ctdb->vnn && !(ctdb->flags & CTDB_FLAG_SELF_CONNECT)) {
-		return ctdb_local_message(ctdb, srvid, data);
-	}
-
-	len = offsetof(struct ctdb_req_message, data) + data.dsize;
-	r = ctdb->methods->allocate_pkt(ctdb, len);
-	CTDB_NO_MEMORY(ctdb, r);
-	talloc_set_name_const(r, "req_message packet");
-
-	r->hdr.length    = len;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REQ_MESSAGE;
-	r->hdr.destnode  = vnn;
-	r->hdr.srcnode   = ctdb->vnn;
-	r->hdr.reqid     = 0;
-	r->srvid         = srvid;
-	r->datalen       = data.dsize;
-	memcpy(&r->data[0], data.dptr, data.dsize);
-	
-	ctdb_queue_packet(ctdb, &r->hdr);
-
-	talloc_free(r);
-	return 0;
-}
-
 
 /*
   when a client goes away, we need to remove its srvid handler from the list
@@ -162,7 +73,7 @@ static int message_handler_destructor(struct ctdb_message_list *m)
 */
 int ctdb_register_message_handler(struct ctdb_context *ctdb, 
 				  TALLOC_CTX *mem_ctx,
-				  uint32_t srvid,
+				  uint64_t srvid,
 				  ctdb_message_fn_t handler,
 				  void *private_data)
 {
@@ -182,3 +93,20 @@ int ctdb_register_message_handler(struct ctdb_context *ctdb,
 
 	return 0;
 }
+
+
+/*
+  setup handler for receipt of ctdb messages from ctdb_send_message()
+*/
+int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data)
+{
+	struct ctdb_message_list *m;
+
+	for (m=ctdb->message_list;m;m=m->next) {
+		if (m->srvid == srvid && m->message_private == private_data) {
+			talloc_free(m);
+			return 0;
+		}
+	}
+	return -1;
+}
diff --git a/source4/cluster/ctdb/common/ctdb_util.c b/source4/cluster/ctdb/common/ctdb_util.c
index 1448e3340a..f8f7cb5150 100644
--- a/source4/cluster/ctdb/common/ctdb_util.c
+++ b/source4/cluster/ctdb/common/ctdb_util.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -51,10 +51,9 @@ void ctdb_set_error(struct ctdb_context *ctdb, const char *fmt, ...)
 /*
   a fatal internal error occurred - no hope for recovery
 */
-_NORETURN_ void ctdb_fatal(struct ctdb_context *ctdb, const char *msg)
+void ctdb_fatal(struct ctdb_context *ctdb, const char *msg)
 {
 	DEBUG(0,("ctdb fatal error: %s\n", msg));
-	fprintf(stderr, "ctdb fatal error: '%s'\n", msg);
 	abort();
 }
 
@@ -65,15 +64,18 @@ int ctdb_parse_address(struct ctdb_context *ctdb,
 		       TALLOC_CTX *mem_ctx, const char *str,
 		       struct ctdb_address *address)
 {
-	char *p;
-	p = strchr(str, ':');
-	if (p == NULL) {
-		ctdb_set_error(ctdb, "Badly formed node '%s'\n", str);
-		return -1;
-	}
+	struct servent *se;
+
+	setservent(0);
+	se = getservbyname("ctdb", "tcp");
+	endservent();
 	
-	address->address = talloc_strndup(mem_ctx, str, p-str);
-	address->port = strtoul(p+1, NULL, 0);
+	address->address = talloc_strdup(mem_ctx, str);
+	if (se == NULL) {
+		address->port = CTDB_PORT;
+	} else {
+		address->port = ntohs(se->s_port);
+	}
 	return 0;
 }
 
@@ -105,7 +107,7 @@ uint32_t ctdb_hash(const TDB_DATA *key)
 /*
   a type checking varient of idr_find
  */
-void *_idr_find_type(struct idr_context *idp, int id, const char *type, const char *location)
+static void *_idr_find_type(struct idr_context *idp, int id, const char *type, const char *location)
 {
 	void *p = idr_find(idp, id);
 	if (p && talloc_check_name(p, type) == NULL) {
@@ -127,3 +129,135 @@ void ctdb_latency(double *latency, struct timeval t)
 		*latency = l;
 	}
 }
+
+uint32_t ctdb_reqid_new(struct ctdb_context *ctdb, void *state)
+{
+	uint32_t id;
+
+	id  = ctdb->idr_cnt++ & 0xFFFF;
+	id |= (idr_get_new(ctdb->idr, state, 0xFFFF)<<16);
+	return id;
+}
+
+void *_ctdb_reqid_find(struct ctdb_context *ctdb, uint32_t reqid, const char *type, const char *location)
+{
+	void *p;
+
+	p = _idr_find_type(ctdb->idr, (reqid>>16)&0xFFFF, type, location);
+	if (p == NULL) {
+		DEBUG(0, ("Could not find idr:%u\n",reqid));
+	}
+
+	return p;
+}
+
+
+void ctdb_reqid_remove(struct ctdb_context *ctdb, uint32_t reqid)
+{
+	int ret;
+
+	ret = idr_remove(ctdb->idr, (reqid>>16)&0xFFFF);
+	if (ret != 0) {
+		DEBUG(0, ("Removing idr that does not exist\n"));
+	}
+}
+
+
+/*
+  form a ctdb_rec_data record from a key/data pair
+ */
+struct ctdb_rec_data *ctdb_marshall_record(TALLOC_CTX *mem_ctx, uint32_t reqid,	TDB_DATA key, TDB_DATA data)
+{
+	size_t length;
+	struct ctdb_rec_data *d;
+
+	length = offsetof(struct ctdb_rec_data, data) + key.dsize + data.dsize;
+	d = (struct ctdb_rec_data *)talloc_size(mem_ctx, length);
+	if (d == NULL) {
+		return NULL;
+	}
+	d->length = length;
+	d->reqid = reqid;
+	d->keylen = key.dsize;
+	d->datalen = data.dsize;
+	memcpy(&d->data[0], key.dptr, key.dsize);
+	memcpy(&d->data[key.dsize], data.dptr, data.dsize);
+	return d;
+}
+
+#if HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+/*
+  if possible, make this task real time
+ */
+void ctdb_set_realtime(bool enable)
+{
+#if HAVE_SCHED_SETSCHEDULER
+	struct sched_param p;
+	p.__sched_priority = 1;
+
+	if (enable) {
+		if (sched_setscheduler(getpid(), SCHED_FIFO, &p) == -1) {
+			DEBUG(0,("Unable to set scheduler to SCHED_FIFO (%s)\n", strerror(errno)));
+		} else {
+			DEBUG(0,("Set scheduler to SCHED_FIFO\n"));
+		}
+	} else {
+		sched_setscheduler(getpid(), SCHED_OTHER, &p);
+	}
+#endif
+}
+
+void set_nonblocking(int fd)
+{
+	unsigned v;
+	v = fcntl(fd, F_GETFL, 0);
+        fcntl(fd, F_SETFL, v | O_NONBLOCK);
+}
+
+void set_close_on_exec(int fd)
+{
+	unsigned v;
+	v = fcntl(fd, F_GETFD, 0);
+        fcntl(fd, F_SETFD, v | FD_CLOEXEC);
+}
+
+
+/*
+  parse a ip:port pair
+ */
+bool parse_ip_port(const char *s, struct sockaddr_in *ip)
+{
+	const char *p;
+	char *endp = NULL;
+	unsigned port;
+	char buf[16];
+
+	ip->sin_family = AF_INET;
+
+	p = strchr(s, ':');
+	if (p == NULL) {
+		return false;
+	}
+
+	if (p - s > 15) {
+		return false;
+	}
+
+	port = strtoul(p+1, &endp, 10);
+	if (endp == NULL || *endp != 0) {
+		/* trailing garbage */
+		return false;
+	}
+	ip->sin_port = htons(port);
+
+	strlcpy(buf, s, 1+p-s);
+
+	if (inet_aton(buf, &ip->sin_addr) == 0) {
+		return false;
+	}
+
+	return true;
+}
diff --git a/source4/cluster/ctdb/common/system.c b/source4/cluster/ctdb/common/system.c
new file mode 100644
index 0000000000..1e536f5e8a
--- /dev/null
+++ b/source4/cluster/ctdb/common/system.c
@@ -0,0 +1,385 @@
+/* 
+   ctdb recovery code
+
+   Copyright (C) Ronnie Sahlberg  2007
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+#include "lib/events/events.h"
+#include <net/ethernet.h>
+#include <net/if_arp.h>
+
+
+
+/*
+  send gratuitous arp reply after we have taken over an ip address
+
+  saddr is the address we are trying to claim
+  iface is the interface name we will be using to claim the address
+ */
+int ctdb_sys_send_arp(const struct sockaddr_in *saddr, const char *iface)
+{
+	int s, ret;
+	struct sockaddr sa;
+	struct ether_header *eh;
+	struct arphdr *ah;
+	struct ifreq if_hwaddr;
+	unsigned char buffer[64]; /*minimum eth frame size */
+	char *ptr;
+
+	/* for now, we only handle AF_INET addresses */
+	if (saddr->sin_family != AF_INET) {
+		DEBUG(0,(__location__ " not an ipv4 address (family is %u)\n", saddr->sin_family));
+		return -1;
+	}
+
+	s = socket(AF_INET, SOCK_PACKET, htons(ETHERTYPE_ARP));
+	if (s == -1){
+		DEBUG(0,(__location__ " failed to open raw socket\n"));
+		return -1;
+	}
+
+	/* get the mac address */
+	strcpy(if_hwaddr.ifr_name, iface);
+	ret = ioctl(s, SIOCGIFHWADDR, &if_hwaddr);
+	if ( ret < 0 ) {
+		close(s);
+		DEBUG(0,(__location__ " ioctl failed\n"));
+		return -1;
+	}
+	if (ARPHRD_LOOPBACK == if_hwaddr.ifr_hwaddr.sa_family) {
+		DEBUG(3,("Ignoring loopback arp request\n"));
+		close(s);
+		return 0;
+	}
+	if (if_hwaddr.ifr_hwaddr.sa_family != AF_LOCAL) {
+		close(s);
+		errno = EINVAL;
+		DEBUG(0,(__location__ " not an ethernet address family (0x%x)\n",
+			 if_hwaddr.ifr_hwaddr.sa_family));
+		return -1;
+	}
+
+
+	memset(buffer, 0 , 64);
+	eh = (struct ether_header *)buffer;
+	memset(eh->ether_dhost, 0xff, ETH_ALEN);
+	memcpy(eh->ether_shost, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	eh->ether_type = htons(ETHERTYPE_ARP);
+
+	ah = (struct arphdr *)&buffer[sizeof(struct ether_header)];
+	ah->ar_hrd = htons(ARPHRD_ETHER);
+	ah->ar_pro = htons(ETH_P_IP);
+	ah->ar_hln = ETH_ALEN;
+	ah->ar_pln = 4;
+
+	/* send a gratious arp */
+	ah->ar_op  = htons(ARPOP_REQUEST);
+	ptr = (char *)&ah[1];
+	memcpy(ptr, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+	memset(ptr, 0, ETH_ALEN); 
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+
+	strncpy(sa.sa_data, iface, sizeof(sa.sa_data));
+	ret = sendto(s, buffer, 64, 0, &sa, sizeof(sa));
+	if (ret < 0 ){
+		close(s);
+		DEBUG(0,(__location__ " failed sendto\n"));
+		return -1;
+	}
+
+	/* send unsolicited arp reply broadcast */
+	ah->ar_op  = htons(ARPOP_REPLY);
+	ptr = (char *)&ah[1];
+	memcpy(ptr, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+	memcpy(ptr, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+
+	strncpy(sa.sa_data, iface, sizeof(sa.sa_data));
+	ret = sendto(s, buffer, 64, 0, &sa, sizeof(sa));
+	if (ret < 0 ){
+		DEBUG(0,(__location__ " failed sendto\n"));
+		return -1;
+	}
+
+	close(s);
+	return 0;
+}
+
+
+/*
+  uint16 checksum for n bytes
+ */
+static uint32_t uint16_checksum(uint16_t *data, size_t n)
+{
+	uint32_t sum=0;
+	while (n>=2) {
+		sum += (uint32_t)ntohs(*data);
+		data++;
+		n -= 2;
+	}
+	if (n == 1) {
+		sum += (uint32_t)ntohs(*(uint8_t *)data);
+	}
+	return sum;
+}
+
+/*
+  simple TCP checksum - assumes data is multiple of 2 bytes long
+ */
+static uint16_t tcp_checksum(uint16_t *data, size_t n, struct iphdr *ip)
+{
+	uint32_t sum = uint16_checksum(data, n);
+	uint16_t sum2;
+	sum += uint16_checksum((uint16_t *)&ip->saddr, sizeof(ip->saddr));
+	sum += uint16_checksum((uint16_t *)&ip->daddr, sizeof(ip->daddr));
+	sum += ip->protocol + n;
+	sum = (sum & 0xFFFF) + (sum >> 16);
+	sum = (sum & 0xFFFF) + (sum >> 16);
+	sum2 = htons(sum);
+	sum2 = ~sum2;
+	if (sum2 == 0) {
+		return 0xFFFF;
+	}
+	return sum2;
+}
+
+/*
+  Send tcp segment from the specified IP/port to the specified
+  destination IP/port. 
+
+  This is used to trigger the receiving host into sending its own ACK,
+  which should trigger early detection of TCP reset by the client
+  after IP takeover
+
+  This can also be used to send RST segments (if rst is true) and also
+  if correct seq and ack numbers are provided.
+ */
+int ctdb_sys_send_tcp(const struct sockaddr_in *dest, 
+		      const struct sockaddr_in *src,
+		      uint32_t seq, uint32_t ack, int rst)
+{
+	int s, ret;
+	uint32_t one = 1;
+	struct {
+		struct iphdr ip;
+		struct tcphdr tcp;
+	} pkt;
+
+	/* for now, we only handle AF_INET addresses */
+	if (src->sin_family != AF_INET || dest->sin_family != AF_INET) {
+		DEBUG(0,(__location__ " not an ipv4 address\n"));
+		return -1;
+	}
+
+	s = socket(AF_INET, SOCK_RAW, htons(IPPROTO_RAW));
+	if (s == -1) {
+		DEBUG(0,(__location__ " failed to open raw socket (%s)\n",
+			 strerror(errno)));
+		return -1;
+	}
+
+	ret = setsockopt(s, SOL_IP, IP_HDRINCL, &one, sizeof(one));
+	if (ret != 0) {
+		DEBUG(0,(__location__ " failed to setup IP headers (%s)\n",
+			 strerror(errno)));
+		close(s);
+		return -1;
+	}
+
+	ZERO_STRUCT(pkt);
+	pkt.ip.version  = 4;
+	pkt.ip.ihl      = sizeof(pkt.ip)/4;
+	pkt.ip.tot_len  = htons(sizeof(pkt));
+	pkt.ip.ttl      = 255;
+	pkt.ip.protocol = IPPROTO_TCP;
+	pkt.ip.saddr    = src->sin_addr.s_addr;
+	pkt.ip.daddr    = dest->sin_addr.s_addr;
+	pkt.ip.check    = 0;
+
+	pkt.tcp.source   = src->sin_port;
+	pkt.tcp.dest     = dest->sin_port;
+	pkt.tcp.seq      = seq;
+	pkt.tcp.ack_seq  = ack;
+	pkt.tcp.ack      = 1;
+	if (rst) {
+		pkt.tcp.rst      = 1;
+	}
+	pkt.tcp.doff     = sizeof(pkt.tcp)/4;
+	pkt.tcp.window   = htons(1234);
+	pkt.tcp.check    = tcp_checksum((uint16_t *)&pkt.tcp, sizeof(pkt.tcp), &pkt.ip);
+
+	ret = sendto(s, &pkt, sizeof(pkt), 0, dest, sizeof(*dest));
+	if (ret != sizeof(pkt)) {
+		DEBUG(0,(__location__ " failed sendto (%s)\n", strerror(errno)));
+		close(s);
+		return -1;
+	}
+
+	close(s);
+	return 0;
+}
+
+
+/*
+  see if we currently have an interface with the given IP
+
+  we try to bind to it, and if that fails then we don't have that IP
+  on an interface
+ */
+bool ctdb_sys_have_ip(const char *ip)
+{
+	struct sockaddr_in sin;
+	int s;
+	int ret;
+
+	sin.sin_port = 0;
+	inet_aton(ip, &sin.sin_addr);
+	sin.sin_family = AF_INET;
+	s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+	if (s == -1) {
+		return false;
+	}
+	ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
+	close(s);
+	return ret == 0;
+}
+
+static void ctdb_wait_handler(struct event_context *ev, struct timed_event *te, 
+			      struct timeval yt, void *p)
+{
+	uint32_t *timed_out = (uint32_t *)p;
+	(*timed_out) = 1;
+}
+
+/* This function is used to kill (RST) the specified tcp connection.
+
+   This function is not asynchronous and will block until the operation
+   was successful or it timesout.
+ */
+int ctdb_sys_kill_tcp(struct event_context *ev,
+		      const struct sockaddr_in *dst, 
+		      const struct sockaddr_in *src)
+{
+	int s, ret;
+	uint32_t timedout;
+	TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+#define RCVPKTSIZE 100
+	char pkt[RCVPKTSIZE];
+	struct ether_header *eth;
+	struct iphdr *ip;
+	struct tcphdr *tcp;
+
+	/* Open a socket to capture all traffic */
+	s=socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
+	if (s == -1){
+		DEBUG(0,(__location__ " failed to open raw socket\n"));
+		return -1;
+	}
+
+	/* We wait for up to 1 second for the ACK coming back */
+	timedout = 0;
+	event_add_timed(ev, tmp_ctx, timeval_current_ofs(1, 0), ctdb_wait_handler, &timedout);
+
+	/* Send a tickle ack to probe what the real seq/ack numbers are */
+	ctdb_sys_send_tcp(dst, src, 0, 0, 0);
+
+	/* Wait until we either time out or we succeeds in sending the RST */
+	while (timedout==0) {
+		event_loop_once(ev);
+
+		ret = recv(s, pkt, RCVPKTSIZE, MSG_TRUNC);
+		if (ret < sizeof(*eth)+sizeof(*ip)) {
+			continue;
+		}
+
+		/* Ethernet */
+		eth = (struct ether_header *)pkt;
+		/* We only want IP packets */
+		if (ntohs(eth->ether_type) != ETHERTYPE_IP) {
+			continue;
+		}
+	
+		/* IP */
+		ip = (struct iphdr *)(eth+1);
+		/* We only want IPv4 packets */
+		if (ip->version != 4) {
+			continue;
+		}
+		/* Dont look at fragments */
+		if ((ntohs(ip->frag_off)&0x1fff) != 0) {
+			continue;
+		}
+		/* we only want TCP */
+		if (ip->protocol != IPPROTO_TCP) {
+			continue;
+		}
+
+		/* We only want packets sent from the guy we tickled */
+		if (ip->saddr != dst->sin_addr.s_addr) {
+			continue;
+		}
+		/* We only want packets sent to us */
+		if (ip->daddr != src->sin_addr.s_addr) {
+			continue;
+		}
+
+		/* make sure its not a short packet */
+		if (offsetof(struct tcphdr, ack_seq) + 4 + 
+		    (ip->ihl*4) + sizeof(*eth) > ret) {
+			continue;
+		}
+
+		/* TCP */
+		tcp = (struct tcphdr *)((ip->ihl*4) + (char *)ip);
+		
+		/* We only want replies from the port we tickled */
+		if (tcp->source != dst->sin_port) {
+			continue;
+		}
+		if (tcp->dest != src->sin_port) {
+			continue;
+		}
+
+		ctdb_sys_send_tcp(dst, src, tcp->ack_seq, tcp->seq, 1);
+
+		close(s);
+		talloc_free(tmp_ctx);
+
+		return 0;
+	}
+
+	close(s);
+	talloc_free(tmp_ctx);
+	DEBUG(0,(__location__ " timedout waiting for tickle ack reply\n"));
+
+	return -1;
+}
diff --git a/source4/cluster/ctdb/config.guess b/source4/cluster/ctdb/config.guess
index 354dbe175a..386626b2ba 100755
--- a/source4/cluster/ctdb/config.guess
+++ b/source4/cluster/ctdb/config.guess
@@ -3,7 +3,7 @@
 #   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
 #   2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
 
-timestamp='2005-08-03'
+timestamp='2007-07-10'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
diff --git a/source4/cluster/ctdb/config.h.in b/source4/cluster/ctdb/config.h.in
new file mode 100644
index 0000000000..7e04473ce9
--- /dev/null
+++ b/source4/cluster/ctdb/config.h.in
@@ -0,0 +1,756 @@
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Whether strndup is broken */
+#undef BROKEN_STRNDUP
+
+/* Whether strnlen is broken */
+#undef BROKEN_STRNLEN
+
+/* Define to 1 if you have the <acl/libacl.h> header file. */
+#undef HAVE_ACL_LIBACL_H
+
+/* Define to 1 if you have the <alloca.h> header file. */
+#undef HAVE_ALLOCA_H
+
+/* Define to 1 if you have the <arpa/inet.h> header file. */
+#undef HAVE_ARPA_INET_H
+
+/* Define to 1 if you have the `asprintf' function. */
+#undef HAVE_ASPRINTF
+
+/* Whether the bool type is available */
+#undef HAVE_BOOL
+
+/* Define to 1 if you have the `bzero' function. */
+#undef HAVE_BZERO
+
+/* Whether there is a C99 compliant vsnprintf */
+#undef HAVE_C99_VSNPRINTF
+
+/* Define to 1 if you have the `chroot' function. */
+#undef HAVE_CHROOT
+
+/* Define to 1 if you have the `chsize' function. */
+#undef HAVE_CHSIZE
+
+/* Whether or not we have comparison_fn_t */
+#undef HAVE_COMPARISON_FN_T
+
+/* Define to 1 if you have the <compat.h> header file. */
+#undef HAVE_COMPAT_H
+
+/* Define to 1 if you have the <ctype.h> header file. */
+#undef HAVE_CTYPE_H
+
+/* Define to 1 if you have the declaration of `asprintf', and to 0 if you
+   don't. */
+#undef HAVE_DECL_ASPRINTF
+
+/* Define to 1 if you have the declaration of `snprintf', and to 0 if you
+   don't. */
+#undef HAVE_DECL_SNPRINTF
+
+/* Define to 1 if you have the declaration of `vasprintf', and to 0 if you
+   don't. */
+#undef HAVE_DECL_VASPRINTF
+
+/* Define to 1 if you have the declaration of `vsnprintf', and to 0 if you
+   don't. */
+#undef HAVE_DECL_VSNPRINTF
+
+/* Define to 1 if you have the <direct.h> header file. */
+#undef HAVE_DIRECT_H
+
+/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
+   */
+#undef HAVE_DIRENT_H
+
+/* Define to 1 if you have the `dlclose' function. */
+#undef HAVE_DLCLOSE
+
+/* Define to 1 if you have the `dlerror' function. */
+#undef HAVE_DLERROR
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
+
+/* Define to 1 if you have the `dlopen' function. */
+#undef HAVE_DLOPEN
+
+/* Define to 1 if you have the `dlsym' function. */
+#undef HAVE_DLSYM
+
+/* Define to 1 if you have the `epoll_create' function. */
+#undef HAVE_EPOLL_CREATE
+
+/* Whether errno() is available */
+#undef HAVE_ERRNO_DECL
+
+/* Whether epoll available */
+#undef HAVE_EVENTS_EPOLL
+
+/* Define to 1 if you have the <fcntl.h> header file. */
+#undef HAVE_FCNTL_H
+
+/* Define to 1 if you have the <float.h> header file. */
+#undef HAVE_FLOAT_H
+
+/* Define to 1 if you have the <fnmatch.h> header file. */
+#undef HAVE_FNMATCH_H
+
+/* Define to 1 if you have the `ftruncate' function. */
+#undef HAVE_FTRUNCATE
+
+/* Whether there is a __FUNCTION__ macro */
+#undef HAVE_FUNCTION_MACRO
+
+/* Define to 1 if you have the `getdents' function. */
+#undef HAVE_GETDENTS
+
+/* Define to 1 if you have the `getdirentries' function. */
+#undef HAVE_GETDIRENTRIES
+
+/* Define to 1 if you have the <getopt.h> header file. */
+#undef HAVE_GETOPT_H
+
+/* Define to 1 if you have the `getpagesize' function. */
+#undef HAVE_GETPAGESIZE
+
+/* Define to 1 if you have the `getpgrp' function. */
+#undef HAVE_GETPGRP
+
+/* Define to 1 if you have the <grp.h> header file. */
+#undef HAVE_GRP_H
+
+/* Whether the compiler supports immediate structures */
+#undef HAVE_IMMEDIATE_STRUCTURES
+
+/* Define to 1 if you have the <infiniband/verbs.h> header file. */
+#undef HAVE_INFINIBAND_VERBS_H
+
+/* Define to 1 if you have the `initgroups' function. */
+#undef HAVE_INITGROUPS
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the `ibverbs' library (-libverbs). */
+#undef HAVE_LIBIBVERBS
+
+/* Define to 1 if you have the `rdmacm' library (-lrdmacm). */
+#undef HAVE_LIBRDMACM
+
+/* Define to 1 if you have the <limits.h> header file. */
+#undef HAVE_LIMITS_H
+
+/* Define to 1 if you have the <locale.h> header file. */
+#undef HAVE_LOCALE_H
+
+/* Define to 1 if the system has the type `long long'. */
+#undef HAVE_LONG_LONG
+
+/* Define to 1 if you have the `lstat' function. */
+#undef HAVE_LSTAT
+
+/* Define to 1 if you have the `memcpy' function. */
+#undef HAVE_MEMCPY
+
+/* Define to 1 if you have the `memmove' function. */
+#undef HAVE_MEMMOVE
+
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
+
+/* Define to 1 if you have the `memset' function. */
+#undef HAVE_MEMSET
+
+/* Define if target mkdir supports mode option */
+#undef HAVE_MKDIR_MODE
+
+/* Define to 1 if you have the `mkdtemp' function. */
+#undef HAVE_MKDTEMP
+
+/* Define to 1 if you have the `mktime' function. */
+#undef HAVE_MKTIME
+
+/* Define to 1 if you have the `mmap' function. */
+#undef HAVE_MMAP
+
+/* Define to 1 if you have the <mntent.h> header file. */
+#undef HAVE_MNTENT_H
+
+/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
+#undef HAVE_NDIR_H
+
+/* Define to 1 if you have the <netdb.h> header file. */
+#undef HAVE_NETDB_H
+
+/* Define to 1 if you have the <netinet/in.h> header file. */
+#undef HAVE_NETINET_IN_H
+
+/* Define to 1 if you have the <netinet/in_ip.h> header file. */
+#undef HAVE_NETINET_IN_IP_H
+
+/* Define to 1 if you have the <netinet/in_systm.h> header file. */
+#undef HAVE_NETINET_IN_SYSTM_H
+
+/* Define to 1 if you have the <netinet/ip.h> header file. */
+#undef HAVE_NETINET_IP_H
+
+/* Define to 1 if you have the <netinet/tcp.h> header file. */
+#undef HAVE_NETINET_TCP_H
+
+/* usability of net/if.h */
+#undef HAVE_NET_IF_H
+
+/* Whether the open(2) accepts O_DIRECT */
+#undef HAVE_OPEN_O_DIRECT
+
+/* Define to 1 if you have the `pipe' function. */
+#undef HAVE_PIPE
+
+/* Define to 1 if you have the <popt.h> header file. */
+#undef HAVE_POPT_H
+
+/* Define to 1 if you have the `pread' function. */
+#undef HAVE_PREAD
+
+/* Whether pread() is available */
+#undef HAVE_PREAD_DECL
+
+/* Define to 1 if you have the `printf' function. */
+#undef HAVE_PRINTF
+
+/* Define to 1 if you have the <pwd.h> header file. */
+#undef HAVE_PWD_H
+
+/* Define to 1 if you have the `pwrite' function. */
+#undef HAVE_PWRITE
+
+/* Whether pwrite() is available */
+#undef HAVE_PWRITE_DECL
+
+/* Define to 1 if you have the `rand' function. */
+#undef HAVE_RAND
+
+/* Define to 1 if you have the `random' function. */
+#undef HAVE_RANDOM
+
+/* Define to 1 if you have the <rdma/rdma_cma.h> header file. */
+#undef HAVE_RDMA_RDMA_CMA_H
+
+/* Define to 1 if you have the `rename' function. */
+#undef HAVE_RENAME
+
+/* Define to 1 if you have the <sched.h> header file. */
+#undef HAVE_SCHED_H
+
+/* Define to 1 if you have the `sched_setscheduler' function. */
+#undef HAVE_SCHED_SETSCHEDULER
+
+/* Whether mkstemp is secure */
+#undef HAVE_SECURE_MKSTEMP
+
+/* Define to 1 if you have the `setbuffer' function. */
+#undef HAVE_SETBUFFER
+
+/* Define to 1 if you have the `setegid' function. */
+#undef HAVE_SETEGID
+
+/* Define to 1 if you have the `setenv' function. */
+#undef HAVE_SETENV
+
+/* Whether setenv() is available */
+#undef HAVE_SETENV_DECL
+
+/* Define to 1 if you have the `seteuid' function. */
+#undef HAVE_SETEUID
+
+/* Define to 1 if you have the <setjmp.h> header file. */
+#undef HAVE_SETJMP_H
+
+/* Define to 1 if you have the `setlinebuf' function. */
+#undef HAVE_SETLINEBUF
+
+/* Define to 1 if you have the `setresgid' function. */
+#undef HAVE_SETRESGID
+
+/* Whether setresgid() is available */
+#undef HAVE_SETRESGID_DECL
+
+/* Define to 1 if you have the `setresuid' function. */
+#undef HAVE_SETRESUID
+
+/* Whether setresuid() is available */
+#undef HAVE_SETRESUID_DECL
+
+/* Define to 1 if you have the <shadow.h> header file. */
+#undef HAVE_SHADOW_H
+
+/* Whether we have the atomic_t variable type */
+#undef HAVE_SIG_ATOMIC_T_TYPE
+
+/* Define to 1 if you have the `snprintf' function. */
+#undef HAVE_SNPRINTF
+
+/* Define to 1 if you have the `socketpair' function. */
+#undef HAVE_SOCKETPAIR
+
+/* Whether the sockaddr_in struct has a sin_len property */
+#undef HAVE_SOCK_SIN_LEN
+
+/* Define to 1 if you have the `srand' function. */
+#undef HAVE_SRAND
+
+/* Define to 1 if you have the `srandom' function. */
+#undef HAVE_SRANDOM
+
+/* Define to 1 if you have the <standards.h> header file. */
+#undef HAVE_STANDARDS_H
+
+/* Define to 1 if you have the <stdarg.h> header file. */
+#undef HAVE_STDARG_H
+
+/* Define to 1 if you have the <stdbool.h> header file. */
+#undef HAVE_STDBOOL_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdio.h> header file. */
+#undef HAVE_STDIO_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the `strcasestr' function. */
+#undef HAVE_STRCASESTR
+
+/* Define to 1 if you have the `strdup' function. */
+#undef HAVE_STRDUP
+
+/* Define to 1 if you have the `strerror' function. */
+#undef HAVE_STRERROR
+
+/* Define to 1 if you have the `strftime' function. */
+#undef HAVE_STRFTIME
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the `strlcat' function. */
+#undef HAVE_STRLCAT
+
+/* Define to 1 if you have the `strlcpy' function. */
+#undef HAVE_STRLCPY
+
+/* Define to 1 if you have the `strndup' function. */
+#undef HAVE_STRNDUP
+
+/* Define to 1 if you have the `strnlen' function. */
+#undef HAVE_STRNLEN
+
+/* Define to 1 if you have the `strtok_r' function. */
+#undef HAVE_STRTOK_R
+
+/* Define to 1 if you have the `strtoll' function. */
+#undef HAVE_STRTOLL
+
+/* Define to 1 if you have the `strtoq' function. */
+#undef HAVE_STRTOQ
+
+/* Define to 1 if you have the `strtoull' function. */
+#undef HAVE_STRTOULL
+
+/* Define to 1 if you have the `strtouq' function. */
+#undef HAVE_STRTOUQ
+
+/* Define to 1 if `st_rdev' is member of `struct stat'. */
+#undef HAVE_STRUCT_STAT_ST_RDEV
+
+/* Define to 1 if your `struct stat' has `st_rdev'. Deprecated, use
+   `HAVE_STRUCT_STAT_ST_RDEV' instead. */
+#undef HAVE_ST_RDEV
+
+/* Define to 1 if you have the `syslog' function. */
+#undef HAVE_SYSLOG
+
+/* Define to 1 if you have the <syslog.h> header file. */
+#undef HAVE_SYSLOG_H
+
+/* Define to 1 if you have the <sys/acl.h> header file. */
+#undef HAVE_SYS_ACL_H
+
+/* Define to 1 if you have the <sys/capability.h> header file. */
+#undef HAVE_SYS_CAPABILITY_H
+
+/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
+   */
+#undef HAVE_SYS_DIR_H
+
+/* Define to 1 if you have the <sys/epoll.h> header file. */
+#undef HAVE_SYS_EPOLL_H
+
+/* Define to 1 if you have the <sys/fcntl.h> header file. */
+#undef HAVE_SYS_FCNTL_H
+
+/* Define to 1 if you have the <sys/filio.h> header file. */
+#undef HAVE_SYS_FILIO_H
+
+/* Define to 1 if you have the <sys/filsys.h> header file. */
+#undef HAVE_SYS_FILSYS_H
+
+/* Define to 1 if you have the <sys/fs/s5param.h> header file. */
+#undef HAVE_SYS_FS_S5PARAM_H
+
+/* Define to 1 if you have the <sys/id.h> header file. */
+#undef HAVE_SYS_ID_H
+
+/* Define to 1 if you have the <sys/ioctl.h> header file. */
+#undef HAVE_SYS_IOCTL_H
+
+/* Define to 1 if you have the <sys/ipc.h> header file. */
+#undef HAVE_SYS_IPC_H
+
+/* Define to 1 if you have the <sys/mman.h> header file. */
+#undef HAVE_SYS_MMAN_H
+
+/* Define to 1 if you have the <sys/mode.h> header file. */
+#undef HAVE_SYS_MODE_H
+
+/* Define to 1 if you have the <sys/mount.h> header file. */
+#undef HAVE_SYS_MOUNT_H
+
+/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
+   */
+#undef HAVE_SYS_NDIR_H
+
+/* Define to 1 if you have the <sys/param.h> header file. */
+#undef HAVE_SYS_PARAM_H
+
+/* Define to 1 if you have the <sys/priv.h> header file. */
+#undef HAVE_SYS_PRIV_H
+
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#undef HAVE_SYS_RESOURCE_H
+
+/* Define to 1 if you have the <sys/security.h> header file. */
+#undef HAVE_SYS_SECURITY_H
+
+/* Define to 1 if you have the <sys/select.h> header file. */
+#undef HAVE_SYS_SELECT_H
+
+/* Define to 1 if you have the <sys/shm.h> header file. */
+#undef HAVE_SYS_SHM_H
+
+/* Define to 1 if you have the <sys/socket.h> header file. */
+#undef HAVE_SYS_SOCKET_H
+
+/* Define to 1 if you have the <sys/sockio.h> header file. */
+#undef HAVE_SYS_SOCKIO_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/syslog.h> header file. */
+#undef HAVE_SYS_SYSLOG_H
+
+/* Define to 1 if you have the <sys/termio.h> header file. */
+#undef HAVE_SYS_TERMIO_H
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+#undef HAVE_SYS_TIME_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <sys/un.h> header file. */
+#undef HAVE_SYS_UN_H
+
+/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */
+#undef HAVE_SYS_WAIT_H
+
+/* Define to 1 if you have the <termios.h> header file. */
+#undef HAVE_TERMIOS_H
+
+/* Define to 1 if you have the <termio.h> header file. */
+#undef HAVE_TERMIO_H
+
+/* Define to 1 if you have the `timegm' function. */
+#undef HAVE_TIMEGM
+
+/* Define to 1 if you have the <time.h> header file. */
+#undef HAVE_TIME_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to 1 if you have the `unsetenv' function. */
+#undef HAVE_UNSETENV
+
+/* Define to 1 if you have the `usleep' function. */
+#undef HAVE_USLEEP
+
+/* Define to 1 if you have the `utime' function. */
+#undef HAVE_UTIME
+
+/* Define to 1 if you have the <utime.h> header file. */
+#undef HAVE_UTIME_H
+
+/* Define to 1 if you have the <vararg.h> header file. */
+#undef HAVE_VARARG_H
+
+/* Define to 1 if you have the `vasprintf' function. */
+#undef HAVE_VASPRINTF
+
+/* Whether va_copy() is available */
+#undef HAVE_VA_COPY
+
+/* Whether the C compiler understands volatile */
+#undef HAVE_VOLATILE
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#undef HAVE_VSNPRINTF
+
+/* Define to 1 if you have the `vsyslog' function. */
+#undef HAVE_VSYSLOG
+
+/* Define to 1 if you have the `waitpid' function. */
+#undef HAVE_WAITPID
+
+/* Define to 1 if you have the <windows.h> header file. */
+#undef HAVE_WINDOWS_H
+
+/* Define to 1 if you have the <winsock2.h> header file. */
+#undef HAVE_WINSOCK2_H
+
+/* Define to 1 if you have the <ws2tcpip.h> header file. */
+#undef HAVE_WS2TCPIP_H
+
+/* Whether the _Bool type is available */
+#undef HAVE__Bool
+
+/* Whether the __VA_ARGS__ macro is available */
+#undef HAVE__VA_ARGS__MACRO
+
+/* Define to 1 if you have the `__strtoll' function. */
+#undef HAVE___STRTOLL
+
+/* Define to 1 if you have the `__strtoull' function. */
+#undef HAVE___STRTOULL
+
+/* Whether __va_copy() is available */
+#undef HAVE___VA_COPY
+
+/* Whether there is a __func__ macro */
+#undef HAVE_func_MACRO
+
+/* Whether MMAP is broken */
+#undef MMAP_BLACKLIST
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Whether getpass should be replaced */
+#undef REPLACE_GETPASS
+
+/* Whether inet_ntoa should be replaced */
+#undef REPLACE_INET_NTOA
+
+/* replace readdir */
+#undef REPLACE_READDIR
+
+/* replace readdir using getdents() */
+#undef REPLACE_READDIR_GETDENTS
+
+/* replace readdir using getdirentries() */
+#undef REPLACE_READDIR_GETDIRENTRIES
+
+/* Whether strptime should be replaced */
+#undef REPLACE_STRPTIME
+
+/* Define as the return type of signal handlers (`int' or `void'). */
+#undef RETSIGTYPE
+
+/* Whether seekdir returns an int */
+#undef SEEKDIR_RETURNS_INT
+
+/* The size of `char', as computed by sizeof. */
+#undef SIZEOF_CHAR
+
+/* The size of `int', as computed by sizeof. */
+#undef SIZEOF_INT
+
+/* The size of `long', as computed by sizeof. */
+#undef SIZEOF_LONG
+
+/* The size of `long long', as computed by sizeof. */
+#undef SIZEOF_LONG_LONG
+
+/* The size of `off_t', as computed by sizeof. */
+#undef SIZEOF_OFF_T
+
+/* The size of `short', as computed by sizeof. */
+#undef SIZEOF_SHORT
+
+/* The size of `size_t', as computed by sizeof. */
+#undef SIZEOF_SIZE_T
+
+/* The size of `ssize_t', as computed by sizeof. */
+#undef SIZEOF_SSIZE_T
+
+/* The size of `void *', as computed by sizeof. */
+#undef SIZEOF_VOID_P
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* Whether telldir takes a const pointer */
+#undef TELLDIR_TAKES_CONST_DIR
+
+/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
+#undef TIME_WITH_SYS_TIME
+
+/* Use infiniband */
+#undef USE_INFINIBAND
+
+/* Define to 1 if your processor stores words with the most significant byte
+   first (like Motorola and SPARC, unlike Intel and VAX). */
+#undef WORDS_BIGENDIAN
+
+/* Define to 1 if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+# undef _ALL_SOURCE
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+#undef _FILE_OFFSET_BITS
+
+/* Enable GNU extensions on systems that have them.  */
+#ifndef _GNU_SOURCE
+# undef _GNU_SOURCE
+#endif
+
+/* Define for large files, on AIX-style hosts. */
+#undef _LARGE_FILES
+
+/* Define to 1 if on MINIX. */
+#undef _MINIX
+
+#ifndef _OSF_SOURCE
+# define _OSF_SOURCE 1
+#endif
+
+/* Define to 2 if the system does not provide POSIX.1 features except with
+   this defined. */
+#undef _POSIX_1_SOURCE
+
+/* Whether to enable POSIX support */
+#undef _POSIX_C_SOURCE
+
+/* Define to 1 if you need to in order for `stat' and other things to work. */
+#undef _POSIX_SOURCE
+
+/* Whether to enable System V compatibility */
+#undef _SYSV
+
+#ifndef _XOPEN_SOURCE_EXTENDED
+# define _XOPEN_SOURCE_EXTENDED 1
+#endif
+
+/* Enable extensions on Solaris.  */
+#ifndef __EXTENSIONS__
+# undef __EXTENSIONS__
+#endif
+#ifndef _POSIX_PTHREAD_SEMANTICS
+# undef _POSIX_PTHREAD_SEMANTICS
+#endif
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef gid_t
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+#undef inline
+#endif
+
+/* Define to `unsigned' if <sys/types.h> does not define. */
+#undef ino_t
+
+/* Define to `short' if <sys/types.h> does not define. */
+#undef int16_t
+
+/* Define to `long' if <sys/types.h> does not define. */
+#undef int32_t
+
+/* Define to `long long' if <sys/types.h> does not define. */
+#undef int64_t
+
+/* Define to `char' if <sys/types.h> does not define. */
+#undef int8_t
+
+/* Define to `unsigned long long' if <sys/types.h> does not define. */
+#undef intptr_t
+
+/* Define to `off_t' if <sys/types.h> does not define. */
+#undef loff_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef mode_t
+
+/* Define to `long int' if <sys/types.h> does not define. */
+#undef off_t
+
+/* Define to `loff_t' if <sys/types.h> does not define. */
+#undef offset_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef pid_t
+
+/* Define to `unsigned long long' if <sys/types.h> does not define. */
+#undef ptrdiff_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef size_t
+
+/* Socket length type */
+#undef socklen_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef ssize_t
+
+/* Define to `int' if <sys/types.h> doesn't define. */
+#undef uid_t
+
+/* Define to `unsigned short' if <sys/types.h> does not define. */
+#undef uint16_t
+
+/* Define to `unsigned long' if <sys/types.h> does not define. */
+#undef uint32_t
+
+/* Define to `unsigned long long' if <sys/types.h> does not define. */
+#undef uint64_t
+
+/* Define to `unsigned char' if <sys/types.h> does not define. */
+#undef uint8_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef uint_t
diff --git a/source4/cluster/ctdb/config.mk b/source4/cluster/ctdb/config.mk
index 076562297f..61f1e36823 100644
--- a/source4/cluster/ctdb/config.mk
+++ b/source4/cluster/ctdb/config.mk
@@ -7,24 +7,12 @@ OBJ_FILES = brlock_ctdb.o
 OBJ_FILES = opendb_ctdb.o
 
 ##################
-[SUBSYSTEM::ctdb_tcp]
-OBJ_FILES = \
-		tcp/tcp_init.o \
-		tcp/tcp_io.o \
-		tcp/tcp_connect.o
-
-##################
 [SUBSYSTEM::ctdb]
 OBJ_FILES = \
 		ctdb_cluster.o \
-		common/ctdb.o \
-		common/ctdb_call.o \
-		common/ctdb_message.o \
-		common/ctdb_ltdb.o \
-		common/ctdb_util.o \
+		client/ctdb_client.o \
 		common/ctdb_io.o \
-		common/ctdb_client.o \
-		common/ctdb_daemon.o \
-		common/ctdb_lockwait.o
+		common/ctdb_ltdb.o \
+		common/ctdb_message.o \
+		common/ctdb_util.o
 PUBLIC_DEPENDENCIES = LIBTDB LIBTALLOC
-PRIVATE_DEPENDENCIES = ctdb_tcp
diff --git a/source4/cluster/ctdb/config.sub b/source4/cluster/ctdb/config.sub
new file mode 100755
index 0000000000..665fc1bda4
--- /dev/null
+++ b/source4/cluster/ctdb/config.sub
@@ -0,0 +1,1577 @@
+#! /bin/sh
+# Configuration validation subroutine script.
+#   Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
+#   2000, 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+
+timestamp='2007-07-10'
+
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+
+# Please send patches to <config-patches@gnu.org>.  Submit a context
+# diff and a properly formatted ChangeLog entry.
+#
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# or in some cases, the newer four-part form:
+#	CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+me=`echo "$0" | sed -e 's,.*/,,'`
+
+usage="\
+Usage: $0 [OPTION] CPU-MFR-OPSYS
+       $0 [OPTION] ALIAS
+
+Canonicalize a configuration name.
+
+Operation modes:
+  -h, --help         print this help, then exit
+  -t, --time-stamp   print date of last modification, then exit
+  -v, --version      print version number, then exit
+
+Report bugs and patches to <config-patches@gnu.org>."
+
+version="\
+GNU config.sub ($timestamp)
+
+Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005
+Free Software Foundation, Inc.
+
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
+
+help="
+Try \`$me --help' for more information."
+
+# Parse command line
+while test $# -gt 0 ; do
+  case $1 in
+    --time-stamp | --time* | -t )
+       echo "$timestamp" ; exit ;;
+    --version | -v )
+       echo "$version" ; exit ;;
+    --help | --h* | -h )
+       echo "$usage"; exit ;;
+    -- )     # Stop option processing
+       shift; break ;;
+    - )	# Use stdin as input.
+       break ;;
+    -* )
+       echo "$me: invalid option $1$help"
+       exit 1 ;;
+
+    *local*)
+       # First pass through any local machine types.
+       echo $1
+       exit ;;
+
+    * )
+       break ;;
+  esac
+done
+
+case $# in
+ 0) echo "$me: missing argument$help" >&2
+    exit 1;;
+ 1) ;;
+ *) echo "$me: too many arguments$help" >&2
+    exit 1;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any).
+# Here we must recognize all the valid KERNEL-OS combinations.
+maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
+case $maybe_os in
+  nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \
+  kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*)
+    os=-$maybe_os
+    basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
+    ;;
+  *)
+    basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+    if [ $basic_machine != $1 ]
+    then os=`echo $1 | sed 's/.*-/-/'`
+    else os=; fi
+    ;;
+esac
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
+	-apple | -axis | -knuth | -cray)
+		os=
+		basic_machine=$1
+		;;
+	-sim | -cisco | -oki | -wec | -winbond)
+		os=
+		basic_machine=$1
+		;;
+	-scout)
+		;;
+	-wrs)
+		os=-vxworks
+		basic_machine=$1
+		;;
+	-chorusos*)
+		os=-chorusos
+		basic_machine=$1
+		;;
+ 	-chorusrdb)
+ 		os=-chorusrdb
+		basic_machine=$1
+ 		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco5)
+		os=-sco3.2v5
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-udk*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'`
+		;;
+	-lynx*)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+	-psos*)
+		os=-psos
+		;;
+	-mint | -mint[0-9]*)
+		basic_machine=m68k-atari
+		os=-mint
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	1750a | 580 \
+	| a29k \
+	| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
+	| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
+	| am33_2.0 \
+	| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \
+	| bfin \
+	| c4x | clipper \
+	| d10v | d30v | dlx | dsp16xx \
+	| fr30 | frv \
+	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
+	| i370 | i860 | i960 | ia64 \
+	| ip2k | iq2000 \
+	| m32r | m32rle | m68000 | m68k | m88k | maxq | mcore \
+	| mips | mipsbe | mipseb | mipsel | mipsle \
+	| mips16 \
+	| mips64 | mips64el \
+	| mips64vr | mips64vrel \
+	| mips64orion | mips64orionel \
+	| mips64vr4100 | mips64vr4100el \
+	| mips64vr4300 | mips64vr4300el \
+	| mips64vr5000 | mips64vr5000el \
+	| mips64vr5900 | mips64vr5900el \
+	| mipsisa32 | mipsisa32el \
+	| mipsisa32r2 | mipsisa32r2el \
+	| mipsisa64 | mipsisa64el \
+	| mipsisa64r2 | mipsisa64r2el \
+	| mipsisa64sb1 | mipsisa64sb1el \
+	| mipsisa64sr71k | mipsisa64sr71kel \
+	| mipstx39 | mipstx39el \
+	| mn10200 | mn10300 \
+	| ms1 \
+	| msp430 \
+	| ns16k | ns32k \
+	| or32 \
+	| pdp10 | pdp11 | pj | pjl \
+	| powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
+	| pyramid \
+	| sh | sh[1234] | sh[24]a | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \
+	| sh64 | sh64le \
+	| sparc | sparc64 | sparc64b | sparc86x | sparclet | sparclite \
+	| sparcv8 | sparcv9 | sparcv9b \
+	| strongarm \
+	| tahoe | thumb | tic4x | tic80 | tron \
+	| v850 | v850e \
+	| we32k \
+	| x86 | xscale | xscalee[bl] | xstormy16 | xtensa \
+	| z8k)
+		basic_machine=$basic_machine-unknown
+		;;
+	m32c)
+		basic_machine=$basic_machine-unknown
+		;;
+	m6811 | m68hc11 | m6812 | m68hc12)
+		# Motorola 68HC11/12.
+		basic_machine=$basic_machine-unknown
+		os=-none
+		;;
+	m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k)
+		;;
+
+	# We use `pc' rather than `unknown'
+	# because (1) that's what they normally are, and
+	# (2) the word "unknown" tends to confuse beginning users.
+	i*86 | x86_64)
+	  basic_machine=$basic_machine-pc
+	  ;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	580-* \
+	| a29k-* \
+	| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
+	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
+	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
+	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
+	| avr-* \
+	| bfin-* | bs2000-* \
+	| c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \
+	| clipper-* | craynv-* | cydra-* \
+	| d10v-* | d30v-* | dlx-* \
+	| elxsi-* \
+	| f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \
+	| h8300-* | h8500-* \
+	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
+	| i*86-* | i860-* | i960-* | ia64-* \
+	| ip2k-* | iq2000-* \
+	| m32r-* | m32rle-* \
+	| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
+	| m88110-* | m88k-* | maxq-* | mcore-* \
+	| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
+	| mips16-* \
+	| mips64-* | mips64el-* \
+	| mips64vr-* | mips64vrel-* \
+	| mips64orion-* | mips64orionel-* \
+	| mips64vr4100-* | mips64vr4100el-* \
+	| mips64vr4300-* | mips64vr4300el-* \
+	| mips64vr5000-* | mips64vr5000el-* \
+	| mips64vr5900-* | mips64vr5900el-* \
+	| mipsisa32-* | mipsisa32el-* \
+	| mipsisa32r2-* | mipsisa32r2el-* \
+	| mipsisa64-* | mipsisa64el-* \
+	| mipsisa64r2-* | mipsisa64r2el-* \
+	| mipsisa64sb1-* | mipsisa64sb1el-* \
+	| mipsisa64sr71k-* | mipsisa64sr71kel-* \
+	| mipstx39-* | mipstx39el-* \
+	| mmix-* \
+	| ms1-* \
+	| msp430-* \
+	| none-* | np1-* | ns16k-* | ns32k-* \
+	| orion-* \
+	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
+	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
+	| pyramid-* \
+	| romp-* | rs6000-* \
+	| sh-* | sh[1234]-* | sh[24]a-* | sh[23]e-* | sh[34]eb-* | shbe-* \
+	| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
+	| sparc-* | sparc64-* | sparc64b-* | sparc86x-* | sparclet-* \
+	| sparclite-* \
+	| sparcv8-* | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \
+	| tahoe-* | thumb-* \
+	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
+	| tron-* \
+	| v850-* | v850e-* | vax-* \
+	| we32k-* \
+	| x86-* | x86_64-* | xps100-* | xscale-* | xscalee[bl]-* \
+	| xstormy16-* | xtensa-* \
+	| ymp-* \
+	| z8k-*)
+		;;
+	m32c-*)
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	386bsd)
+		basic_machine=i386-unknown
+		os=-bsd
+		;;
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	a29khif)
+		basic_machine=a29k-amd
+		os=-udi
+		;;
+    	abacus)
+		basic_machine=abacus-unknown
+		;;
+	adobe68k)
+		basic_machine=m68010-adobe
+		os=-scout
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amd64)
+		basic_machine=x86_64-pc
+		;;
+	amd64-*)
+		basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-unknown
+		;;
+	amigaos | amigados)
+		basic_machine=m68k-unknown
+		os=-amigaos
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-unknown
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	apollo68bsd)
+		basic_machine=m68k-apollo
+		os=-bsd
+		;;
+	aux)
+		basic_machine=m68k-apple
+		os=-aux
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	c90)
+		basic_machine=c90-cray
+		os=-unicos
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | j90)
+		basic_machine=j90-cray
+		os=-unicos
+		;;
+	craynv)
+		basic_machine=craynv-cray
+		os=-unicosmp
+		;;
+	cr16c)
+		basic_machine=cr16c-unknown
+		os=-elf
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	crisv32 | crisv32-* | etraxfs*)
+		basic_machine=crisv32-axis
+		;;
+	cris | cris-* | etrax*)
+		basic_machine=cris-axis
+		;;
+	crx)
+		basic_machine=crx-unknown
+		os=-elf
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	decsystem10* | dec10*)
+		basic_machine=pdp10-dec
+		os=-tops10
+		;;
+	decsystem20* | dec20*)
+		basic_machine=pdp10-dec
+		os=-tops20
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	djgpp)
+		basic_machine=i586-pc
+		os=-msdosdjgpp
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	es1800 | OSE68k | ose68k | ose | OSE)
+		basic_machine=m68k-ericsson
+		os=-ose
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	go32)
+		basic_machine=i386-pc
+		os=-go32
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	h8300xray)
+		basic_machine=h8300-hitachi
+		os=-xray
+		;;
+	h8500hms)
+		basic_machine=h8500-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp3k9[0-9][0-9] | hp9[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k6[0-9][0-9] | hp6[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	hp9k7[0-79][0-9] | hp7[0-79][0-9])
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k78[0-9] | hp78[0-9])
+		# FIXME: really hppa2.0-hp
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893)
+		# FIXME: really hppa2.0-hp
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][13679] | hp8[0-9][13679])
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	hppa-next)
+		os=-nextstep3
+		;;
+	hppaosf)
+		basic_machine=hppa1.1-hp
+		os=-osf
+		;;
+	hppro)
+		basic_machine=hppa1.1-hp
+		os=-proelf
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i*86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-sysv32
+		;;
+	i*86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-sysv4
+		;;
+	i*86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-sysv
+		;;
+	i*86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'`
+		os=-solaris2
+		;;
+	i386mach)
+		basic_machine=i386-mach
+		os=-mach
+		;;
+	i386-vsta | vsta)
+		basic_machine=i386-unknown
+		os=-vsta
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	mingw32)
+		basic_machine=i386-pc
+		os=-mingw32
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	*mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*)
+		basic_machine=m68k-atari
+		os=-mint
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	monitor)
+		basic_machine=m68k-rom68k
+		os=-coff
+		;;
+	morphos)
+		basic_machine=powerpc-unknown
+		os=-morphos
+		;;
+	msdos)
+		basic_machine=i386-pc
+		os=-msdos
+		;;
+	mvs)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	netbsd386)
+		basic_machine=i386-unknown
+		os=-netbsd
+		;;
+	netwinder)
+		basic_machine=armv4l-rebel
+		os=-linux
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	necv70)
+		basic_machine=v70-nec
+		os=-sysv
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	mon960)
+		basic_machine=i960-intel
+		os=-mon960
+		;;
+	nonstopux)
+		basic_machine=mips-compaq
+		os=-nonstopux
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	nsr-tandem)
+		basic_machine=nsr-tandem
+		;;
+	op50n-* | op60c-*)
+		basic_machine=hppa1.1-oki
+		os=-proelf
+		;;
+	openrisc | openrisc-*)
+		basic_machine=or32-unknown
+		;;
+	os400)
+		basic_machine=powerpc-ibm
+		os=-os400
+		;;
+	OSE68000 | ose68000)
+		basic_machine=m68000-ericsson
+		os=-ose
+		;;
+	os68k)
+		basic_machine=m68k-none
+		os=-os68k
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+	pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium | p5 | k5 | k6 | nexgen | viac3)
+		basic_machine=i586-pc
+		;;
+	pentiumpro | p6 | 6x86 | athlon | athlon_*)
+		basic_machine=i686-pc
+		;;
+	pentiumii | pentium2 | pentiumiii | pentium3)
+		basic_machine=i686-pc
+		;;
+	pentium4)
+		basic_machine=i786-pc
+		;;
+	pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*)
+		basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pentiumpro-* | p6-* | 6x86-* | athlon-*)
+		basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*)
+		basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pentium4-*)
+		basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	power)	basic_machine=power-ibm
+		;;
+	ppc)	basic_machine=powerpc-unknown
+		;;
+	ppc-*)	basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	ppcle | powerpclittle | ppc-le | powerpc-little)
+		basic_machine=powerpcle-unknown
+		;;
+	ppcle-* | powerpclittle-*)
+		basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	ppc64)	basic_machine=powerpc64-unknown
+		;;
+	ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	ppc64le | powerpc64little | ppc64-le | powerpc64-little)
+		basic_machine=powerpc64le-unknown
+		;;
+	ppc64le-* | powerpc64little-*)
+		basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	pw32)
+		basic_machine=i586-unknown
+		os=-pw32
+		;;
+	rom68k)
+		basic_machine=m68k-rom68k
+		os=-coff
+		;;
+	rm[46]00)
+		basic_machine=mips-siemens
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	s390 | s390-*)
+		basic_machine=s390-ibm
+		;;
+	s390x | s390x-*)
+		basic_machine=s390x-ibm
+		;;
+	sa29200)
+		basic_machine=a29k-amd
+		os=-udi
+		;;
+	sb1)
+		basic_machine=mipsisa64sb1-unknown
+		;;
+	sb1el)
+		basic_machine=mipsisa64sb1el-unknown
+		;;
+	sei)
+		basic_machine=mips-sei
+		os=-seiux
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sh64)
+		basic_machine=sh64-unknown
+		;;
+	sparclite-wrs | simso-wrs)
+		basic_machine=sparclite-wrs
+		os=-vxworks
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	st2000)
+		basic_machine=m68k-tandem
+		;;
+	stratus)
+		basic_machine=i860-stratus
+		os=-sysv4
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun4sol2)
+		basic_machine=sparc-sun
+		os=-solaris2
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	sv1)
+		basic_machine=sv1-cray
+		os=-unicos
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	t3e)
+		basic_machine=alphaev5-cray
+		os=-unicos
+		;;
+	t90)
+		basic_machine=t90-cray
+		os=-unicos
+		;;
+	tic54x | c54x*)
+		basic_machine=tic54x-unknown
+		os=-coff
+		;;
+	tic55x | c55x*)
+		basic_machine=tic55x-unknown
+		os=-coff
+		;;
+	tic6x | c6x*)
+		basic_machine=tic6x-unknown
+		os=-coff
+		;;
+	tx39)
+		basic_machine=mipstx39-unknown
+		;;
+	tx39el)
+		basic_machine=mipstx39el-unknown
+		;;
+	toad1)
+		basic_machine=pdp10-xkl
+		os=-tops20
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	tpf)
+		basic_machine=s390x-ibm
+		os=-tpf
+		;;
+	udi29k)
+		basic_machine=a29k-amd
+		os=-udi
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	v810 | necv810)
+		basic_machine=v810-nec
+		os=-none
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vpp*|vx|vx-*)
+		basic_machine=f301-fujitsu
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	vxworks29k)
+		basic_machine=a29k-wrs
+		os=-vxworks
+		;;
+	w65*)
+		basic_machine=w65-wdc
+		os=-none
+		;;
+	w89k-*)
+		basic_machine=hppa1.1-winbond
+		os=-proelf
+		;;
+	xbox)
+		basic_machine=i686-pc
+		os=-mingw32
+		;;
+	xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	z8k-*-coff)
+		basic_machine=z8k-unknown
+		os=-sim
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	w89k)
+		basic_machine=hppa1.1-winbond
+		;;
+	op50n)
+		basic_machine=hppa1.1-oki
+		;;
+	op60c)
+		basic_machine=hppa1.1-oki
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	mmix)
+		basic_machine=mmix-knuth
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp10)
+		# there are many clones, so DEC is not a safe bet
+		basic_machine=pdp10-unknown
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sh[1234] | sh[24]a | sh[34]eb | sh[1234]le | sh[23]ele)
+		basic_machine=sh-unknown
+		;;
+	sparc | sparcv8 | sparcv9 | sparcv9b)
+		basic_machine=sparc-sun
+		;;
+	cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	mac | mpw | mac-mpw)
+		basic_machine=m68k-apple
+		;;
+	pmac | pmac-mpw)
+		basic_machine=powerpc-apple
+		;;
+	*-unknown)
+		# Make sure to match an already-canonicalized machine name.
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+        # First match some system type aliases
+        # that might get confused with valid system types.
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-svr4*)
+		os=-sysv4
+		;;
+	-unixware*)
+		os=-sysv4.2uw
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative MUST END IN A *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
+	      | -aos* \
+	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
+	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \
+	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
+	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
+	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
+	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
+	      | -chorusos* | -chorusrdb* \
+	      | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
+	      | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \
+	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
+	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
+	      | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
+	      | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
+	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
+	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
+	      | -skyos* | -haiku*)
+	# Remember, each alternative MUST END IN *, to match a version number.
+		;;
+	-qnx*)
+		case $basic_machine in
+		    x86-* | i*86-*)
+			;;
+		    *)
+			os=-nto$os
+			;;
+		esac
+		;;
+	-nto-qnx*)
+		;;
+	-nto*)
+		os=`echo $os | sed -e 's|nto|nto-qnx|'`
+		;;
+	-sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \
+	      | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \
+	      | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*)
+		;;
+	-mac*)
+		os=`echo $os | sed -e 's|mac|macos|'`
+		;;
+	-linux-dietlibc)
+		os=-linux-dietlibc
+		;;
+	-linux*)
+		os=`echo $os | sed -e 's|linux|linux-gnu|'`
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-opened*)
+		os=-openedition
+		;;
+        -os400*)
+		os=-os400
+		;;
+	-wince*)
+		os=-wince
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-atheos*)
+		os=-atheos
+		;;
+	-syllable*)
+		os=-syllable
+		;;
+	-386bsd)
+		os=-bsd
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-nova*)
+		os=-rtmk-nova
+		;;
+	-ns2 )
+		os=-nextstep2
+		;;
+	-nsk*)
+		os=-nsk
+		;;
+	# Preserve the version number of sinix5.
+	-sinix5.*)
+		os=`echo $os | sed -e 's|sinix|sysv|'`
+		;;
+	-sinix*)
+		os=-sysv4
+		;;
+        -tpf*)
+		os=-tpf
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-ose*)
+		os=-ose
+		;;
+	-es1800*)
+		os=-ose
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+		os=-mint
+		;;
+	-aros*)
+		os=-aros
+		;;
+	-kaos*)
+		os=-kaos
+		;;
+	-zvmoe)
+		os=-zvmoe
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+	arm*-rebel)
+		os=-linux
+		;;
+	arm*-semi)
+		os=-aout
+		;;
+    c4x-* | tic4x-*)
+        os=-coff
+        ;;
+	# This must come before the *-dec entry.
+	pdp10-*)
+		os=-tops20
+		;;
+	pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	m68*-apollo)
+		os=-domain
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	m68*-cisco)
+		os=-aout
+		;;
+	mips*-cisco)
+		os=-elf
+		;;
+	mips*-*)
+		os=-elf
+		;;
+	or32-*)
+		os=-coff
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-be)
+		os=-beos
+		;;
+	*-haiku)
+		os=-haiku
+		;;
+	*-ibm)
+		os=-aix
+		;;
+    	*-knuth)
+		os=-mmixware
+		;;
+	*-wec)
+		os=-proelf
+		;;
+	*-winbond)
+		os=-proelf
+		;;
+	*-oki)
+		os=-proelf
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigaos
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-next )
+		os=-nextstep
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+	*-gould)
+		os=-sysv
+		;;
+	*-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+	*-sgi)
+		os=-irix
+		;;
+	*-siemens)
+		os=-sysv4
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	f30[01]-fujitsu | f700-fujitsu)
+		os=-uxpv
+		;;
+	*-rom68k)
+		os=-coff
+		;;
+	*-*bug)
+		os=-coff
+		;;
+	*-apple)
+		os=-macos
+		;;
+	*-atari*)
+		os=-mint
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-beos*)
+				vendor=be
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-mpeix*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs* | -opened*)
+				vendor=ibm
+				;;
+			-os400*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+			-tpf*)
+				vendor=ibm
+				;;
+			-vxsim* | -vxworks* | -windiss*)
+				vendor=wrs
+				;;
+			-aux*)
+				vendor=apple
+				;;
+			-hms*)
+				vendor=hitachi
+				;;
+			-mpw* | -macos*)
+				vendor=apple
+				;;
+			-*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*)
+				vendor=atari
+				;;
+			-vos*)
+				vendor=stratus
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
+exit
+
+# Local variables:
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "timestamp='"
+# time-stamp-format: "%:y-%02m-%02d"
+# time-stamp-end: "'"
+# End:
diff --git a/source4/cluster/ctdb/configure.ac b/source4/cluster/ctdb/configure.ac
index 784eef2190..86a70d157d 100644
--- a/source4/cluster/ctdb/configure.ac
+++ b/source4/cluster/ctdb/configure.ac
@@ -12,7 +12,7 @@ AC_DEFUN([SMB_LIBRARY_ENABLE], [echo -n ""])
 AC_DEFUN([SMB_EXT_LIB], [echo -n ""])
 AC_DEFUN([SMB_ENABLE], [echo -n ""])
 AC_INIT(ctdb.h)
-AC_CONFIG_SRCDIR([tests/ctdb_test.c])
+AC_CONFIG_SRCDIR([server/ctdbd.c])
 
 AC_LIBREPLACE_ALL_CHECKS
 
@@ -24,10 +24,25 @@ AC_CONFIG_HEADER(config.h)
 
 EXTRA_OBJ=""
 
+m4_include(libpopt.m4)
 m4_include(libtalloc.m4)
 m4_include(libtdb.m4)
+m4_include(libevents.m4)
 m4_include(ib/config.m4)
 
+AC_CHECK_HEADERS(sched.h)
+AC_CHECK_FUNCS(sched_setscheduler)
+
+AC_CACHE_CHECK([for sin_len in sock],ctdb_cv_HAVE_SOCK_SIN_LEN,[
+AC_TRY_COMPILE([#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>],
+[struct sockaddr_in sock; sock.sin_len = sizeof(sock);],
+ctdb_cv_HAVE_SOCK_SIN_LEN=yes,ctdb_cv_HAVE_SOCK_SIN_LEN=no)])
+if test x"$ctdb_cv_HAVE_SOCK_SIN_LEN" = x"yes"; then
+    AC_DEFINE(HAVE_SOCK_SIN_LEN,1,[Whether the sockaddr_in struct has a sin_len property])
+fi
+
 AC_SUBST(EXTRA_OBJ)
 
 AC_OUTPUT(Makefile)
diff --git a/source4/cluster/ctdb/ctdb_cluster.c b/source4/cluster/ctdb/ctdb_cluster.c
index caade047d5..b7aa52a7eb 100644
--- a/source4/cluster/ctdb/ctdb_cluster.c
+++ b/source4/cluster/ctdb/ctdb_cluster.c
@@ -112,7 +112,7 @@ struct ctdb_handler_state {
 /*
   dispatch incoming ctdb messages
 */
-static void ctdb_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+static void ctdb_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 				 TDB_DATA data, void *private)
 {
 	struct ctdb_handler_state *s = talloc_get_type(private, 
@@ -196,22 +196,10 @@ static struct cluster_ops cluster_ctdb_ops = {
 /* initialise ctdb */
 void cluster_ctdb_init(struct event_context *ev, const char *model)
 {
-	const char *nlist;
-	const char *address;
-	const char *transport;
 	struct cluster_state *state;
-	int ret, lacount, i;
-	const char *db_list[] = { "brlock", "opendb" };
 
-	nlist = lp_parm_string(-1, "ctdb", "nlist");
-	if (nlist == NULL) return;
-
-	address = lp_parm_string(-1, "ctdb", "address");
-	if (address == NULL) return;
-
-	transport = lp_parm_string(-1, "ctdb", "transport");
-	if (transport == NULL) {
-		transport = "tcp";
+	if (!lp_parm_bool(-1, "ctdb", "enable", False)) {
+		return;
 	}
 
 	state = talloc(ev, struct cluster_state);
@@ -224,51 +212,6 @@ void cluster_ctdb_init(struct event_context *ev, const char *model)
 
 	cluster_ctdb_ops.private = state;
 
-	ret = ctdb_set_transport(state->ctdb, transport);
-	if (ret == -1) {
-		DEBUG(0,("ctdb_set_transport failed - %s\n",
-			 ctdb_errstr(state->ctdb)));
-		goto failed;
-	}
-
-	if (lp_parm_bool(-1, "ctdb", "selfconnect", False)) {
-		DEBUG(0,("Enabling ctdb selfconnect\n"));
-		ctdb_set_flags(state->ctdb, CTDB_FLAG_SELF_CONNECT);
-	}
-
-	lacount = lp_parm_int(-1, "ctdb", "maxlacount", -1);
-	if (lacount != -1) {
-		ctdb_set_max_lacount(state->ctdb, lacount);
-	}
-
-	/* tell ctdb what address to listen on */
-        ret = ctdb_set_address(state->ctdb, address);
-        if (ret == -1) {
-                DEBUG(0,("ctdb_set_address failed - %s\n", ctdb_errstr(state->ctdb)));
-		goto failed;
-        }
-
-	ret = ctdb_set_tdb_dir(state->ctdb, lp_lockdir());
-	if (ret == -1) {
-		DEBUG(0,("ctdb_set_tdb_dir failed - %s\n", ctdb_errstr(state->ctdb)));
-		goto failed;
-	}
-
-        /* tell ctdb what nodes are available */
-        ret = ctdb_set_nlist(state->ctdb, nlist);
-        if (ret == -1) {
-                DEBUG(0,("ctdb_set_nlist failed - %s\n", ctdb_errstr(state->ctdb)));
-		goto failed;
-        }
-
-	/* attach all the databases we will need */
-	for (i=0;i<ARRAY_SIZE(db_list);i++) {
-		struct ctdb_db_context *ctdb_db;
-		ctdb_db = ctdb_attach(state->ctdb, db_list[i], TDB_INTERNAL, 
-				      O_RDWR|O_CREAT|O_TRUNC, 0666);
-		if (ctdb_db == NULL) goto failed;
-	}
-
 	cluster_set_ops(&cluster_ctdb_ops);
 
 	/* nasty hack for now ... */
@@ -277,21 +220,9 @@ void cluster_ctdb_init(struct event_context *ev, const char *model)
 		brl_ctdb_init_ops();
 	}
 
-	/* start the protocol running */
-	ret = ctdb_start(state->ctdb);
-        if (ret == -1) {
-                DEBUG(0,("ctdb_start failed - %s\n", ctdb_errstr(state->ctdb)));
-		goto failed;
-        }
-
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(state->ctdb);
-
 	return;
 	
 failed:
 	DEBUG(0,("cluster_ctdb_init failed\n"));
 	talloc_free(state);
 }
-
diff --git a/source4/cluster/ctdb/doc/ctdb.1 b/source4/cluster/ctdb/doc/ctdb.1
new file mode 100644
index 0000000000..e8f764bd86
--- /dev/null
+++ b/source4/cluster/ctdb/doc/ctdb.1
@@ -0,0 +1,403 @@
+.\"     Title: ctdb
+.\"    Author: 
+.\" Generator: DocBook XSL Stylesheets v1.72.0 <http://docbook.sf.net/>
+.\"      Date: 07/10/2007
+.\"    Manual: 
+.\"    Source: 
+.\"
+.TH "CTDB" "1" "07/10/2007" "" ""
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.SH "NAME"
+ctdb \- clustered tdb database management utility
+.SH "SYNOPSIS"
+.HP 29
+\fBctdb [ OPTIONS ] COMMAND ...\fR
+.HP 5
+\fBctdb\fR [\-n\ <node>] [\-Y] [\-t\ <timeout>] [\-?\ \-\-help] [\-\-usage] [\-d\ \-\-debug=<INTEGER>] [\-\-socket=<filename>]
+.SH "DESCRIPTION"
+.PP
+ctdb is a utility to view and manage a ctdb cluster.
+.SH "OPTIONS"
+.PP
+\-n <node>
+.RS 4
+This specifies on which node to execute the command. Default is to run the command on the deamon running on the local host.
+.RE
+.PP
+\-Y
+.RS 4
+Produce output in machinereadable form for easier parsing by scripts. Not all commands support this option.
+.RE
+.PP
+\-t <timeout>
+.RS 4
+How long should ctdb wait for a command to complete before timing out. Default is 3 seconds.
+.RE
+.PP
+\-? \-\-help
+.RS 4
+Print some help text to the screen.
+.RE
+.PP
+\-\-usage
+.RS 4
+Print useage information to the screen.
+.RE
+.PP
+\-d \-\-debug=<debuglevel>
+.RS 4
+Change the debug level for the command. Default is 0.
+.RE
+.PP
+\-\-socket=<filename>
+.RS 4
+Specify the socketname to use when connecting to the local ctdb daemon. The default is /tmp/ctdb.socket .
+.sp
+You only need to specify this parameter if you run multiple ctdb daemons on the same physical host and thus can not use the default name for the domain socket.
+.RE
+.SH "ADMINISTRATIVE COMMANDS"
+.PP
+These are commands used to monitor and administrate a CTDB cluster.
+.SS "status"
+.PP
+This command shows the current status of the ctdb node.
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+\fBnode status\fR
+.RS
+.PP
+Node status reflects the current status of the node. There are four possible states:
+.PP
+OK \- This node is fully functional.
+.PP
+DISCONNECTED \- This node could not be connected through the network and is currently not parcipitating in the cluster. If there is a public IP address associated with this node it should have been taken over by a different node. No services are running on this node.
+.PP
+DISABLED \- This node has been administratively disabled. This node is still functional and participates in the CTDB cluster but its IP addresses have been taken over by a different node and no services are currently being hosted.
+.PP
+UNHEALTHY \- A service provided by this node is malfunctioning and should be investigated. The CTDB daemon itself is operational and participates in the cluster. Its public IP address has been taken over by a different node and no services are currnetly being hosted. All unhealthy nodes should be investigated and require an administrative action to rectify.
+.PP
+BANNED \- This node failed too many recovery attempts and has been banned from participating in the cluster for a period of RecoveryBanPeriod seconds. Any public IP address has been taken over by other nodes. This node does not provide any services. All banned nodes should be investigated and require an administrative action to rectify. This node does not perticipate in the CTDB cluster but can still be communicated with. I.e. ctdb commands can be sent to it.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+\fBgeneration\fR
+.RS
+.PP
+The generation id is a number that indicates the current generation of a cluster instance. Each time a cluster goes through a reconfiguration or a recovery its generation id will be changed.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+\fBVNNMAP\fR
+.RS
+.PP
+The list of Virtual Node Numbers. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records. Only nodes that are parcipitating in the vnnmap can become lmaster or dmaster for a database record.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+\fBRecovery mode\fR
+.RS
+.PP
+This is the current recovery mode of the cluster. There are two possible modes:
+.PP
+NORMAL \- The cluster is fully operational.
+.PP
+RECOVERY \- The cluster databases have all been frozen, pausing all services while the cluster awaits a recovery process to complete. A recovery process should finish within seconds. If a cluster is stuck in the RECOVERY state this would indicate a cluster malfunction which needs to be investigated.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+\fBRecovery master\fR
+.RS
+.PP
+This is the cluster node that is currently designated as the recovery master. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired.
+.RE
+.PP
+Example: ctdb status
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+Number of nodes:4
+vnn:0 11.1.2.200       OK (THIS NODE)
+vnn:1 11.1.2.201       OK
+vnn:2 11.1.2.202       OK
+vnn:3 11.1.2.203       OK
+Generation:1362079228
+Size:4
+hash:0 lmaster:0
+hash:1 lmaster:1
+hash:2 lmaster:2
+hash:3 lmaster:3
+Recovery mode:NORMAL (0)
+Recovery master:0
+      
+.fi
+.RE
+.SS "ping"
+.PP
+This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly.
+.PP
+Example: ctdb ping
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+response from 0 time=0.000054 sec  (3 clients)
+response from 1 time=0.000144 sec  (2 clients)
+response from 2 time=0.000105 sec  (2 clients)
+response from 3 time=0.000114 sec  (2 clients)
+      
+.fi
+.RE
+.SS "ip"
+.PP
+This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip.
+.PP
+Example: ctdb ip
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+Number of nodes:4
+12.1.1.1         0
+12.1.1.2         1
+12.1.1.3         2
+12.1.1.4         3
+      
+.fi
+.RE
+.SS "getvar <name>"
+.PP
+Get the runtime value of a tuneable variable.
+.PP
+Example: ctdb getvar MaxRedirectCount
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+MaxRedirectCount    = 3
+      
+.fi
+.RE
+.SS "setvar <name> <value>"
+.PP
+Set the runtime value of a tuneable variable.
+.PP
+Example: ctdb setvar MaxRedirectCount 5
+.SS "listvars"
+.PP
+List all tuneable variables.
+.PP
+Example: ctdb listvars
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+MaxRedirectCount    = 5
+SeqnumFrequency     = 1
+ControlTimeout      = 60
+TraverseTimeout     = 20
+KeepaliveInterval   = 2
+KeepaliveLimit      = 3
+MaxLACount          = 7
+RecoverTimeout      = 5
+RecoverInterval     = 1
+ElectionTimeout     = 3
+TakeoverTimeout     = 5
+MonitorInterval     = 15
+EventScriptTimeout  = 20
+RecoveryGracePeriod = 60
+RecoveryBanPeriod   = 300
+      
+.fi
+.RE
+.SS "statistics"
+.PP
+Collect statistics from the CTDB daemon about how many calls it has served.
+.PP
+Example: ctdb statistics
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+CTDB version 1
+ num_clients                        3
+ frozen                             0
+ recovering                         0
+ client_packets_sent           360489
+ client_packets_recv           360466
+ node_packets_sent             480931
+ node_packets_recv             240120
+ keepalive_packets_sent             4
+ keepalive_packets_recv             3
+ node
+     req_call                       2
+     reply_call                     2
+     req_dmaster                    0
+     reply_dmaster                  0
+     reply_error                    0
+     req_message                   42
+     req_control               120408
+     reply_control             360439
+ client
+     req_call                       2
+     req_message                   24
+     req_control               360440
+ timeouts
+     call                           0
+     control                        0
+     traverse                       0
+ total_calls                        2
+ pending_calls                      0
+ lockwait_calls                     0
+ pending_lockwait_calls             0
+ memory_used                     5040
+ max_hop_count                      0
+ max_call_latency                   4.948321 sec
+ max_lockwait_latency               0.000000 sec
+      
+.fi
+.RE
+.SS "statisticsreset"
+.PP
+This command is used to clear all statistics counters in a node.
+.PP
+Example: ctdb statisticsreset
+.SS "getdebug"
+.PP
+Get the current debug level for the node. the debug level controls what information is written to the log file.
+.SS "setdebug <debuglevel>"
+.PP
+Set the debug level of a node. This is a number between 0 and 9 and controls what information will be written to the logfile.
+.SS "getpid"
+.PP
+This command will return the process id of the ctdb daemon.
+.SS "disable"
+.PP
+This command is used to administratively disable a node in the cluster. A disabled node will still participate in the cluster and host clustered TDB records but its public ip address has been taken over by a different node and it no longer hosts any services.
+.SS "enable"
+.PP
+Re\-enable a node that has been administratively disabled.
+.SS "ban <bantime|0>"
+.PP
+Administratively ban a node for bantime seconds. A bantime of 0 means that the node should be permanently banned.
+.PP
+A banned node does not participate in the cluster and does not host any records for the clustered TDB. Its ip address has been taken over by an other node and no services are hosted.
+.PP
+Nodes are automatically banned if they are the cause of too many cluster recoveries.
+.SS "unban"
+.PP
+This command is used to unban a node that has either been administratively banned using the ban command or has been automatically banned by the recovery daemon.
+.SS "shutdown"
+.PP
+This command will shutdown a specific CTDB daemon.
+.SS "recover"
+.PP
+This command will trigger the recovery daemon to do a cluster recovery.
+.SS "killtcp <srcip:port> <dstip:port>"
+.PP
+This command will kill the specified TCP connection by issuing a TCP RST to the srcip:port endpoint.
+.SS "tickle <srcip:port> <dstip:port>"
+.PP
+This command will will send a TCP tickle to the source host for the specified TCP connection. A TCP tickle is a TCP ACK packet with an invalid sequence and acknowledge number and will when received by the source host result in it sending an immediate correct ACK back to the other end.
+.PP
+TCP tickles are useful to "tickle" clients after a IP failover has occured since this will make the client immediately recognize the TCP connection has been disrupted and that the client will need to reestablish. This greatly speeds up the time it takes for a client to detect and reestablish after an IP failover in the ctdb cluster.
+.SH "DEBUGGING COMMANDS"
+.PP
+These commands are primarily used for CTDB development and testing and should not be used for normal administration.
+.SS "process\-exists <pid>"
+.PP
+This command checks if a specific process exists on the CTDB host. This is mainly used by Samba to check if remote instances of samba are still running or not.
+.SS "getdbmap"
+.PP
+This command lists all clustered TDB databases that the CTDB daemon has attahced to.
+.PP
+Example: ctdb getdbmap
+.PP
+Example output:
+.sp
+.RS 4
+.nf
+Number of databases:4
+dbid:0x42fe72c5 name:locking.tdb path:/var/ctdb/locking.tdb.0
+dbid:0x1421fb78 name:brlock.tdb path:/var/ctdb/brlock.tdb.0
+dbid:0x17055d90 name:connections.tdb path:/var/ctdb/connections.tdb.0
+dbid:0xc0bdde6a name:sessionid.tdb path:/var/ctdb/sessionid.tdb.0
+      
+.fi
+.RE
+.SS "catdb <dbname>"
+.PP
+This command will dump a clustered TDB database to the screen. This is a debugging command.
+.SS "getmonmode"
+.PP
+This command returns the monutoring mode of a node. The monitoring mode is either ACTIVE or DISABLED. Normally a node will continously monitor that all other nodes that are expected are in fact connected and that they respond to commands.
+.PP
+ACTIVE \- This is the normal mode. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster.
+.PP
+DISABLED \- This node is not monitoring that other nodes are available. In this mode a node failure will not be detected and no recovery will be performed. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery.
+.SS "setmonmode <0|1>"
+.PP
+This command can be used to explicitely disable/enable monitoring mode on a node. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery. To do this, set monitoring mode to 0 on all nodes before attaching with GDB. Remember to set monitoring mode back to 1 afterwards.
+.SS "attach <dbname>"
+.PP
+This is a debugging command. This command will make the CTDB daemon create a new CTDB database and attach to it.
+.SS "dumpmemory"
+.PP
+This is a debugging command. This command will make the ctdb daemon to write a fill memory allocation map to the log file.
+.SS "freeze"
+.PP
+This command will lock all the local TDB databases causing clients that are accessing these TDBs such as samba3 to block until the databases are thawed.
+.PP
+This is primarily used by the recovery daemon to stop all samba daemons from accessing any databases while the database is recovered and rebuilt.
+.SS "thaw"
+.PP
+Thaw a previously frozen node.
+.SH "SEE ALSO"
+.PP
+ctdbd(1), onnode(1)
+\fI\%http://ctdb.samba.org/\fR
+.SH "COPYRIGHT/LICENSE"
+.sp
+.RS 4
+.nf
+Copyright (C) Andrew Tridgell 2007
+Copyright (C) Ronnie sahlberg 2007
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see http://www.gnu.org/licenses/.
+.fi
+.RE
diff --git a/source4/cluster/ctdb/doc/ctdb.1.html b/source4/cluster/ctdb/doc/ctdb.1.html
new file mode 100644
index 0000000000..165dc29781
--- /dev/null
+++ b/source4/cluster/ctdb/doc/ctdb.1.html
@@ -0,0 +1,277 @@
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>ctdb</title><meta name="generator" content="DocBook XSL Stylesheets V1.72.0"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" lang="en"><a name="ctdb.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>ctdb &#8212; clustered tdb database management utility</p></div><div class="refsynopsisdiv"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">ctdb [ OPTIONS ] COMMAND ...</code> </p></div><div class="cmdsynopsis"><p><code class="command">ctdb</code>  [-n &lt;node&gt;] [-Y] [-t &lt;timeout&gt;] [-? --help] [--usage] [-d --debug=&lt;INTEGER&gt;] [--socket=&lt;filename&gt;]</p></div></div><div class="refsect1" lang="en"><a name="id2488867"></a><h2>DESCRIPTION</h2><p>
+      ctdb is a utility to view and manage a ctdb cluster.
+    </p></div><div class="refsect1" lang="en"><a name="id2488877"></a><h2>OPTIONS</h2><div class="variablelist"><dl><dt><span class="term">-n &lt;node&gt;</span></dt><dd><p>
+            This specifies on which node to execute the command. Default is
+            to run the command on the deamon running on the local host.
+          </p></dd><dt><span class="term">-Y</span></dt><dd><p>
+            Produce output in machinereadable form for easier parsing by scripts. Not all commands support this option.
+          </p></dd><dt><span class="term">-t &lt;timeout&gt;</span></dt><dd><p>
+            How long should ctdb wait for a command to complete before timing out. Default is 3 seconds.
+          </p></dd><dt><span class="term">-? --help</span></dt><dd><p>
+            Print some help text to the screen.
+          </p></dd><dt><span class="term">--usage</span></dt><dd><p>
+            Print useage information to the screen.
+          </p></dd><dt><span class="term">-d --debug=&lt;debuglevel&gt;</span></dt><dd><p>
+            Change the debug level for the command. Default is 0.
+          </p></dd><dt><span class="term">--socket=&lt;filename&gt;</span></dt><dd><p>
+            Specify the socketname to use when connecting to the local ctdb 
+            daemon. The default is /tmp/ctdb.socket .
+          </p><p>
+            You only need to specify this parameter if you run multiple ctdb 
+            daemons on the same physical host and thus can not use the default
+            name for the domain socket.
+          </p></dd></dl></div></div><div class="refsect1" lang="en"><a name="id2488991"></a><h2>Administrative Commands</h2><p>
+      These are commands used to monitor and administrate a CTDB cluster.
+    </p><div class="refsect2" lang="en"><a name="id2489000"></a><h3>status</h3><p>
+        This command shows the current status of the ctdb node.
+      </p><div class="refsect3" lang="en"><a name="id2489009"></a><h4>node status</h4><p>
+          Node status reflects the current status of the node. There are four possible states:
+        </p><p>
+          OK - This node is fully functional.
+        </p><p>
+          DISCONNECTED - This node could not be connected through the network and is currently not parcipitating in the cluster. If there is a public IP address associated with this node it should have been taken over by a different node. No services are running on this node.
+        </p><p>
+          DISABLED - This node has been administratively disabled. This node is still functional and participates in the CTDB cluster but its IP addresses have been taken over by a different node and no services are currently being hosted.
+        </p><p>
+          UNHEALTHY - A service provided by this node is malfunctioning and should be investigated. The CTDB daemon itself is operational and participates in the cluster. Its public IP address has been taken over by a different node and no services are currnetly being hosted. All unhealthy nodes should be investigated and require an administrative action to rectify.
+        </p><p>
+          BANNED - This node failed too many recovery attempts and has been banned from participating in the cluster for a period of RecoveryBanPeriod seconds. Any public IP address has been taken over by other nodes. This node does not provide any services. All banned nodes should be investigated and require an administrative action to rectify. This node does not perticipate in the CTDB cluster but can still be communicated with. I.e. ctdb commands can be sent to it.
+        </p></div><div class="refsect3" lang="en"><a name="id2489061"></a><h4>generation</h4><p>
+          The generation id is a number that indicates the current generation 
+          of a cluster instance. Each time a cluster goes through a 
+          reconfiguration or a recovery its generation id will be changed.
+        </p></div><div class="refsect3" lang="en"><a name="id2490207"></a><h4>VNNMAP</h4><p>
+          The list of Virtual Node Numbers. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records.
+          Only nodes that are parcipitating in the vnnmap can become lmaster or dmaster for a database record.
+        </p></div><div class="refsect3" lang="en"><a name="id2490221"></a><h4>Recovery mode</h4><p>
+          This is the current recovery mode of the cluster. There are two possible modes:
+        </p><p>
+          NORMAL - The cluster is fully operational.
+        </p><p>
+          RECOVERY - The cluster databases have all been frozen, pausing all services while the cluster awaits a recovery process to complete. A recovery process should finish within seconds. If a cluster is stuck in the RECOVERY state this would indicate a cluster malfunction which needs to be investigated.
+        </p></div><div class="refsect3" lang="en"><a name="id2490244"></a><h4>Recovery master</h4><p>
+          This is the cluster node that is currently designated as the recovery master. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired.
+        </p></div><p>
+	Example: ctdb status
+      </p><p>Example output:</p><pre class="screen">
+Number of nodes:4
+vnn:0 11.1.2.200       OK (THIS NODE)
+vnn:1 11.1.2.201       OK
+vnn:2 11.1.2.202       OK
+vnn:3 11.1.2.203       OK
+Generation:1362079228
+Size:4
+hash:0 lmaster:0
+hash:1 lmaster:1
+hash:2 lmaster:2
+hash:3 lmaster:3
+Recovery mode:NORMAL (0)
+Recovery master:0
+      </pre></div><div class="refsect2" lang="en"><a name="id2490275"></a><h3>ping</h3><p>
+        This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly.
+      </p><p>
+	Example: ctdb ping
+      </p><p>
+	Example output:
+      </p><pre class="screen">
+response from 0 time=0.000054 sec  (3 clients)
+response from 1 time=0.000144 sec  (2 clients)
+response from 2 time=0.000105 sec  (2 clients)
+response from 3 time=0.000114 sec  (2 clients)
+      </pre></div><div class="refsect2" lang="en"><a name="id2490302"></a><h3>ip</h3><p>
+        This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip.
+      </p><p>
+	Example: ctdb ip
+      </p><p>
+	Example output:
+      </p><pre class="screen">
+Number of nodes:4
+12.1.1.1         0
+12.1.1.2         1
+12.1.1.3         2
+12.1.1.4         3
+      </pre></div><div class="refsect2" lang="en"><a name="id2490327"></a><h3>getvar &lt;name&gt;</h3><p>
+        Get the runtime value of a tuneable variable.
+      </p><p>
+	Example: ctdb getvar MaxRedirectCount
+      </p><p>
+	Example output:
+      </p><pre class="screen">
+MaxRedirectCount    = 3
+      </pre></div><div class="refsect2" lang="en"><a name="id2490350"></a><h3>setvar &lt;name&gt; &lt;value&gt;</h3><p>
+        Set the runtime value of a tuneable variable.
+      </p><p>
+	Example: ctdb setvar MaxRedirectCount 5
+      </p></div><div class="refsect2" lang="en"><a name="id2490365"></a><h3>listvars</h3><p>
+        List all tuneable variables.
+      </p><p>
+	Example: ctdb listvars
+      </p><p>
+	Example output:
+      </p><pre class="screen">
+MaxRedirectCount    = 5
+SeqnumFrequency     = 1
+ControlTimeout      = 60
+TraverseTimeout     = 20
+KeepaliveInterval   = 2
+KeepaliveLimit      = 3
+MaxLACount          = 7
+RecoverTimeout      = 5
+RecoverInterval     = 1
+ElectionTimeout     = 3
+TakeoverTimeout     = 5
+MonitorInterval     = 15
+EventScriptTimeout  = 20
+RecoveryGracePeriod = 60
+RecoveryBanPeriod   = 300
+      </pre></div><div class="refsect2" lang="en"><a name="id2490393"></a><h3>statistics</h3><p>
+        Collect statistics from the CTDB daemon about how many calls it has served.
+      </p><p>
+	Example: ctdb statistics
+      </p><p>
+	Example output:
+      </p><pre class="screen">
+CTDB version 1
+ num_clients                        3
+ frozen                             0
+ recovering                         0
+ client_packets_sent           360489
+ client_packets_recv           360466
+ node_packets_sent             480931
+ node_packets_recv             240120
+ keepalive_packets_sent             4
+ keepalive_packets_recv             3
+ node
+     req_call                       2
+     reply_call                     2
+     req_dmaster                    0
+     reply_dmaster                  0
+     reply_error                    0
+     req_message                   42
+     req_control               120408
+     reply_control             360439
+ client
+     req_call                       2
+     req_message                   24
+     req_control               360440
+ timeouts
+     call                           0
+     control                        0
+     traverse                       0
+ total_calls                        2
+ pending_calls                      0
+ lockwait_calls                     0
+ pending_lockwait_calls             0
+ memory_used                     5040
+ max_hop_count                      0
+ max_call_latency                   4.948321 sec
+ max_lockwait_latency               0.000000 sec
+      </pre></div><div class="refsect2" lang="en"><a name="id2490436"></a><h3>statisticsreset</h3><p>
+        This command is used to clear all statistics counters in a node.
+      </p><p>
+	Example: ctdb statisticsreset
+      </p></div><div class="refsect2" lang="en"><a name="id2490450"></a><h3>getdebug</h3><p>
+        Get the current debug level for the node. the debug level controls what information is written to the log file.
+      </p></div><div class="refsect2" lang="en"><a name="id2490461"></a><h3>setdebug &lt;debuglevel&gt;</h3><p>
+        Set the debug level of a node. This is a number between 0 and 9 and controls what information will be written to the logfile.
+      </p></div><div class="refsect2" lang="en"><a name="id2536585"></a><h3>getpid</h3><p>
+        This command will return the process id of the ctdb daemon.
+      </p></div><div class="refsect2" lang="en"><a name="id2536595"></a><h3>disable</h3><p>
+        This command is used to administratively disable a node in the cluster.
+        A disabled node will still participate in the cluster and host
+        clustered TDB records but its public ip address has been taken over by
+        a different node and it no longer hosts any services.
+      </p></div><div class="refsect2" lang="en"><a name="id2536613"></a><h3>enable</h3><p>
+        Re-enable a node that has been administratively disabled.
+      </p></div><div class="refsect2" lang="en"><a name="id2536623"></a><h3>ban &lt;bantime|0&gt;</h3><p>
+        Administratively ban a node for bantime seconds. A bantime of 0 means that the node should be permanently banned. 
+      </p><p>
+        A banned node does not participate in the cluster and does not host any records for the clustered TDB. Its ip address has been taken over by an other node and no services are hosted.
+      </p><p>
+        Nodes are automatically banned if they are the cause of too many
+        cluster recoveries.
+      </p></div><div class="refsect2" lang="en"><a name="id2536646"></a><h3>unban</h3><p>
+        This command is used to unban a node that has either been 
+        administratively banned using the ban command or has been automatically
+        banned by the recovery daemon.
+      </p></div><div class="refsect2" lang="en"><a name="id2536658"></a><h3>shutdown</h3><p>
+        This command will shutdown a specific CTDB daemon.
+      </p></div><div class="refsect2" lang="en"><a name="id2536668"></a><h3>recover</h3><p>
+        This command will trigger the recovery daemon to do a cluster
+        recovery.
+      </p></div><div class="refsect2" lang="en"><a name="id2536679"></a><h3>killtcp &lt;srcip:port&gt; &lt;dstip:port&gt;</h3><p>
+        This command will kill the specified TCP connection by issuing a
+        TCP RST to the srcip:port endpoint.
+      </p></div><div class="refsect2" lang="en"><a name="id2536690"></a><h3>tickle &lt;srcip:port&gt; &lt;dstip:port&gt;</h3><p>
+        This command will will send a TCP tickle to the source host for the
+        specified TCP connection.
+	A TCP tickle is a TCP ACK packet with an invalid sequence and 
+	acknowledge number and will when received by the source host result
+        in it sending an immediate correct ACK back to the other end.
+      </p><p>
+        TCP tickles are useful to "tickle" clients after a IP failover has 
+        occured since this will make the client immediately recognize the 
+        TCP connection has been disrupted and that the client will need
+        to reestablish. This greatly speeds up the time it takes for a client
+        to detect and reestablish after an IP failover in the ctdb cluster.
+      </p></div></div><div class="refsect1" lang="en"><a name="id2536716"></a><h2>Debugging Commands</h2><p>
+      These commands are primarily used for CTDB development and testing and
+      should not be used for normal administration.
+    </p><div class="refsect2" lang="en"><a name="id2536726"></a><h3>process-exists &lt;pid&gt;</h3><p>
+        This command checks if a specific process exists on the CTDB host. This is mainly used by Samba to check if remote instances of samba are still running or not.
+      </p></div><div class="refsect2" lang="en"><a name="id2536738"></a><h3>getdbmap</h3><p>
+        This command lists all clustered TDB databases that the CTDB daemon has attahced to.
+      </p><p>
+	Example: ctdb getdbmap
+      </p><p>
+	Example output:
+      </p><pre class="screen">
+Number of databases:4
+dbid:0x42fe72c5 name:locking.tdb path:/var/ctdb/locking.tdb.0
+dbid:0x1421fb78 name:brlock.tdb path:/var/ctdb/brlock.tdb.0
+dbid:0x17055d90 name:connections.tdb path:/var/ctdb/connections.tdb.0
+dbid:0xc0bdde6a name:sessionid.tdb path:/var/ctdb/sessionid.tdb.0
+      </pre></div><div class="refsect2" lang="en"><a name="id2536766"></a><h3>catdb &lt;dbname&gt;</h3><p>
+        This command will dump a clustered TDB database to the screen. This is a debugging command.
+      </p></div><div class="refsect2" lang="en"><a name="id2536777"></a><h3>getmonmode</h3><p>
+        This command returns the monutoring mode of a node. The monitoring mode is either ACTIVE or DISABLED. Normally a node will continously monitor that all other nodes that are expected are in fact connected and that they respond to commands.
+      </p><p>
+        ACTIVE - This is the normal mode. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster.
+      </p><p>
+        DISABLED - This node is not monitoring that other nodes are available. In this mode a node failure will not be detected and no recovery will be performed. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery.
+      </p></div><div class="refsect2" lang="en"><a name="id2536808"></a><h3>setmonmode &lt;0|1&gt;</h3><p>
+        This command can be used to explicitely disable/enable monitoring mode on a node. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery. To do this, set monitoring mode to 0 on all nodes before attaching with GDB. Remember to set monitoring mode back to 1 afterwards.
+      </p></div><div class="refsect2" lang="en"><a name="id2536823"></a><h3>attach &lt;dbname&gt;</h3><p>
+        This is a debugging command. This command will make the CTDB daemon create a new CTDB database and attach to it.
+      </p></div><div class="refsect2" lang="en"><a name="id2536835"></a><h3>dumpmemory</h3><p>
+        This is a debugging command. This command will make the ctdb daemon to write a fill memory allocation map to the log file.
+      </p></div><div class="refsect2" lang="en"><a name="id2536846"></a><h3>freeze</h3><p>
+        This command will lock all the local TDB databases causing clients 
+        that are accessing these TDBs such as samba3 to block until the
+        databases are thawed.
+      </p><p>
+        This is primarily used by the recovery daemon to stop all samba
+        daemons from accessing any databases while the database is recovered
+        and rebuilt.
+      </p></div><div class="refsect2" lang="en"><a name="id2536864"></a><h3>thaw</h3><p>
+        Thaw a previously frozen node.
+      </p></div></div><div class="refsect1" lang="en"><a name="id2536875"></a><h2>SEE ALSO</h2><p>
+      ctdbd(1), onnode(1)
+      <a href="http://ctdb.samba.org/" target="_top">http://ctdb.samba.org/</a>
+    </p></div><div class="refsect1" lang="en"><a name="id2536888"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br>
+Copyright (C) Andrew Tridgell 2007<br>
+Copyright (C) Ronnie sahlberg 2007<br>
+<br>
+This program is free software; you can redistribute it and/or modify<br>
+it under the terms of the GNU General Public License as published by<br>
+the Free Software Foundation; either version 3 of the License, or (at<br>
+your option) any later version.<br>
+<br>
+This program is distributed in the hope that it will be useful, but<br>
+WITHOUT ANY WARRANTY; without even the implied warranty of<br>
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU<br>
+General Public License for more details.<br>
+<br>
+You should have received a copy of the GNU General Public License<br>
+along with this program; if not, see http://www.gnu.org/licenses/.<br>
+</p></div></div></div></body></html>
diff --git a/source4/cluster/ctdb/doc/ctdb.1.xml b/source4/cluster/ctdb/doc/ctdb.1.xml
new file mode 100644
index 0000000000..6ed50afb67
--- /dev/null
+++ b/source4/cluster/ctdb/doc/ctdb.1.xml
@@ -0,0 +1,553 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE refentry PUBLIC "-//Samba-Team//DTD DocBook V4.2-Based Variant V1.0//EN" "http://www.samba.org/samba/DTD/samba-doc">
+<refentry id="ctdb.1">
+
+<refmeta>
+	<refentrytitle>ctdb</refentrytitle>
+	<manvolnum>1</manvolnum>
+</refmeta>
+
+
+<refnamediv>
+	<refname>ctdb</refname>
+        <refpurpose>clustered tdb database management utility</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+	<cmdsynopsis>
+		<command>ctdb [ OPTIONS ] COMMAND ...</command>
+	</cmdsynopsis>
+	
+	<cmdsynopsis>
+		<command>ctdb</command>
+		<arg choice="opt">-n &lt;node&gt;</arg>
+		<arg choice="opt">-Y</arg>
+		<arg choice="opt">-t &lt;timeout&gt;</arg>
+		<arg choice="opt">-? --help</arg>
+		<arg choice="opt">--usage</arg>
+		<arg choice="opt">-d --debug=&lt;INTEGER&gt;</arg>
+		<arg choice="opt">--socket=&lt;filename&gt;</arg>
+	</cmdsynopsis>
+	
+</refsynopsisdiv>
+
+  <refsect1><title>DESCRIPTION</title>
+    <para>
+      ctdb is a utility to view and manage a ctdb cluster.
+    </para>
+  </refsect1>
+
+
+  <refsect1>
+    <title>OPTIONS</title>
+
+    <variablelist>
+      <varlistentry><term>-n &lt;node&gt;</term>
+        <listitem>
+          <para>
+            This specifies on which node to execute the command. Default is
+            to run the command on the deamon running on the local host.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>-Y</term>
+        <listitem>
+          <para>
+            Produce output in machinereadable form for easier parsing by scripts. Not all commands support this option.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>-t &lt;timeout&gt;</term>
+        <listitem>
+          <para>
+            How long should ctdb wait for a command to complete before timing out. Default is 3 seconds.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>-? --help</term>
+        <listitem>
+          <para>
+            Print some help text to the screen.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--usage</term>
+        <listitem>
+          <para>
+            Print useage information to the screen.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>-d --debug=&lt;debuglevel&gt;</term>
+        <listitem>
+          <para>
+            Change the debug level for the command. Default is 0.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--socket=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            Specify the socketname to use when connecting to the local ctdb 
+            daemon. The default is /tmp/ctdb.socket .
+          </para>
+          <para>
+            You only need to specify this parameter if you run multiple ctdb 
+            daemons on the same physical host and thus can not use the default
+            name for the domain socket.
+          </para>
+        </listitem>
+      </varlistentry>
+
+    </variablelist>
+  </refsect1>
+
+
+  <refsect1><title>Administrative Commands</title>
+    <para>
+      These are commands used to monitor and administrate a CTDB cluster.
+    </para>
+
+    <refsect2><title>status</title>
+      <para>
+        This command shows the current status of the ctdb node.
+      </para>
+
+      <refsect3><title>node status</title>
+        <para>
+          Node status reflects the current status of the node. There are four possible states:
+        </para>
+        <para>
+          OK - This node is fully functional.
+        </para>
+        <para>
+          DISCONNECTED - This node could not be connected through the network and is currently not parcipitating in the cluster. If there is a public IP address associated with this node it should have been taken over by a different node. No services are running on this node.
+        </para>
+        <para>
+          DISABLED - This node has been administratively disabled. This node is still functional and participates in the CTDB cluster but its IP addresses have been taken over by a different node and no services are currently being hosted.
+        </para>
+        <para>
+          UNHEALTHY - A service provided by this node is malfunctioning and should be investigated. The CTDB daemon itself is operational and participates in the cluster. Its public IP address has been taken over by a different node and no services are currnetly being hosted. All unhealthy nodes should be investigated and require an administrative action to rectify.
+        </para>
+        <para>
+          BANNED - This node failed too many recovery attempts and has been banned from participating in the cluster for a period of RecoveryBanPeriod seconds. Any public IP address has been taken over by other nodes. This node does not provide any services. All banned nodes should be investigated and require an administrative action to rectify. This node does not perticipate in the CTDB cluster but can still be communicated with. I.e. ctdb commands can be sent to it.
+        </para>
+      </refsect3>
+
+      <refsect3><title>generation</title>
+        <para>
+          The generation id is a number that indicates the current generation 
+          of a cluster instance. Each time a cluster goes through a 
+          reconfiguration or a recovery its generation id will be changed.
+        </para>
+      </refsect3>
+
+      <refsect3><title>VNNMAP</title>
+        <para>
+          The list of Virtual Node Numbers. This is a list of all nodes that actively participates in the cluster and that share the workload of hosting the Clustered TDB database records.
+          Only nodes that are parcipitating in the vnnmap can become lmaster or dmaster for a database record.
+        </para>
+      </refsect3>
+
+      <refsect3><title>Recovery mode</title>
+        <para>
+          This is the current recovery mode of the cluster. There are two possible modes:
+        </para>
+        <para>
+          NORMAL - The cluster is fully operational.
+        </para>
+        <para>
+          RECOVERY - The cluster databases have all been frozen, pausing all services while the cluster awaits a recovery process to complete. A recovery process should finish within seconds. If a cluster is stuck in the RECOVERY state this would indicate a cluster malfunction which needs to be investigated.
+        </para>
+      </refsect3>
+
+      <refsect3><title>Recovery master</title>
+        <para>
+          This is the cluster node that is currently designated as the recovery master. This node is responsible of monitoring the consistency of the cluster and to perform the actual recovery process when reqired.
+        </para>
+      </refsect3>
+
+      <para>
+	Example: ctdb status
+      </para>
+      <para>Example output:</para>
+      <screen format="linespecific">
+Number of nodes:4
+vnn:0 11.1.2.200       OK (THIS NODE)
+vnn:1 11.1.2.201       OK
+vnn:2 11.1.2.202       OK
+vnn:3 11.1.2.203       OK
+Generation:1362079228
+Size:4
+hash:0 lmaster:0
+hash:1 lmaster:1
+hash:2 lmaster:2
+hash:3 lmaster:3
+Recovery mode:NORMAL (0)
+Recovery master:0
+      </screen>
+    </refsect2>
+
+    <refsect2><title>ping</title>
+      <para>
+        This command will "ping" all CTDB daemons in the cluster to verify that they are processing commands correctly.
+      </para>
+      <para>
+	Example: ctdb ping
+      </para>
+      <para>
+	Example output:
+      </para>
+      <screen format="linespecific">
+response from 0 time=0.000054 sec  (3 clients)
+response from 1 time=0.000144 sec  (2 clients)
+response from 2 time=0.000105 sec  (2 clients)
+response from 3 time=0.000114 sec  (2 clients)
+      </screen>
+    </refsect2>
+
+    <refsect2><title>ip</title>
+      <para>
+        This command will display the list of public addresses that are provided by the cluster and which physical node is currently serving this ip.
+      </para>
+      <para>
+	Example: ctdb ip
+      </para>
+      <para>
+	Example output:
+      </para>
+      <screen format="linespecific">
+Number of nodes:4
+12.1.1.1         0
+12.1.1.2         1
+12.1.1.3         2
+12.1.1.4         3
+      </screen>
+    </refsect2>
+
+    <refsect2><title>getvar &lt;name&gt;</title>
+      <para>
+        Get the runtime value of a tuneable variable.
+      </para>
+      <para>
+	Example: ctdb getvar MaxRedirectCount
+      </para>
+      <para>
+	Example output:
+      </para>
+      <screen format="linespecific">
+MaxRedirectCount    = 3
+      </screen>
+    </refsect2>
+
+    <refsect2><title>setvar &lt;name&gt; &lt;value&gt;</title>
+      <para>
+        Set the runtime value of a tuneable variable.
+      </para>
+      <para>
+	Example: ctdb setvar MaxRedirectCount 5
+      </para>
+    </refsect2>
+
+    <refsect2><title>listvars</title>
+      <para>
+        List all tuneable variables.
+      </para>
+      <para>
+	Example: ctdb listvars
+      </para>
+      <para>
+	Example output:
+      </para>
+      <screen format="linespecific">
+MaxRedirectCount    = 5
+SeqnumFrequency     = 1
+ControlTimeout      = 60
+TraverseTimeout     = 20
+KeepaliveInterval   = 2
+KeepaliveLimit      = 3
+MaxLACount          = 7
+RecoverTimeout      = 5
+RecoverInterval     = 1
+ElectionTimeout     = 3
+TakeoverTimeout     = 5
+MonitorInterval     = 15
+EventScriptTimeout  = 20
+RecoveryGracePeriod = 60
+RecoveryBanPeriod   = 300
+      </screen>
+    </refsect2>
+
+    <refsect2><title>statistics</title>
+      <para>
+        Collect statistics from the CTDB daemon about how many calls it has served.
+      </para>
+      <para>
+	Example: ctdb statistics
+      </para>
+      <para>
+	Example output:
+      </para>
+      <screen format="linespecific">
+CTDB version 1
+ num_clients                        3
+ frozen                             0
+ recovering                         0
+ client_packets_sent           360489
+ client_packets_recv           360466
+ node_packets_sent             480931
+ node_packets_recv             240120
+ keepalive_packets_sent             4
+ keepalive_packets_recv             3
+ node
+     req_call                       2
+     reply_call                     2
+     req_dmaster                    0
+     reply_dmaster                  0
+     reply_error                    0
+     req_message                   42
+     req_control               120408
+     reply_control             360439
+ client
+     req_call                       2
+     req_message                   24
+     req_control               360440
+ timeouts
+     call                           0
+     control                        0
+     traverse                       0
+ total_calls                        2
+ pending_calls                      0
+ lockwait_calls                     0
+ pending_lockwait_calls             0
+ memory_used                     5040
+ max_hop_count                      0
+ max_call_latency                   4.948321 sec
+ max_lockwait_latency               0.000000 sec
+      </screen>
+    </refsect2>
+
+    <refsect2><title>statisticsreset</title>
+      <para>
+        This command is used to clear all statistics counters in a node.
+      </para>
+      <para>
+	Example: ctdb statisticsreset
+      </para>
+    </refsect2>
+
+    <refsect2><title>getdebug</title>
+      <para>
+        Get the current debug level for the node. the debug level controls what information is written to the log file.
+      </para>
+    </refsect2>
+
+    <refsect2><title>setdebug &lt;debuglevel&gt;</title>
+      <para>
+        Set the debug level of a node. This is a number between 0 and 9 and controls what information will be written to the logfile.
+      </para>
+    </refsect2>
+
+    <refsect2><title>getpid</title>
+      <para>
+        This command will return the process id of the ctdb daemon.
+      </para>
+    </refsect2>
+
+    <refsect2><title>disable</title>
+      <para>
+        This command is used to administratively disable a node in the cluster.
+        A disabled node will still participate in the cluster and host
+        clustered TDB records but its public ip address has been taken over by
+        a different node and it no longer hosts any services.
+      </para>
+    </refsect2>
+
+    <refsect2><title>enable</title>
+      <para>
+        Re-enable a node that has been administratively disabled.
+      </para>
+    </refsect2>
+
+    <refsect2><title>ban &lt;bantime|0&gt;</title>
+      <para>
+        Administratively ban a node for bantime seconds. A bantime of 0 means that the node should be permanently banned. 
+      </para>
+      <para>
+        A banned node does not participate in the cluster and does not host any records for the clustered TDB. Its ip address has been taken over by an other node and no services are hosted.
+      </para>
+      <para>
+        Nodes are automatically banned if they are the cause of too many
+        cluster recoveries.
+      </para>
+    </refsect2>
+
+    <refsect2><title>unban</title>
+      <para>
+        This command is used to unban a node that has either been 
+        administratively banned using the ban command or has been automatically
+        banned by the recovery daemon.
+      </para>
+    </refsect2>
+
+    <refsect2><title>shutdown</title>
+      <para>
+        This command will shutdown a specific CTDB daemon.
+      </para>
+    </refsect2>
+
+    <refsect2><title>recover</title>
+      <para>
+        This command will trigger the recovery daemon to do a cluster
+        recovery.
+      </para>
+    </refsect2>
+
+    <refsect2><title>killtcp &lt;srcip:port&gt; &lt;dstip:port&gt;</title>
+      <para>
+        This command will kill the specified TCP connection by issuing a
+        TCP RST to the srcip:port endpoint.
+      </para>
+    </refsect2>
+
+    <refsect2><title>tickle &lt;srcip:port&gt; &lt;dstip:port&gt;</title>
+      <para>
+        This command will will send a TCP tickle to the source host for the
+        specified TCP connection.
+	A TCP tickle is a TCP ACK packet with an invalid sequence and 
+	acknowledge number and will when received by the source host result
+        in it sending an immediate correct ACK back to the other end.
+      </para>
+      <para>
+        TCP tickles are useful to "tickle" clients after a IP failover has 
+        occured since this will make the client immediately recognize the 
+        TCP connection has been disrupted and that the client will need
+        to reestablish. This greatly speeds up the time it takes for a client
+        to detect and reestablish after an IP failover in the ctdb cluster.
+      </para>
+    </refsect2>
+
+  </refsect1>
+
+
+  <refsect1><title>Debugging Commands</title>
+    <para>
+      These commands are primarily used for CTDB development and testing and
+      should not be used for normal administration.
+    </para>
+    <refsect2><title>process-exists &lt;pid&gt;</title>
+      <para>
+        This command checks if a specific process exists on the CTDB host. This is mainly used by Samba to check if remote instances of samba are still running or not.
+      </para>
+    </refsect2>
+
+    <refsect2><title>getdbmap</title>
+      <para>
+        This command lists all clustered TDB databases that the CTDB daemon has attahced to.
+      </para>
+      <para>
+	Example: ctdb getdbmap
+      </para>
+      <para>
+	Example output:
+      </para>
+      <screen format="linespecific">
+Number of databases:4
+dbid:0x42fe72c5 name:locking.tdb path:/var/ctdb/locking.tdb.0
+dbid:0x1421fb78 name:brlock.tdb path:/var/ctdb/brlock.tdb.0
+dbid:0x17055d90 name:connections.tdb path:/var/ctdb/connections.tdb.0
+dbid:0xc0bdde6a name:sessionid.tdb path:/var/ctdb/sessionid.tdb.0
+      </screen>
+    </refsect2>
+
+    <refsect2><title>catdb &lt;dbname&gt;</title>
+      <para>
+        This command will dump a clustered TDB database to the screen. This is a debugging command.
+      </para>
+    </refsect2>
+
+    <refsect2><title>getmonmode</title>
+      <para>
+        This command returns the monutoring mode of a node. The monitoring mode is either ACTIVE or DISABLED. Normally a node will continously monitor that all other nodes that are expected are in fact connected and that they respond to commands.
+      </para>
+      <para>
+        ACTIVE - This is the normal mode. The node is actively monitoring all other nodes, both that the transport is connected and also that the node responds to commands. If a node becomes unavailable, it will be marked as DISCONNECTED and a recovery is initiated to restore the cluster.
+      </para>
+      <para>
+        DISABLED - This node is not monitoring that other nodes are available. In this mode a node failure will not be detected and no recovery will be performed. This mode is useful when for debugging purposes one wants to attach GDB to a ctdb process but wants to prevent the rest of the cluster from marking this node as DISCONNECTED and do a recovery.
+      </para>
+    </refsect2>
+
+
+    <refsect2><title>setmonmode &lt;0|1&gt;</title>
+      <para>
+        This command can be used to explicitely disable/enable monitoring mode on a node. The main purpose is if one wants to attach GDB to a running ctdb daemon but wants to prevent the other nodes from marking it as DISCONNECTED and issuing a recovery. To do this, set monitoring mode to 0 on all nodes before attaching with GDB. Remember to set monitoring mode back to 1 afterwards.
+      </para>
+    </refsect2>
+
+    <refsect2><title>attach &lt;dbname&gt;</title>
+      <para>
+        This is a debugging command. This command will make the CTDB daemon create a new CTDB database and attach to it.
+      </para>
+    </refsect2>
+
+    <refsect2><title>dumpmemory</title>
+      <para>
+        This is a debugging command. This command will make the ctdb daemon to write a fill memory allocation map to the log file.
+      </para>
+    </refsect2>
+
+    <refsect2><title>freeze</title>
+      <para>
+        This command will lock all the local TDB databases causing clients 
+        that are accessing these TDBs such as samba3 to block until the
+        databases are thawed.
+      </para>
+      <para>
+        This is primarily used by the recovery daemon to stop all samba
+        daemons from accessing any databases while the database is recovered
+        and rebuilt.
+      </para>
+    </refsect2>
+
+    <refsect2><title>thaw</title>
+      <para>
+        Thaw a previously frozen node.
+      </para>
+    </refsect2>
+
+  </refsect1>
+
+
+  <refsect1><title>SEE ALSO</title>
+    <para>
+      ctdbd(1), onnode(1)
+      <ulink url="http://ctdb.samba.org/"/>
+    </para>
+  </refsect1>
+  <refsect1><title>COPYRIGHT/LICENSE</title>
+<literallayout>
+Copyright (C) Andrew Tridgell 2007
+Copyright (C) Ronnie sahlberg 2007
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see http://www.gnu.org/licenses/.
+</literallayout>
+  </refsect1>
+</refentry>
diff --git a/source4/cluster/ctdb/doc/ctdbd.1 b/source4/cluster/ctdb/doc/ctdbd.1
new file mode 100644
index 0000000000..682d813028
--- /dev/null
+++ b/source4/cluster/ctdb/doc/ctdbd.1
@@ -0,0 +1,213 @@
+.\"     Title: ctdbd
+.\"    Author: 
+.\" Generator: DocBook XSL Stylesheets v1.72.0 <http://docbook.sf.net/>
+.\"      Date: 07/10/2007
+.\"    Manual: 
+.\"    Source: 
+.\"
+.TH "CTDBD" "1" "07/10/2007" "" ""
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.SH "NAME"
+ctdbd \- The CTDB cluster daemon
+.SH "SYNOPSIS"
+.HP 6
+\fBctdbd\fR
+.HP 6
+\fBctdbd\fR {\-\-reclock=<filename>} {\-\-nlist=<filename>} {\-\-dbdir=<directory>} [\-?\ \-\-help] [\-\-usage] [\-i\ \-\-interactive] [\-\-public\-addresses=<filename>] [\-\-public\-interface=<interface>] [\-\-event\-script=<filename>] [\-\-logfile=<filename>] [\-\-listen=<address>] [\-\-transport=<STRING>] [\-\-socket=<filename>] [\-d\ \-\-debug=<INTEGER>] [\-\-torture]
+.SH "DESCRIPTION"
+.PP
+ctdbd is the main ctdb daemon.
+.PP
+ctdbd provides a clustered version of the TDB database with automatic rebuild/recovery of the databases upon nodefailures.
+.PP
+Combined with a cluster filesystem ctdbd provides a full HA environment for services such as clustered Samba and NFS as well as other services.
+.PP
+ctdbd provides monitoring of all nodes in the cluster and automatically reconfigures the cluster and recovers upon node failures.
+.PP
+ctdbd is the main component in clustered Samba that provides a high\-awailability load\-sharing CIFS server cluster.
+.SH "OPTIONS"
+.PP
+\-? \-\-help
+.RS 4
+Print some help text to the screen.
+.RE
+.PP
+\-\-usage
+.RS 4
+Print useage information to the screen.
+.RE
+.PP
+\-\-reclock=<filename>
+.RS 4
+This is the name of the lock file stored of the shared cluster filesystem that ctdbd uses to arbitrate which node has the role of recovery\-master. This file must be stored on shared storage.
+.RE
+.PP
+\-\-nlist=<filename>
+.RS 4
+This file contains a list of the private ip addresses of every node in the cluster. There is one line/ip address for each node. This file must be the same for all nodes in the cluster.
+.sp
+This file is usually /etc/ctdb/nodes .
+.RE
+.PP
+\-\-dbdir=<directory>
+.RS 4
+This is the directory on local storage where ctdbd keeps the local copy of the TDB databases. This directory is local for each node and should not be stored on the shared cluster filesystem.
+.sp
+This directory would usually be /var/ctdb .
+.RE
+.PP
+\-i \-\-interactive
+.RS 4
+By default ctdbd will detach itself from the shell and run in the background as a daemon. This option makes ctdbd to start in interactive mode.
+.RE
+.PP
+\-\-public_addresses=<filename>
+.RS 4
+When used with IP takeover this specifies a file containing the public ip addresses to use on the cluster. This file contains one entry for each node in the cluster.
+.sp
+This is usually the file /etc/ctdb/public_addresses
+.RE
+.PP
+\-\-public\-interface=<interface>
+.RS 4
+When used with IP takeover this option specifies which physical interface should be used to attach the public addresses to.
+.RE
+.PP
+\-\-event\-script=<filename>
+.RS 4
+This option is used to specify which events script that ctdbd will use to manage services when the cluster configuration changes.
+.sp
+This will normally be /etc/ctdb/events which is part of the ctdb distribution.
+.RE
+.PP
+\-\-logfile=<filename>
+.RS 4
+This is the file where ctdbd will write its log. This is usually /var/log/log.ctdb .
+.RE
+.PP
+\-\-listen=<address>
+.RS 4
+This specifies which ip address ctdb will bind to. By default ctdbd will bind to the first address it finds in the /etc/ctdb/nodes file and which is also present on the local system in which case you do not need to provide this option.
+.sp
+This option is only required when you want to run multiple ctdbd daemons/nodes on the same physical host in which case there would be multiple entries in /etc/ctdb/nodes what would match a local interface.
+.RE
+.PP
+\-\-transport=<STRING>
+.RS 4
+This option specifies which transport to use for ctdbd internode communications. The default is "tcp".
+.sp
+Suported transports are "tcp" and "infiniband".
+.RE
+.PP
+\-\-socket=<filename>
+.RS 4
+This specifies the name of the domain socket that ctdbd will create. This socket is used for local clients to attach to and communicate with the ctdbd daemon.
+.sp
+The default is /tmp/ctdb.socket . You only need to use this option if you plan to run multiple ctdbd daemons on the same physical host.
+.RE
+.PP
+\-d \-\-debug=<DEBUGLEVEL>
+.RS 4
+This option sets the debuglevel on the ctdbd daemon which controls what will be written to the logfile. The default is 0 which will only log important events and errors. A larger number will provide additional logging.
+.RE
+.PP
+\-\-torture
+.RS 4
+This option is only used for development and testing of ctdbd. It adds artificial errors and failures to the common codepaths in ctdbd to verify that ctdbd can recover correctly for failures.
+.sp
+You do NOT want to use this option unless you are developing and testing new functionality in ctdbd.
+.RE
+.SH "PRIVATE VS PUBLIC ADDRESSES"
+.PP
+When used for ip takeover in a HA environment, each node in a ctdb cluster has two ip addresses assigned to it. One private and one public.
+.SS "Private address"
+.PP
+This is the physical ip address of the node which is configured in linux and attached to a physical interface. This address uniquely identifies a physical node in the cluster and is the ip addresses that ctdbd will use to communicate with the ctdbd daemons on the other nodes in the cluster.
+.PP
+The private addresses are configured in /etc/ctdb/nodes (unless the \-\-nlist option is used) and contain one line for each node in the cluster. Each line contains the private ip address for one node in the cluster.
+.PP
+Each node is assigned an internal node number which corresponds to which line in the nodes file that has the local private address of the node.
+.PP
+Since the private addresses are only available to the network when the corresponding node is up and running you should not use these addresses for clients to connect to services provided by the cluster. Instead client applications should only attach to the public addresses since these are guaranteed to always be available.
+
+      Example /etc/ctdb/nodes for a four node cluster:
+      
+.sp
+.RS 4
+.nf
+        10.1.1.1
+        10.1.1.2
+        10.1.1.3
+        10.1.1.4
+      
+.fi
+.RE
+.SS "Public address"
+.PP
+A public address on the other hand is not attached to an interface. This address is managed by ctdbd itself and is attached/detached to a physical node at runtime. You should NOT have this address configured to an interface in linux. Let ctdbd manage these addresses.
+.PP
+The ctdb cluster will assign/reassign these public addresses across the available nodes in the cluster. When one node fails, its public address will be migrated to and taken over by a different node in the cluster to ensure that all public addresses are always available to clients.
+.PP
+These addresses are not physically attached to a specific node. The 'ctdb ip' command can be used to view the current assignment of public addresses and which physical node is currently serving it.
+.PP
+By default, each node will when operational always serve its primary public address which is the corresponding line for that node number in the public addresses file. I.e. as long as node X is available and fully oprational it will always be the node that serves the corresponding public address.
+.PP
+The list of public addresses also contain the netmask for that address. the reason for this is because ctdbd needs to know which mask to use when it adds/removes the address from a physical node. This netmask is also used by ctdbd when making decisions on which node should take over a public ip address for a failed node. A node will only be allowed to take over a public address from a different node IFF that public address resides in the same subnet as the primary public address for that node.
+
+      Example /etc/ctdb/public_addresses for a four node cluster:
+      
+.sp
+.RS 4
+.nf
+        11.1.1.1/24
+        11.1.1.2/24
+        11.1.2.1/24
+        11.1.2.2/24
+      
+.fi
+.RE
+.PP
+In this example, if node 3 fails, its public address can be taken over by node 2 since node 2 is on the same subnet as 3 but not by node 0 or node 1 since node 0 and 1 are both on a different subnet from node 3.
+.SH "NODE STATUS"
+.PP
+The current status of each node in the cluster can be viewed by the 'ctdb status' command.
+.PP
+There are five possible for a node.
+.PP
+OK \- This node is fully functional.
+.PP
+DISCONNECTED \- This node could not be connected through the network and is currently not parcipitating in the cluster. If there is a public IP address associated with this node it should have been taken over by a different node. No services are running on this node.
+.PP
+DISABLED \- This node has been administratively disabled. This node is still functional and participates in the CTDB cluster but its IP addresses have been taken over by a different node and no services are currently being hosted.
+.PP
+UNHEALTHY \- A service provided by this node is malfunctioning and should be investigated. The CTDB daemon itself is operational and participates in the cluster. Its public IP address has been taken over by a different node and no services are currently being hosted. All unhealthy nodes should be investigated and require an administrative action to rectify.
+.PP
+BANNED \- This node failed too many recovery attempts and has been banned from participating in the cluster for a period of RecoveryBanPeriod seconds. Any public IP address has been taken over by other nodes. This node does not provide any services. All banned nodes should be investigated and require an administrative action to rectify. This node does not perticipate in the CTDB cluster but can still be communicated with. I.e. ctdb commands can be sent to it.
+.SH "SEE ALSO"
+.PP
+ctdb(1), onnode(1)
+\fI\%http://ctdb.samba.org/\fR
+.SH "COPYRIGHT/LICENSE"
+.sp
+.RS 4
+.nf
+Copyright (C) Andrew Tridgell 2007
+Copyright (C) Ronnie sahlberg 2007
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see http://www.gnu.org/licenses/.
+.fi
+.RE
diff --git a/source4/cluster/ctdb/doc/ctdbd.1.html b/source4/cluster/ctdb/doc/ctdbd.1.html
new file mode 100644
index 0000000000..49d2fdc468
--- /dev/null
+++ b/source4/cluster/ctdb/doc/ctdbd.1.html
@@ -0,0 +1,182 @@
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>ctdbd</title><meta name="generator" content="DocBook XSL Stylesheets V1.72.0"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" lang="en"><a name="ctdbd.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>ctdbd &#8212; The CTDB cluster daemon</p></div><div class="refsynopsisdiv"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">ctdbd</code> </p></div><div class="cmdsynopsis"><p><code class="command">ctdbd</code>  {--reclock=&lt;filename&gt;} {--nlist=&lt;filename&gt;} {--dbdir=&lt;directory&gt;} [-? --help] [--usage] [-i --interactive] [--public-addresses=&lt;filename&gt;] [--public-interface=&lt;interface&gt;] [--event-script=&lt;filename&gt;] [--logfile=&lt;filename&gt;] [--listen=&lt;address&gt;] [--transport=&lt;STRING&gt;] [--socket=&lt;filename&gt;] [-d --debug=&lt;INTEGER&gt;] [--torture]</p></div></div><div class="refsect1" lang="en"><a name="id2488930"></a><h2>DESCRIPTION</h2><p>
+      ctdbd is the main ctdb daemon.
+    </p><p>
+      ctdbd provides a clustered version of the TDB database with automatic rebuild/recovery of the databases upon nodefailures.
+    </p><p>
+      Combined with a cluster filesystem ctdbd provides a full HA environment for services such as clustered Samba and NFS as well as other services.
+    </p><p>
+      ctdbd provides monitoring of all nodes in the cluster and automatically reconfigures the cluster and recovers upon node failures.
+    </p><p>
+      ctdbd is the main component in clustered Samba that provides a high-awailability load-sharing CIFS server cluster.
+    </p></div><div class="refsect1" lang="en"><a name="id2488962"></a><h2>OPTIONS</h2><div class="variablelist"><dl><dt><span class="term">-? --help</span></dt><dd><p>
+            Print some help text to the screen.
+          </p></dd><dt><span class="term">--usage</span></dt><dd><p>
+            Print useage information to the screen.
+          </p></dd><dt><span class="term">--reclock=&lt;filename&gt;</span></dt><dd><p>
+            This is the name of the lock file stored of the shared cluster filesystem that ctdbd uses to arbitrate which node has the role of recovery-master.
+            This file must be stored on shared storage.
+          </p></dd><dt><span class="term">--nlist=&lt;filename&gt;</span></dt><dd><p>
+            This file contains a list of the private ip addresses of every node in the cluster. There is one line/ip address for each node. This file must be the same for all nodes in the cluster.
+          </p><p>
+            This file is usually /etc/ctdb/nodes .
+          </p></dd><dt><span class="term">--dbdir=&lt;directory&gt;</span></dt><dd><p>
+            This is the directory on local storage where ctdbd keeps the local
+            copy of the TDB databases. This directory is local for each node and should not be stored on the shared cluster filesystem.
+          </p><p>
+            This directory would usually be /var/ctdb .
+          </p></dd><dt><span class="term">-i --interactive</span></dt><dd><p>
+            By default ctdbd will detach itself from the shell and run in
+            the background as a daemon. This option makes ctdbd to start in interactive mode.
+          </p></dd><dt><span class="term">--public_addresses=&lt;filename&gt;</span></dt><dd><p>
+            When used with IP takeover this specifies a file containing the public ip addresses to use on the cluster. This file contains one entry for each node in the cluster.
+          </p><p>
+            This is usually the file /etc/ctdb/public_addresses
+          </p></dd><dt><span class="term">--public-interface=&lt;interface&gt;</span></dt><dd><p>
+            When used with IP takeover this option specifies which physical interface should be used to attach the public addresses to.
+          </p></dd><dt><span class="term">--event-script=&lt;filename&gt;</span></dt><dd><p>
+            This option is used to specify which events script that ctdbd will
+            use to manage services when the cluster configuration changes.
+          </p><p>
+            This will normally be /etc/ctdb/events which is part of the ctdb distribution.
+          </p></dd><dt><span class="term">--logfile=&lt;filename&gt;</span></dt><dd><p>
+            This is the file where ctdbd will write its log. This is usually /var/log/log.ctdb .
+          </p></dd><dt><span class="term">--listen=&lt;address&gt;</span></dt><dd><p>
+            This specifies which ip address ctdb will bind to. By default ctdbd will bind to the first address it finds in the /etc/ctdb/nodes file and which is also present on the local system in which case you do not need to provide this option.
+          </p><p>
+            This option is only required when you want to run multiple ctdbd daemons/nodes on the same physical host in which case there would be multiple entries in /etc/ctdb/nodes what would match a local interface.
+          </p></dd><dt><span class="term">--transport=&lt;STRING&gt;</span></dt><dd><p>
+            This option specifies which transport to use for ctdbd internode communications. The default is "tcp".
+          </p><p>
+            Suported transports are "tcp" and "infiniband".
+          </p></dd><dt><span class="term">--socket=&lt;filename&gt;</span></dt><dd><p>
+            This specifies the name of the domain socket that ctdbd will create. This socket is used for local clients to attach to and communicate with the ctdbd daemon.
+          </p><p>
+            The default is /tmp/ctdb.socket . You only need to use this option if you plan to run multiple ctdbd daemons on the same physical host.
+          </p></dd><dt><span class="term">-d --debug=&lt;DEBUGLEVEL&gt;</span></dt><dd><p>
+            This option sets the debuglevel on the ctdbd daemon which controls what will be written to the logfile. The default is 0 which will only log important events and errors. A larger number will provide additional logging.
+          </p></dd><dt><span class="term">--torture</span></dt><dd><p>
+            This option is only used for development and testing of ctdbd. It adds artificial errors and failures to the common codepaths in ctdbd to verify that ctdbd can recover correctly for failures.
+          </p><p>
+            You do NOT want to use this option unless you are developing and testing new functionality in ctdbd.
+          </p></dd></dl></div></div><div class="refsect1" lang="en"><a name="id2490376"></a><h2>Private vs Public addresses</h2><p>
+      When used for ip takeover in a HA environment, each node in a ctdb 
+      cluster has two ip addresses assigned to it. One private and one public.
+    </p><div class="refsect2" lang="en"><a name="id2490386"></a><h3>Private address</h3><p>
+        This is the physical ip address of the node which is configured in 
+        linux and attached to a physical interface. This address uniquely
+        identifies a physical node in the cluster and is the ip addresses
+        that ctdbd will use to communicate with the ctdbd daemons on the
+        other nodes in the cluster.
+      </p><p>
+        The private addresses are configured in /etc/ctdb/nodes 
+        (unless the --nlist option is used) and contain one line for each 
+        node in the cluster. Each line contains the private ip address for one
+        node in the cluster.
+      </p><p>
+        Each node is assigned an internal node number which corresponds to 
+        which line in the nodes file that has the local private address
+        of the node.
+      </p><p>
+        Since the private addresses are only available to the network when the
+        corresponding node is up and running you should not use these addresses
+        for clients to connect to services provided by the cluster. Instead
+        client applications should only attach to the public addresses since
+        these are guaranteed to always be available.
+      </p>
+      Example /etc/ctdb/nodes for a four node cluster:
+      <pre class="screen">
+        10.1.1.1
+        10.1.1.2
+        10.1.1.3
+        10.1.1.4
+      </pre></div><div class="refsect2" lang="en"><a name="id2490432"></a><h3>Public address</h3><p>
+        A public address on the other hand is not attached to an interface.
+        This address is managed by ctdbd itself and is attached/detached to
+        a physical node at runtime. You should NOT have this address configured
+        to an interface in linux. Let ctdbd manage these addresses.
+      </p><p>
+        The ctdb cluster will assign/reassign these public addresses across the
+        available nodes in the cluster. When one node fails, its public address
+        will be migrated to and taken over by a different node in the cluster
+        to ensure that all public addresses are always available to clients.
+      </p><p>
+        These addresses are not physically attached to a specific node. 
+        The 'ctdb ip' command can be used to view the current assignment of 
+        public addresses and which physical node is currently serving it.
+      </p><p>
+        By default, each node will when operational always serve its primary 
+        public address which is the corresponding line for that node number 
+        in the public addresses file. I.e. as long as node X is available and
+        fully oprational it will always be the node that serves the 
+        corresponding public address.
+      </p><p>
+        The list of public addresses also contain the netmask for that address.
+        the reason for this is because ctdbd needs to know which mask to use
+        when it adds/removes the address from a physical node. This netmask
+        is also used by ctdbd when making decisions on which node should take
+        over a public ip address for a failed node.
+        A node will only be allowed to take over a public address from a
+        different node IFF that public address resides in the same subnet
+        as the primary public address for that node.
+      </p>
+      Example /etc/ctdb/public_addresses for a four node cluster:
+      <pre class="screen">
+        11.1.1.1/24
+        11.1.1.2/24
+        11.1.2.1/24
+        11.1.2.2/24
+      </pre><p>
+        In this example, if node 3 fails, its public address can be taken over 
+        by node 2 since node 2 is on the same subnet as 3 but not by node 0 or 
+        node 1 since node 0 and 1 are both on a different subnet from node 3.
+      </p></div></div><div class="refsect1" lang="en"><a name="id2536612"></a><h2>Node status</h2><p>
+      The current status of each node in the cluster can be viewed by the 
+      'ctdb status' command.
+    </p><p>
+      There are five possible for a node.
+    </p><p>
+      OK - This node is fully functional.
+    </p><p>
+      DISCONNECTED - This node could not be connected through the network 
+      and is currently not parcipitating in the cluster. If there is a 
+      public IP address associated with this node it should have been taken 
+      over by a different node. No services are running on this node.
+    </p><p>
+      DISABLED - This node has been administratively disabled. This node is 
+      still functional and participates in the CTDB cluster but its IP 
+      addresses have been taken over by a different node and no services are 
+      currently being hosted.
+    </p><p>
+      UNHEALTHY - A service provided by this node is malfunctioning and should 
+      be investigated. The CTDB daemon itself is operational and participates 
+      in the cluster. Its public IP address has been taken over by a different 
+      node and no services are currently being hosted. All unhealthy nodes 
+      should be investigated and require an administrative action to rectify.
+    </p><p>
+      BANNED - This node failed too many recovery attempts and has been banned 
+      from participating in the cluster for a period of RecoveryBanPeriod 
+      seconds. Any public IP address has been taken over by other nodes. This 
+      node does not provide any services. All banned nodes should be 
+      investigated and require an administrative action to rectify. This node 
+      does not perticipate in the CTDB cluster but can still be communicated 
+      with. I.e. ctdb commands can be sent to it.
+    </p></div><div class="refsect1" lang="en"><a name="id2536669"></a><h2>SEE ALSO</h2><p>
+      ctdb(1), onnode(1)
+      <a href="http://ctdb.samba.org/" target="_top">http://ctdb.samba.org/</a>
+    </p></div><div class="refsect1" lang="en"><a name="id2536682"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br>
+Copyright (C) Andrew Tridgell 2007<br>
+Copyright (C) Ronnie sahlberg 2007<br>
+<br>
+This program is free software; you can redistribute it and/or modify<br>
+it under the terms of the GNU General Public License as published by<br>
+the Free Software Foundation; either version 3 of the License, or (at<br>
+your option) any later version.<br>
+<br>
+This program is distributed in the hope that it will be useful, but<br>
+WITHOUT ANY WARRANTY; without even the implied warranty of<br>
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU<br>
+General Public License for more details.<br>
+<br>
+You should have received a copy of the GNU General Public License<br>
+along with this program; if not, see http://www.gnu.org/licenses/.<br>
+</p></div></div></div></body></html>
diff --git a/source4/cluster/ctdb/doc/ctdbd.1.xml b/source4/cluster/ctdb/doc/ctdbd.1.xml
new file mode 100644
index 0000000000..cbff8d8b5b
--- /dev/null
+++ b/source4/cluster/ctdb/doc/ctdbd.1.xml
@@ -0,0 +1,379 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE refentry PUBLIC "-//Samba-Team//DTD DocBook V4.2-Based Variant V1.0//EN" "http://www.samba.org/samba/DTD/samba-doc">
+<refentry id="ctdbd.1">
+
+<refmeta>
+	<refentrytitle>ctdbd</refentrytitle>
+	<manvolnum>1</manvolnum>
+</refmeta>
+
+
+<refnamediv>
+	<refname>ctdbd</refname>
+	<refpurpose>The CTDB cluster daemon</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+	<cmdsynopsis>
+		<command>ctdbd</command>
+	</cmdsynopsis>
+	
+	<cmdsynopsis>
+		<command>ctdbd</command>
+		<arg choice="req">--reclock=&lt;filename&gt;</arg>
+		<arg choice="req">--nlist=&lt;filename&gt;</arg>
+		<arg choice="req">--dbdir=&lt;directory&gt;</arg>
+		<arg choice="opt">-? --help</arg>
+		<arg choice="opt">--usage</arg>
+		<arg choice="opt">-i --interactive</arg>
+		<arg choice="opt">--public-addresses=&lt;filename&gt;</arg>
+		<arg choice="opt">--public-interface=&lt;interface&gt;</arg>
+		<arg choice="opt">--event-script=&lt;filename&gt;</arg>
+		<arg choice="opt">--logfile=&lt;filename&gt;</arg>
+		<arg choice="opt">--listen=&lt;address&gt;</arg>
+		<arg choice="opt">--transport=&lt;STRING&gt;</arg>
+		<arg choice="opt">--socket=&lt;filename&gt;</arg>
+		<arg choice="opt">-d --debug=&lt;INTEGER&gt;</arg>
+		<arg choice="opt">--torture</arg>
+	</cmdsynopsis>
+	
+</refsynopsisdiv>
+
+  <refsect1><title>DESCRIPTION</title>
+    <para>
+      ctdbd is the main ctdb daemon.
+    </para>
+    <para>
+      ctdbd provides a clustered version of the TDB database with automatic rebuild/recovery of the databases upon nodefailures.
+    </para>
+    <para>
+      Combined with a cluster filesystem ctdbd provides a full HA environment for services such as clustered Samba and NFS as well as other services.
+    </para>
+    <para>
+      ctdbd provides monitoring of all nodes in the cluster and automatically reconfigures the cluster and recovers upon node failures.
+    </para>
+    <para>
+      ctdbd is the main component in clustered Samba that provides a high-awailability load-sharing CIFS server cluster.
+    </para>
+  </refsect1>
+
+
+  <refsect1>
+    <title>OPTIONS</title>
+
+    <variablelist>
+      <varlistentry><term>-? --help</term>
+        <listitem>
+          <para>
+            Print some help text to the screen.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--usage</term>
+        <listitem>
+          <para>
+            Print useage information to the screen.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--reclock=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            This is the name of the lock file stored of the shared cluster filesystem that ctdbd uses to arbitrate which node has the role of recovery-master.
+            This file must be stored on shared storage.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--nlist=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            This file contains a list of the private ip addresses of every node in the cluster. There is one line/ip address for each node. This file must be the same for all nodes in the cluster.
+          </para>
+          <para>
+            This file is usually /etc/ctdb/nodes .
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--dbdir=&lt;directory&gt;</term>
+        <listitem>
+          <para>
+            This is the directory on local storage where ctdbd keeps the local
+            copy of the TDB databases. This directory is local for each node and should not be stored on the shared cluster filesystem.
+          </para>
+          <para>
+            This directory would usually be /var/ctdb .
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>-i --interactive</term>
+        <listitem>
+          <para>
+            By default ctdbd will detach itself from the shell and run in
+            the background as a daemon. This option makes ctdbd to start in interactive mode.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--public_addresses=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            When used with IP takeover this specifies a file containing the public ip addresses to use on the cluster. This file contains one entry for each node in the cluster.
+          </para>
+          <para>
+            This is usually the file /etc/ctdb/public_addresses
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--public-interface=&lt;interface&gt;</term>
+        <listitem>
+          <para>
+            When used with IP takeover this option specifies which physical interface should be used to attach the public addresses to.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--event-script=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            This option is used to specify which events script that ctdbd will
+            use to manage services when the cluster configuration changes.
+          </para>
+          <para>
+            This will normally be /etc/ctdb/events which is part of the ctdb distribution.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--logfile=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            This is the file where ctdbd will write its log. This is usually /var/log/log.ctdb .
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--listen=&lt;address&gt;</term>
+        <listitem>
+          <para>
+            This specifies which ip address ctdb will bind to. By default ctdbd will bind to the first address it finds in the /etc/ctdb/nodes file and which is also present on the local system in which case you do not need to provide this option.
+          </para>
+          <para>
+            This option is only required when you want to run multiple ctdbd daemons/nodes on the same physical host in which case there would be multiple entries in /etc/ctdb/nodes what would match a local interface.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--transport=&lt;STRING&gt;</term>
+        <listitem>
+          <para>
+            This option specifies which transport to use for ctdbd internode communications. The default is "tcp".
+          </para>
+          <para>
+            Suported transports are "tcp" and "infiniband".
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--socket=&lt;filename&gt;</term>
+        <listitem>
+          <para>
+            This specifies the name of the domain socket that ctdbd will create. This socket is used for local clients to attach to and communicate with the ctdbd daemon.
+          </para>
+          <para>
+            The default is /tmp/ctdb.socket . You only need to use this option if you plan to run multiple ctdbd daemons on the same physical host.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>-d --debug=&lt;DEBUGLEVEL&gt;</term>
+        <listitem>
+          <para>
+            This option sets the debuglevel on the ctdbd daemon which controls what will be written to the logfile. The default is 0 which will only log important events and errors. A larger number will provide additional logging.
+          </para>
+        </listitem>
+      </varlistentry>
+
+      <varlistentry><term>--torture</term>
+        <listitem>
+          <para>
+            This option is only used for development and testing of ctdbd. It adds artificial errors and failures to the common codepaths in ctdbd to verify that ctdbd can recover correctly for failures.
+          </para>
+          <para>
+            You do NOT want to use this option unless you are developing and testing new functionality in ctdbd.
+          </para>
+        </listitem>
+      </varlistentry>
+    </variablelist>
+  </refsect1>
+
+
+  <refsect1><title>Private vs Public addresses</title>
+    <para>
+      When used for ip takeover in a HA environment, each node in a ctdb 
+      cluster has two ip addresses assigned to it. One private and one public.
+    </para>
+
+    <refsect2><title>Private address</title>
+      <para>
+        This is the physical ip address of the node which is configured in 
+        linux and attached to a physical interface. This address uniquely
+        identifies a physical node in the cluster and is the ip addresses
+        that ctdbd will use to communicate with the ctdbd daemons on the
+        other nodes in the cluster.
+      </para>
+      <para>
+        The private addresses are configured in /etc/ctdb/nodes 
+        (unless the --nlist option is used) and contain one line for each 
+        node in the cluster. Each line contains the private ip address for one
+        node in the cluster.
+      </para>
+      <para>
+        Each node is assigned an internal node number which corresponds to 
+        which line in the nodes file that has the local private address
+        of the node.
+      </para>
+      <para>
+        Since the private addresses are only available to the network when the
+        corresponding node is up and running you should not use these addresses
+        for clients to connect to services provided by the cluster. Instead
+        client applications should only attach to the public addresses since
+        these are guaranteed to always be available.
+      </para>
+      Example /etc/ctdb/nodes for a four node cluster:
+      <screen format="linespecific">
+        10.1.1.1
+        10.1.1.2
+        10.1.1.3
+        10.1.1.4
+      </screen>
+    </refsect2>
+    <refsect2><title>Public address</title>
+      <para>
+        A public address on the other hand is not attached to an interface.
+        This address is managed by ctdbd itself and is attached/detached to
+        a physical node at runtime. You should NOT have this address configured
+        to an interface in linux. Let ctdbd manage these addresses.
+      </para>
+      <para>
+        The ctdb cluster will assign/reassign these public addresses across the
+        available nodes in the cluster. When one node fails, its public address
+        will be migrated to and taken over by a different node in the cluster
+        to ensure that all public addresses are always available to clients.
+      </para>
+      <para>
+        These addresses are not physically attached to a specific node. 
+        The 'ctdb ip' command can be used to view the current assignment of 
+        public addresses and which physical node is currently serving it.
+      </para>
+      <para>
+        By default, each node will when operational always serve its primary 
+        public address which is the corresponding line for that node number 
+        in the public addresses file. I.e. as long as node X is available and
+        fully oprational it will always be the node that serves the 
+        corresponding public address.
+      </para>
+      <para>
+        The list of public addresses also contain the netmask for that address.
+        the reason for this is because ctdbd needs to know which mask to use
+        when it adds/removes the address from a physical node. This netmask
+        is also used by ctdbd when making decisions on which node should take
+        over a public ip address for a failed node.
+        A node will only be allowed to take over a public address from a
+        different node IFF that public address resides in the same subnet
+        as the primary public address for that node.
+      </para>
+      Example /etc/ctdb/public_addresses for a four node cluster:
+      <screen format="linespecific">
+        11.1.1.1/24
+        11.1.1.2/24
+        11.1.2.1/24
+        11.1.2.2/24
+      </screen>
+      <para>
+        In this example, if node 3 fails, its public address can be taken over 
+        by node 2 since node 2 is on the same subnet as 3 but not by node 0 or 
+        node 1 since node 0 and 1 are both on a different subnet from node 3.
+      </para>
+    </refsect2>
+  </refsect1>
+
+
+  <refsect1><title>Node status</title>
+    <para>
+      The current status of each node in the cluster can be viewed by the 
+      'ctdb status' command.
+    </para>
+    <para>
+      There are five possible for a node.
+    </para>
+
+    <para>
+      OK - This node is fully functional.
+    </para>
+    
+    <para>
+      DISCONNECTED - This node could not be connected through the network 
+      and is currently not parcipitating in the cluster. If there is a 
+      public IP address associated with this node it should have been taken 
+      over by a different node. No services are running on this node.
+    </para>
+ 
+    <para>
+      DISABLED - This node has been administratively disabled. This node is 
+      still functional and participates in the CTDB cluster but its IP 
+      addresses have been taken over by a different node and no services are 
+      currently being hosted.
+    </para>
+   
+    <para>
+      UNHEALTHY - A service provided by this node is malfunctioning and should 
+      be investigated. The CTDB daemon itself is operational and participates 
+      in the cluster. Its public IP address has been taken over by a different 
+      node and no services are currently being hosted. All unhealthy nodes 
+      should be investigated and require an administrative action to rectify.
+    </para>
+    
+    <para>
+      BANNED - This node failed too many recovery attempts and has been banned 
+      from participating in the cluster for a period of RecoveryBanPeriod 
+      seconds. Any public IP address has been taken over by other nodes. This 
+      node does not provide any services. All banned nodes should be 
+      investigated and require an administrative action to rectify. This node 
+      does not perticipate in the CTDB cluster but can still be communicated 
+      with. I.e. ctdb commands can be sent to it.
+    </para>
+  </refsect1>
+
+
+  <refsect1><title>SEE ALSO</title>
+    <para>
+      ctdb(1), onnode(1)
+      <ulink url="http://ctdb.samba.org/"/>
+    </para>
+  </refsect1>
+  <refsect1><title>COPYRIGHT/LICENSE</title>
+<literallayout>
+Copyright (C) Andrew Tridgell 2007
+Copyright (C) Ronnie sahlberg 2007
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see http://www.gnu.org/licenses/.
+</literallayout>
+  </refsect1>
+</refentry>
diff --git a/source4/cluster/ctdb/doc/onnode.1 b/source4/cluster/ctdb/doc/onnode.1
new file mode 100644
index 0000000000..926df4d145
--- /dev/null
+++ b/source4/cluster/ctdb/doc/onnode.1
@@ -0,0 +1,79 @@
+.\"     Title: onnode
+.\"    Author: 
+.\" Generator: DocBook XSL Stylesheets v1.72.0 <http://docbook.sf.net/>
+.\"      Date: 07/10/2007
+.\"    Manual: 
+.\"    Source: 
+.\"
+.TH "ONNODE" "1" "07/10/2007" "" ""
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.SH "NAME"
+onnode \- run commands on ctdb nodes
+.SH "SYNOPSIS"
+.HP 24
+\fBonnode NODE COMMAND ...\fR
+.SH "DESCRIPTION"
+.PP
+onnode is a utility to run commands on a specific node of a CTDB cluster, or on all nodes.
+.PP
+The NODE option specifies which node to run a command on. You can specify a numeric node number (from 0 to N\-1) or the special node 'all'.
+.PP
+The COMMAND can be any shell command. The onnode utility uses ssh or rsh to connect to the remote nodes and run the command.
+.PP
+If the COMMAND starts with a /, then the command is run via the 'at' service. Otherwise the command is run in the foreground.
+.SH "EXAMPLES"
+.PP
+The following command would show the process ID of ctdb on all nodes
+.sp
+.RS 4
+.nf
+onnode all pidof ctdbd
+      
+.fi
+.RE
+.PP
+The following command would show the last 5 lines of log on each node, preceded by the nodes hostname
+.sp
+.RS 4
+.nf
+onnode all "hostname; tail \-5 /var/log/log.ctdb"
+      
+.fi
+.RE
+.PP
+The following command would restart the ctdb service on all nodes.
+.sp
+.RS 4
+.nf
+onnode all service ctdb restart
+      
+.fi
+.RE
+.SH "SEE ALSO"
+.PP
+ctdbd(1), ctdb(1),
+\fI\%http://ctdb.samba.org/\fR
+.SH "COPYRIGHT/LICENSE"
+.sp
+.RS 4
+.nf
+Copyright (C) Andrew Tridgell 2007
+Copyright (C) Ronnie sahlberg 2007
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see http://www.gnu.org/licenses/.
+.fi
+.RE
diff --git a/source4/cluster/ctdb/doc/onnode.1.html b/source4/cluster/ctdb/doc/onnode.1.html
new file mode 100644
index 0000000000..57ddfec37b
--- /dev/null
+++ b/source4/cluster/ctdb/doc/onnode.1.html
@@ -0,0 +1,45 @@
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"><title>onnode</title><meta name="generator" content="DocBook XSL Stylesheets V1.72.0"></head><body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"><div class="refentry" lang="en"><a name="onnode.1"></a><div class="titlepage"></div><div class="refnamediv"><h2>Name</h2><p>onnode &#8212; run commands on ctdb nodes</p></div><div class="refsynopsisdiv"><h2>Synopsis</h2><div class="cmdsynopsis"><p><code class="command">onnode NODE COMMAND ...</code> </p></div></div><div class="refsect1" lang="en"><a name="id2490457"></a><h2>DESCRIPTION</h2><p>
+      onnode is a utility to run commands on a specific node of a CTDB
+      cluster, or on all nodes.
+    </p><p>
+      The NODE option specifies which node to run a command on. You
+      can specify a numeric node number (from 0 to N-1) or the special
+      node 'all'.
+    </p><p>
+      The COMMAND can be any shell command. The onnode utility uses
+      ssh or rsh to connect to the remote nodes and run the command.
+      </p><p>
+      If the COMMAND starts with a /, then the command is run via the
+      'at' service. Otherwise the command is run in the foreground.
+    </p></div><div class="refsect1" lang="en"><a name="id2487793"></a><h2>EXAMPLES</h2><p>
+      The following command would show the process ID of ctdb on all nodes
+      </p><pre class="screen">
+onnode all pidof ctdbd
+      </pre><p>
+      The following command would show the last 5 lines of log on each
+      node, preceded by the nodes hostname
+      </p><pre class="screen">
+onnode all "hostname; tail -5 /var/log/log.ctdb"
+      </pre><p>
+      The following command would restart the ctdb service on all nodes.
+      </p><pre class="screen">
+onnode all service ctdb restart
+      </pre></div><div class="refsect1" lang="en"><a name="id2488691"></a><h2>SEE ALSO</h2><p>
+      ctdbd(1), ctdb(1), <a href="http://ctdb.samba.org/" target="_top">http://ctdb.samba.org/</a>
+    </p></div><div class="refsect1" lang="en"><a name="id2488704"></a><h2>COPYRIGHT/LICENSE</h2><div class="literallayout"><p><br>
+Copyright (C) Andrew Tridgell 2007<br>
+Copyright (C) Ronnie sahlberg 2007<br>
+<br>
+This program is free software; you can redistribute it and/or modify<br>
+it under the terms of the GNU General Public License as published by<br>
+the Free Software Foundation; either version 3 of the License, or (at<br>
+your option) any later version.<br>
+<br>
+This program is distributed in the hope that it will be useful, but<br>
+WITHOUT ANY WARRANTY; without even the implied warranty of<br>
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU<br>
+General Public License for more details.<br>
+<br>
+You should have received a copy of the GNU General Public License<br>
+along with this program; if not, see http://www.gnu.org/licenses/.<br>
+</p></div></div></div></body></html>
diff --git a/source4/cluster/ctdb/doc/onnode.1.xml b/source4/cluster/ctdb/doc/onnode.1.xml
new file mode 100644
index 0000000000..e044e5fc42
--- /dev/null
+++ b/source4/cluster/ctdb/doc/onnode.1.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE refentry PUBLIC "-//Samba-Team//DTD DocBook V4.2-Based Variant V1.0//EN" "http://www.samba.org/samba/DTD/samba-doc">
+<refentry id="onnode.1">
+
+<refmeta>
+	<refentrytitle>onnode</refentrytitle>
+	<manvolnum>1</manvolnum>
+</refmeta>
+
+
+<refnamediv>
+	<refname>onnode</refname>
+        <refpurpose>run commands on ctdb nodes</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+	<cmdsynopsis>
+		<command>onnode NODE COMMAND ...</command>
+	</cmdsynopsis>
+</refsynopsisdiv>
+
+  <refsect1><title>DESCRIPTION</title>
+    <para>
+      onnode is a utility to run commands on a specific node of a CTDB
+      cluster, or on all nodes.
+    </para>
+    <para>
+      The NODE option specifies which node to run a command on. You
+      can specify a numeric node number (from 0 to N-1) or the special
+      node 'all'.
+    </para>
+    <para>
+      The COMMAND can be any shell command. The onnode utility uses
+      ssh or rsh to connect to the remote nodes and run the command.
+      </para>
+    <para>
+      If the COMMAND starts with a /, then the command is run via the
+      'at' service. Otherwise the command is run in the foreground.
+    </para>
+  </refsect1>
+
+  <refsect1><title>EXAMPLES</title>
+    <para>
+      The following command would show the process ID of ctdb on all nodes
+      </para>
+      <screen format="linespecific">
+onnode all pidof ctdbd
+      </screen>
+
+    <para>
+      The following command would show the last 5 lines of log on each
+      node, preceded by the nodes hostname
+      </para>
+      <screen format="linespecific">
+onnode all "hostname; tail -5 /var/log/log.ctdb"
+      </screen>
+
+    <para>
+      The following command would restart the ctdb service on all nodes.
+      </para>
+      <screen format="linespecific">
+onnode all service ctdb restart
+      </screen>
+
+    </refsect1>
+
+  <refsect1><title>SEE ALSO</title>
+    <para>
+      ctdbd(1), ctdb(1), <ulink url="http://ctdb.samba.org/"/>
+    </para>
+  </refsect1>
+  <refsect1><title>COPYRIGHT/LICENSE</title>
+<literallayout>
+Copyright (C) Andrew Tridgell 2007
+Copyright (C) Ronnie sahlberg 2007
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see http://www.gnu.org/licenses/.
+</literallayout>
+  </refsect1>
+</refentry>
diff --git a/source4/cluster/ctdb/ib/README.txt b/source4/cluster/ctdb/ib/README.txt
index 74fc129c35..40419829ca 100644
--- a/source4/cluster/ctdb/ib/README.txt
+++ b/source4/cluster/ctdb/ib/README.txt
@@ -11,10 +11,9 @@ After then:
 
 Example for testing
 ===================
-bin/ctdb_test --transport ib --nlist ../2nodes_rm.txt --listen 10.0.0.1:9001
-bin/ctdb_test --transport ib --nlist ../2nodes_rm.txt --listen 10.0.0.2:9001
+bin/ctdb_test --transport ib --nlist ../2nodes_rm.txt --listen 10.0.0.1
+bin/ctdb_test --transport ib --nlist ../2nodes_rm.txt --listen 10.0.0.2
 
 where 2nodes_rm.txt:
-10.0.0.1:9001
-10.0.0.2:9001
-
+10.0.0.1
+10.0.0.2
diff --git a/source4/cluster/ctdb/ib/ibw_ctdb.c b/source4/cluster/ctdb/ib/ibw_ctdb.c
index c3d1dc368c..5822ffb186 100644
--- a/source4/cluster/ctdb/ib/ibw_ctdb.c
+++ b/source4/cluster/ctdb/ib/ibw_ctdb.c
@@ -28,6 +28,21 @@
 #include "ibwrapper.h"
 #include "ibw_ctdb.h"
 
+int ctdb_ibw_get_address(struct ctdb_context *ctdb,
+	const char *address, struct in_addr *addr)
+{
+	if (inet_pton(AF_INET, address, addr) <= 0) {
+		struct hostent *he = gethostbyname(address);
+		if (he == NULL || he->h_length > sizeof(*addr)) {
+			ctdb_set_error(ctdb, "invalid nework address '%s'\n", 
+				       address);
+			return -1;
+		}
+		memcpy(addr, he->h_addr, he->h_length);
+	}
+	return 0;
+}
+
 int ctdb_ibw_node_connect(struct ctdb_node *node)
 {
 	struct ctdb_ibw_node *cn = talloc_get_type(node->private_data, struct ctdb_ibw_node);
@@ -38,9 +53,12 @@ int ctdb_ibw_node_connect(struct ctdb_node *node)
 	struct sockaddr_in sock_out;
 
 	memset(&sock_out, 0, sizeof(struct sockaddr_in));
-	inet_pton(AF_INET, node->address.address, &sock_out.sin_addr);
 	sock_out.sin_port = htons(node->address.port);
 	sock_out.sin_family = PF_INET;
+	if (ctdb_ibw_get_address(node->ctdb, node->address.address, &sock_out.sin_addr)) {
+		DEBUG(0, ("ctdb_ibw_node_connect failed\n"));
+		return -1;
+	}
 
 	rc = ibw_connect(cn->conn, &sock_out, node);
 	if (rc) {
diff --git a/source4/cluster/ctdb/ib/ibw_ctdb.h b/source4/cluster/ctdb/ib/ibw_ctdb.h
index 461a884c17..98ea102eac 100644
--- a/source4/cluster/ctdb/ib/ibw_ctdb.h
+++ b/source4/cluster/ctdb/ib/ibw_ctdb.h
@@ -35,6 +35,9 @@ struct ctdb_ibw_node {
 	int	qcnt;
 };
 
+int ctdb_ibw_get_address(struct ctdb_context *ctdb,
+	const char *address, struct in_addr *addr);
+
 int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn);
 int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n);
 
@@ -43,3 +46,5 @@ void ctdb_ibw_node_connect_event(struct event_context *ev, struct timed_event *t
 	struct timeval t, void *private_data);
 
 int ctdb_flush_cn_queue(struct ctdb_ibw_node *cn);
+
+int ctdb_ibw_init(struct ctdb_context *ctdb);
diff --git a/source4/cluster/ctdb/ib/ibw_ctdb_init.c b/source4/cluster/ctdb/ib/ibw_ctdb_init.c
index 523a02c4df..8dbb9c241c 100644
--- a/source4/cluster/ctdb/ib/ibw_ctdb_init.c
+++ b/source4/cluster/ctdb/ib/ibw_ctdb_init.c
@@ -37,7 +37,8 @@ static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
 	memset(&my_addr, 0, sizeof(struct sockaddr_in));
 	my_addr.sin_port = htons(ctdb->address.port);
 	my_addr.sin_family = PF_INET;
-	inet_pton(AF_INET, ctdb->address.address, &my_addr.sin_addr);
+	if (ctdb_ibw_get_address(ctdb, ctdb->address.address, &my_addr.sin_addr))
+		return -1;
 
 	if (ibw_bind(ictx, &my_addr)) {
 		DEBUG(0, ("ctdb_ibw_listen: ibw_bind failed\n"));
@@ -53,41 +54,63 @@ static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
 }
 
 /*
- * Start infiniband
+ * initialise ibw portion of a ctdb node 
  */
-static int ctdb_ibw_start(struct ctdb_context *ctdb)
+static int ctdb_ibw_add_node(struct ctdb_node *node)
+{
+	struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx);
+	struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
+
+	assert(cn!=NULL);
+	cn->conn = ibw_conn_new(ictx, node);
+	node->private_data = (void *)cn;
+
+	return (cn->conn!=NULL ? 0 : -1);
+}
+
+/*
+ * initialise infiniband
+ */
+static int ctdb_ibw_initialise(struct ctdb_context *ctdb)
 {
-	int i;
+	int i, ret;
+
+	ret = ctdb_ibw_init(ctdb);
+	if (ret != 0) {
+		return ret;
+	}
+
+	for (i=0; i<ctdb->num_nodes; i++) {
+		if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) {
+			DEBUG(0, ("methods->add_node failed at %d\n", i));
+			return -1;
+		}
+	}
 
 	/* listen on our own address */
 	if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
 		return -1;
 
-	/* everything async here */
-	for (i=0;i<ctdb->num_nodes;i++) {
-		struct ctdb_node *node = ctdb->nodes[i];
-		if (!(ctdb->flags & CTDB_FLAG_SELF_CONNECT) &&
-			ctdb_same_address(&ctdb->address, &node->address))
-			continue;
-		ctdb_ibw_node_connect(node);
-	}
-
 	return 0;
 }
 
+
 /*
- * initialise ibw portion of a ctdb node 
+ * Start infiniband
  */
-static int ctdb_ibw_add_node(struct ctdb_node *node)
+static int ctdb_ibw_start(struct ctdb_context *ctdb)
 {
-	struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx);
-	struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
+	int i, ret;
 
-	assert(cn!=NULL);
-	cn->conn = ibw_conn_new(ictx, node);
-	node->private_data = (void *)cn;
+	/* everything async here */
+	for (i=0;i<ctdb->num_nodes;i++) {
+		struct ctdb_node *node = ctdb->nodes[i];
+		if (!ctdb_same_address(&ctdb->address, &node->address)) {
+			ctdb_ibw_node_connect(node);
+		}
+	}
 
-	return (cn->conn!=NULL ? 0 : -1);
+	return 0;
 }
 
 static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length)
@@ -176,9 +199,10 @@ static int ctdb_ibw_stop(struct ctdb_context *cctx)
 #endif /* __NOTDEF__ */
 
 static const struct ctdb_methods ctdb_ibw_methods = {
+	.initialise= ctdb_ibw_initialise,
 	.start     = ctdb_ibw_start,
-	.add_node  = ctdb_ibw_add_node,
 	.queue_pkt = ctdb_ibw_queue_pkt,
+	.add_node = ctdb_ibw_add_node,
 	.allocate_pkt = ctdb_ibw_allocate_pkt,
 
 //	.stop = ctdb_ibw_stop
diff --git a/source4/cluster/ctdb/ib/ibwrapper.c b/source4/cluster/ctdb/ib/ibwrapper.c
index 8ef798c88b..31acbc4a2d 100644
--- a/source4/cluster/ctdb/ib/ibwrapper.c
+++ b/source4/cluster/ctdb/ib/ibwrapper.c
@@ -37,8 +37,9 @@
 #include "lib/events/events.h"
 #include "ibwrapper.h"
 
+#include <infiniband/kern-abi.h>
+#include <rdma/rdma_cma_abi.h>
 #include <rdma/rdma_cma.h>
-#include "infiniband/sa-kern-abi.h"
 
 #include "ibwrapper_internal.h"
 #include "lib/util/dlinklist.h"
@@ -82,7 +83,7 @@ static void *ibw_alloc_mr(struct ibw_ctx_priv *pctx, struct ibw_conn_priv *pconn
 
 static void ibw_free_mr(char **ppbuf, struct ibv_mr **ppmr)
 {
-	DEBUG(10, ("ibw_free_mr(%u %u)\n", (uint32_t)*ppbuf, (uint32_t)*ppmr));
+	DEBUG(10, ("ibw_free_mr(%p %p)\n", *ppbuf, *ppmr));
 	if (*ppmr!=NULL) {
 		ibv_dereg_mr(*ppmr);
 		*ppmr = NULL;
@@ -132,7 +133,7 @@ static int ibw_init_memory(struct ibw_conn *conn)
 
 static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx)
 {
-	DEBUG(10, ("ibw_ctx_priv_destruct(%u)\n", (uint32_t)pctx));
+	DEBUG(10, ("ibw_ctx_priv_destruct(%p)\n", pctx));
 
 	/* destroy cm */
 	if (pctx->cm_channel) {
@@ -154,7 +155,7 @@ static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx)
 
 static int ibw_ctx_destruct(struct ibw_ctx *ctx)
 {
-	DEBUG(10, ("ibw_ctx_destruct(%u)\n", (uint32_t)ctx));
+	DEBUG(10, ("ibw_ctx_destruct(%p)\n", ctx));
 	return 0;
 }
 
@@ -216,7 +217,7 @@ static int ibw_wr_destruct(struct ibw_wr *wr)
 
 static int ibw_conn_destruct(struct ibw_conn *conn)
 {
-	DEBUG(10, ("ibw_conn_destruct(%u)\n", (uint32_t)conn));
+	DEBUG(10, ("ibw_conn_destruct(%p)\n", conn));
 	
 	/* important here: ctx is a talloc _parent_ */
 	DLIST_REMOVE(conn->ctx->conn_list, conn);
@@ -427,6 +428,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
 	rc = rdma_get_cm_event(pctx->cm_channel, &event);
 	if (rc) {
 		ctx->state = IBWS_ERROR;
+		event = NULL;
 		sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc);
 		goto error;
 	}
@@ -492,7 +494,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
 
 	case RDMA_CM_EVENT_ESTABLISHED:
 		/* expected after ibw_accept and ibw_connect[not directly] */
-		DEBUG(0, ("ESTABLISHED (conn: %p)\n", cma_id->context));
+		DEBUG(1, ("ESTABLISHED (conn: %p)\n", cma_id->context));
 		conn = talloc_get_type(cma_id->context, struct ibw_conn);
 		assert(conn!=NULL); /* important assumption */
 
@@ -513,17 +515,21 @@ static void ibw_event_handler_cm(struct event_context *ev,
 		sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
 	case RDMA_CM_EVENT_UNREACHABLE:
 		sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
+		goto error;
 	case RDMA_CM_EVENT_REJECTED:
 		sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status);
+		DEBUG(1, ("cm event handler: %s", ibw_lasterr));
 		conn = talloc_get_type(cma_id->context, struct ibw_conn);
 		if (conn) {
+			/* must be done BEFORE connstate */
 			if ((rc=rdma_ack_cm_event(event)))
 				DEBUG(0, ("reject/rdma_ack_cm_event failed with %d\n", rc));
-			event = NULL;
-			pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
-			ibw_conn_priv_destruct(pconn);
+			event = NULL; /* not to touch cma_id or conn */
+			conn->state = IBWC_ERROR;
+			/* it should free the conn */
+			pctx->connstate_func(NULL, conn);
 		}
-		goto error;
+		break; /* this is not strictly an error */
 
 	case RDMA_CM_EVENT_DISCONNECTED:
 		DEBUG(11, ("RDMA_CM_EVENT_DISCONNECTED\n"));
@@ -555,22 +561,26 @@ static void ibw_event_handler_cm(struct event_context *ev,
 
 	return;
 error:
-	if (event!=NULL && (rc=rdma_ack_cm_event(event))) {
-		sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc);
-		goto error;
-	}
-
 	DEBUG(0, ("cm event handler: %s", ibw_lasterr));
 
-	if (cma_id!=pctx->cm_id) {
-		conn = talloc_get_type(cma_id->context, struct ibw_conn);
-		if (conn)
-			conn->state = IBWC_ERROR;
-		pctx->connstate_func(NULL, conn);
-	} else {
-		ctx->state = IBWS_ERROR;
-		pctx->connstate_func(ctx, NULL);
+	if (event!=NULL) {
+		if (cma_id!=NULL && cma_id!=pctx->cm_id) {
+			conn = talloc_get_type(cma_id->context, struct ibw_conn);
+			if (conn) {
+				conn->state = IBWC_ERROR;
+				pctx->connstate_func(NULL, conn);
+			}
+		} else {
+			ctx->state = IBWS_ERROR;
+			pctx->connstate_func(ctx, NULL);
+		}
+
+		if ((rc=rdma_ack_cm_event(event))!=0) {
+			DEBUG(0, ("rdma_ack_cm_event failed with %d\n", rc));
+		}
 	}
+
+	return;
 }
 
 static void ibw_event_handler_verbs(struct event_context *ev,
@@ -966,7 +976,11 @@ struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr,
 	pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx,
 		pctx->cm_channel->fd, EVENT_FD_READ, ibw_event_handler_cm, ctx);
 
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
 	rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx, RDMA_PS_TCP);
+#else
+	rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx);
+#endif
 	if (rc) {
 		rc = errno;
 		sprintf(ibw_lasterr, "rdma_create_id error %d\n", rc);
@@ -1087,7 +1101,11 @@ int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn
 	}
 
 	/* init cm */
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
 	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
+#else
+	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn);
+#endif
 	if (rc) {
 		rc = errno;
 		sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc);
diff --git a/source4/cluster/ctdb/ib/ibwrapper_test.c b/source4/cluster/ctdb/ib/ibwrapper_test.c
index 2942d26ed6..07c78e7709 100644
--- a/source4/cluster/ctdb/ib/ibwrapper_test.c
+++ b/source4/cluster/ctdb/ib/ibwrapper_test.c
@@ -289,7 +289,7 @@ int ibwtest_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn)
 			talloc_free(conn);
 			break;
 		case IBWC_ERROR:
-			DEBUG(10, ("test IBWC_ERROR\n"));
+			DEBUG(10, ("test IBWC_ERROR %s\n", ibw_getLastError()));
 			break;
 		default:
 			assert(0);
@@ -335,7 +335,7 @@ int ibwtest_receive_handler(struct ibw_conn *conn, void *buf, int n)
 				ibw_stop(tcx->ibwctx);
 				goto error;
 			}
-		} else {
+		} else if (op!=TESTOP_SEND_ID) {
 			char *buf2;
 			void *key2;
 
@@ -465,6 +465,19 @@ int ibwtest_parse_attrs(struct ibwtest_ctx *tcx, char *optext,
 	return 0;
 }
 
+static int ibwtest_get_address(const char *address, struct in_addr *addr)
+{
+	if (inet_pton(AF_INET, address, addr) <= 0) {
+		struct hostent *he = gethostbyname(address);
+		if (he == NULL || he->h_length > sizeof(*addr)) {
+			DEBUG(0, ("invalid nework address '%s'\n", address));
+			return -1;
+		}
+		memcpy(addr, he->h_addr, he->h_length);
+	}
+	return 0;
+}
+
 int ibwtest_getdests(struct ibwtest_ctx *tcx, char op)
 {
 	int	i;
@@ -482,7 +495,8 @@ int ibwtest_getdests(struct ibwtest_ctx *tcx, char op)
 	for(i=0; i<tcx->naddrs; i++) {
 		p = tcx->addrs + i;
 		p->sin_family = AF_INET;
-		p->sin_addr.s_addr = inet_addr(attrs[i].name);
+		if (ibwtest_get_address(attrs[i].name, &p->sin_addr))
+			return -1;
 		p->sin_port = htons(atoi(attrs[i].value));
 	}
 
@@ -516,13 +530,14 @@ void ibwtest_usage(struct ibwtest_ctx *tcx, char *name)
 	printf("\t%s -i <id> -o {name:value} -d {addr:port} -t nsec -s\n", name);
 	printf("\t-i <id> is a free text, acting as a server id, max 23 chars [mandatory]\n");
 	printf("\t-o name1:value1,name2:value2,... is a list of (name, value) pairs\n");
-	printf("\t-d addr1:port1,addr2:port2,... is a list of destination ip addresses\n");
+	printf("\t-a addr1:port1,addr2:port2,... is a list of destination ip addresses\n");
 	printf("\t-t nsec delta time between sends in nanosec [default %d]\n", tcx->nsec);
 	printf("\t\t send message periodically and endless when nsec is non-zero\n");
 	printf("\t-s server mode (you have to give exactly one -d address:port in this case)\n");
 	printf("\t-n number of messages to send [default %d]\n", tcx->nmsg);
 	printf("\t-l usec time to sleep in the main loop [default %d]\n", tcx->sleep_usec);
 	printf("\t-v max variable msg size in bytes [default %d], 0=don't send var. size\n", tcx->maxsize);
+	printf("\t-d LogLevel [default %d]\n", LogLevel);	
 	printf("Press ctrl+C to stop the program.\n");
 }
 
@@ -538,13 +553,14 @@ int main(int argc, char *argv[])
 	memset(tcx, 0, sizeof(struct ibwtest_ctx));
 	tcx->nsec = 0;
 	tcx->nmsg = 1000;
+	LogLevel = 0;
 
 	/* here is the only case we can't avoid using global... */
 	testctx = tcx;
 	signal(SIGINT, ibwtest_sigint_handler);
 	srand((unsigned)time(NULL));
 
-	while ((op=getopt(argc, argv, "i:o:d:m:st:n:l:v:")) != -1) {
+	while ((op=getopt(argc, argv, "i:o:d:m:st:n:l:v:a:")) != -1) {
 		switch (op) {
 		case 'i':
 			tcx->id = talloc_strdup(tcx, optarg);
@@ -555,7 +571,7 @@ int main(int argc, char *argv[])
 				&tcx->nattrs, op))
 				goto cleanup;
 			break;
-		case 'd':
+		case 'a':
 			if (ibwtest_getdests(tcx, op))
 				goto cleanup;
 			break;
@@ -574,6 +590,9 @@ int main(int argc, char *argv[])
 		case 'v':
 			tcx->maxsize = (unsigned int)atoi(optarg);
 			break;
+		case 'd':
+			LogLevel = atoi(optarg);
+			break;
 		default:
 			fprintf(stderr, "ERROR: unknown option -%c\n", (char)op);
 			ibwtest_usage(tcx, argv[0]);
diff --git a/source4/cluster/ctdb/include/ctdb.h b/source4/cluster/ctdb/include/ctdb.h
index 9f7f65e8b0..6c25b7fa9d 100644
--- a/source4/cluster/ctdb/include/ctdb.h
+++ b/source4/cluster/ctdb/include/ctdb.h
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #ifndef _CTDB_H
@@ -48,14 +48,53 @@ struct ctdb_call_info {
 /*
   ctdb flags
 */
-#define CTDB_FLAG_SELF_CONNECT (1<<0)
 #define CTDB_FLAG_TORTURE      (1<<1)
 
-
 /* 
    a message handler ID meaning "give me all messages"
  */
-#define CTDB_SRVID_ALL 0xFFFFFFFF
+#define CTDB_SRVID_ALL (~(uint64_t)0)
+
+/*
+  srvid type : RECOVERY
+*/
+#define CTDB_SRVID_RECOVERY	0xF100000000000000LL
+
+/* 
+   a message handler ID meaning that the cluster has been reconfigured
+ */
+#define CTDB_SRVID_RECONFIGURE 0xF200000000000000LL
+
+/* 
+   a message handler ID meaning that an IP address has been released
+ */
+#define CTDB_SRVID_RELEASE_IP 0xF300000000000000LL
+
+/* 
+   a message ID meaning that a nodes flags have changed
+ */
+#define CTDB_SRVID_NODE_FLAGS_CHANGED 0xF400000000000000LL
+
+/* 
+   a message ID meaning that a node should be banned
+ */
+#define CTDB_SRVID_BAN_NODE 0xF500000000000000LL
+
+/* 
+   a message ID meaning that a node should be unbanned
+ */
+#define CTDB_SRVID_UNBAN_NODE 0xF600000000000000LL
+
+
+/* used on the domain socket, send a pdu to the local daemon */
+#define CTDB_CURRENT_NODE     0xF0000001
+/* send a broadcast to all nodes in the cluster, active or not */
+#define CTDB_BROADCAST_ALL    0xF0000002
+/* send a broadcast to all nodes in the current vnn map */
+#define CTDB_BROADCAST_VNNMAP 0xF0000003
+/* send a broadcast to all connected nodes */
+#define CTDB_BROADCAST_CONNECTED 0xF0000004
+
 
 struct event_context;
 
@@ -80,11 +119,6 @@ int ctdb_set_tdb_dir(struct ctdb_context *ctdb, const char *dir);
 void ctdb_set_flags(struct ctdb_context *ctdb, unsigned flags);
 
 /*
-  clear some flags
-*/
-void ctdb_clear_flags(struct ctdb_context *ctdb, unsigned flags);
-
-/*
   set max acess count before a dmaster migration
 */
 void ctdb_set_max_lacount(struct ctdb_context *ctdb, unsigned count);
@@ -94,6 +128,8 @@ void ctdb_set_max_lacount(struct ctdb_context *ctdb, unsigned count);
 */
 int ctdb_set_address(struct ctdb_context *ctdb, const char *address);
 
+int ctdb_set_socketname(struct ctdb_context *ctdb, const char *socketname);
+
 /*
   tell ctdb what nodes are available. This takes a filename, which will contain
   1 node address per line, in a transport specific format
@@ -104,12 +140,12 @@ int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist);
   start the ctdb protocol
 */
 int ctdb_start(struct ctdb_context *ctdb);
+int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork);
 
 /*
   attach to a ctdb database
 */
-struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name, int tdb_flags, 
-				    int open_flags, mode_t mode);
+struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name);
 
 /*
   find an attached ctdb_db handle given a name
@@ -127,7 +163,7 @@ typedef int (*ctdb_fn_t)(struct ctdb_call_info *);
 /*
   setup a ctdb call function
 */
-int ctdb_set_call(struct ctdb_db_context *ctdb_db, ctdb_fn_t fn, int id);
+int ctdb_set_call(struct ctdb_db_context *ctdb_db, ctdb_fn_t fn, uint32_t id);
 
 
 
@@ -138,11 +174,6 @@ int ctdb_set_call(struct ctdb_db_context *ctdb_db, ctdb_fn_t fn, int id);
 int ctdb_call(struct ctdb_db_context *ctdb_db, struct ctdb_call *call);
 
 /*
-  wait for all nodes to be connected - useful for test code
-*/
-void ctdb_connect_wait(struct ctdb_context *ctdb);
-
-/*
   initiate an ordered ctdb cluster shutdown
   this function will never return
 */
@@ -157,9 +188,9 @@ uint32_t ctdb_get_vnn(struct ctdb_context *ctdb);
 uint32_t ctdb_get_num_nodes(struct ctdb_context *ctdb);
 
 /* setup a handler for ctdb messages */
-typedef void (*ctdb_message_fn_t)(struct ctdb_context *, uint32_t srvid, 
+typedef void (*ctdb_message_fn_t)(struct ctdb_context *, uint64_t srvid, 
 				  TDB_DATA data, void *);
-int ctdb_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+int ctdb_set_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 			     ctdb_message_fn_t handler,
 			     void *private_data);
 
@@ -170,46 +201,170 @@ int ctdb_call_recv(struct ctdb_client_call_state *state, struct ctdb_call *call)
 
 /* send a ctdb message */
 int ctdb_send_message(struct ctdb_context *ctdb, uint32_t vnn,
-		      uint32_t srvid, TDB_DATA data);
+		      uint64_t srvid, TDB_DATA data);
 
 
 /* 
    Fetch a ctdb record from a remote node
  . Underneath this will force the
    dmaster for the record to be moved to the local node. 
-
 */
 struct ctdb_record_handle *ctdb_fetch_lock(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx, 
 					   TDB_DATA key, TDB_DATA *data);
 
-
-/*
-  do a fetch lock from a client to the local daemon
-*/
-#define FETCH_LOCK_SUCCESS		0
-#define FETCH_LOCK_LOCKFAILED		1
-#define FETCH_LOCK_FETCHFAILED		2
-#define FETCH_LOCK_DMASTERFAILED	3
-
-int ctdb_client_fetch_lock(struct ctdb_db_context *ctdb_db, 
-						  TALLOC_CTX *mem_ctx, 
-						  TDB_DATA key, TDB_DATA *data);
-
-
 int ctdb_record_store(struct ctdb_record_handle *h, TDB_DATA data);
 
 int ctdb_register_message_handler(struct ctdb_context *ctdb, 
 				  TALLOC_CTX *mem_ctx,
-				  uint32_t srvid,
+				  uint64_t srvid,
 				  ctdb_message_fn_t handler,
 				  void *private_data);
 
 struct ctdb_db_context *find_ctdb_db(struct ctdb_context *ctdb, uint32_t id);
 
 
-struct ctdb_context *ctdb_cmdline_client(struct event_context *ev, const char *ctdb_socket);
+struct ctdb_context *ctdb_cmdline_client(struct event_context *ev);
+
+struct ctdb_statistics;
+int ctdb_ctrl_statistics(struct ctdb_context *ctdb, uint32_t destnode, struct ctdb_statistics *status);
+
+int ctdb_ctrl_shutdown(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+struct ctdb_vnn_map;
+int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, 
+		struct timeval timeout, uint32_t destnode, 
+		TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap);
+int ctdb_ctrl_setvnnmap(struct ctdb_context *ctdb,
+		struct timeval timeout, uint32_t destnode, 
+		TALLOC_CTX *mem_ctx, struct ctdb_vnn_map *vnnmap);
+
+/* table that contains a list of all dbids on a node
+ */
+struct ctdb_dbid_map {
+	uint32_t num;
+	uint32_t dbids[1];
+};
+int ctdb_ctrl_getdbmap(struct ctdb_context *ctdb, 
+	struct timeval timeout, uint32_t destnode, 
+	TALLOC_CTX *mem_ctx, struct ctdb_dbid_map **dbmap);
+
+
+struct ctdb_node_map;
+
+int ctdb_ctrl_getnodemap(struct ctdb_context *ctdb, 
+		    struct timeval timeout, uint32_t destnode, 
+		    TALLOC_CTX *mem_ctx, struct ctdb_node_map **nodemap);
+
+struct ctdb_key_list {
+	uint32_t dbid;
+	uint32_t num;
+	TDB_DATA *keys;
+	struct ctdb_ltdb_header *headers;
+	TDB_DATA *data;
+};
+int ctdb_ctrl_pulldb(struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid, uint32_t lmaster, TALLOC_CTX *mem_ctx, struct ctdb_key_list *keys);
+int ctdb_ctrl_copydb(struct ctdb_context *ctdb, 
+	struct timeval timeout, uint32_t sourcenode, 
+	uint32_t destnode, uint32_t dbid, uint32_t lmaster, 
+	TALLOC_CTX *mem_ctx);
+
+int ctdb_ctrl_getdbpath(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t dbid, TALLOC_CTX *mem_ctx, const char **path);
+int ctdb_ctrl_getdbname(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t dbid, TALLOC_CTX *mem_ctx, const char **name);
+int ctdb_ctrl_createdb(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, const char *name);
+
+int ctdb_ctrl_process_exists(struct ctdb_context *ctdb, uint32_t destnode, pid_t pid);
+
+int ctdb_ctrl_ping(struct ctdb_context *ctdb, uint32_t destnode);
+
+int ctdb_ctrl_get_config(struct ctdb_context *ctdb);
+
+int ctdb_ctrl_get_debuglevel(struct ctdb_context *ctdb, uint32_t destnode, uint32_t *level);
+int ctdb_ctrl_set_debuglevel(struct ctdb_context *ctdb, uint32_t destnode, uint32_t level);
+
+/*
+  change dmaster for all keys in the database to the new value
+ */
+int ctdb_ctrl_setdmaster(struct ctdb_context *ctdb, 
+	struct timeval timeout, uint32_t destnode, 
+	TALLOC_CTX *mem_ctx, uint32_t dbid, uint32_t dmaster);
+
+/*
+  write a record on a specific db (this implicitely updates dmaster of the record to locally be the vnn of the node where the control is executed on)
+ */
+int ctdb_ctrl_write_record(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_CTX *mem_ctx, uint32_t dbid, TDB_DATA key, TDB_DATA data);
+
+#define CTDB_RECOVERY_NORMAL		0
+#define CTDB_RECOVERY_ACTIVE		1
+
+/*
+  get the recovery mode of a remote node
+ */
+int ctdb_ctrl_getrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *recmode);
+/*
+  set the recovery mode of a remote node
+ */
+int ctdb_ctrl_setrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmode);
+/*
+  get the monitoring mode of a remote node
+ */
+int ctdb_ctrl_getmonmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *monmode);
+/*
+  set the monitoringmode of a remote node
+ */
+int ctdb_ctrl_setmonmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t monmode);
+
+/*
+  get the recovery master of a remote node
+ */
+int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *recmaster);
+/*
+  set the recovery master of a remote node
+ */
+int ctdb_ctrl_setrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmaster);
+
+uint32_t *ctdb_get_connected_nodes(struct ctdb_context *ctdb, 
+				   struct timeval timeout, 
+				   TALLOC_CTX *mem_ctx,
+				   uint32_t *num_nodes);
+
+int ctdb_statistics_reset(struct ctdb_context *ctdb, uint32_t destnode);
+
+int ctdb_set_logfile(struct ctdb_context *ctdb, const char *logfile);
+
+typedef int (*ctdb_traverse_func)(struct ctdb_context *, TDB_DATA, TDB_DATA, void *);
+int ctdb_traverse(struct ctdb_db_context *ctdb_db, ctdb_traverse_func fn, void *private_data);
+
+int ctdb_dump_db(struct ctdb_db_context *ctdb_db, FILE *f);
+
+/*
+  get the pid of a ctdb daemon
+ */
+int ctdb_ctrl_getpid(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *pid);
+
+int ctdb_ctrl_freeze(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+int ctdb_ctrl_thaw(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_getvnn(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_get_tunable(struct ctdb_context *ctdb, 
+			  struct timeval timeout, 
+			  uint32_t destnode,
+			  const char *name, uint32_t *value);
+
+int ctdb_ctrl_set_tunable(struct ctdb_context *ctdb, 
+			  struct timeval timeout, 
+			  uint32_t destnode,
+			  const char *name, uint32_t value);
+
+int ctdb_ctrl_list_tunables(struct ctdb_context *ctdb, 
+			    struct timeval timeout, 
+			    uint32_t destnode,
+			    TALLOC_CTX *mem_ctx,
+			    const char ***list, uint32_t *count);
 
-struct ctdb_status;
-int ctdb_status(struct ctdb_context *ctdb, struct ctdb_status *status);
+int ctdb_ctrl_modflags(struct ctdb_context *ctdb, 
+		       struct timeval timeout, 
+		       uint32_t destnode, 
+		       uint32_t set, uint32_t clear);
 
 #endif
diff --git a/source4/cluster/ctdb/include/ctdb_private.h b/source4/cluster/ctdb/include/ctdb_private.h
index 70c1c59998..bd64f978eb 100644
--- a/source4/cluster/ctdb/include/ctdb_private.h
+++ b/source4/cluster/ctdb/include/ctdb_private.h
@@ -3,33 +3,60 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #ifndef _CTDB_PRIVATE_H
 #define _CTDB_PRIVATE_H
 
 #include "ctdb.h"
+#include <sys/socket.h>
+#include <netinet/in.h>
 
 /* location of daemon socket */
 #define CTDB_PATH	"/tmp/ctdb.socket"
 
+/* default ctdb port number */
+#define CTDB_PORT 4379
+
 /* we must align packets to ensure ctdb works on all architectures (eg. sparc) */
 #define CTDB_DS_ALIGNMENT 8
 
 
-#define CTDB_NULL_FUNC 0xf0000001
+#define CTDB_NULL_FUNC      0xFF000001
+#define CTDB_FETCH_FUNC     0xFF000002
+
+/* all tunable variables go in here */
+struct ctdb_tunable {
+	uint32_t max_redirect_count;
+	uint32_t seqnum_frequency;
+	uint32_t control_timeout;
+	uint32_t traverse_timeout;
+	uint32_t keepalive_interval;
+	uint32_t keepalive_limit;
+	uint32_t max_lacount;
+	uint32_t recover_timeout;
+	uint32_t recover_interval;
+	uint32_t election_timeout;
+	uint32_t takeover_timeout;
+	uint32_t monitor_interval;
+	uint32_t script_timeout;
+	uint32_t recovery_grace_period;
+	uint32_t recovery_ban_period;
+	uint32_t database_hash_size;
+	uint32_t rerecovery_timeout;
+};
 
 /*
   an installed ctdb remote call
@@ -60,6 +87,24 @@ struct ctdb_address {
 typedef void (*ctdb_queue_cb_fn_t)(uint8_t *data, size_t length,
 				   void *private_data);
 
+/* used for callbacks in ctdb_control requests */
+typedef void (*ctdb_control_callback_fn_t)(struct ctdb_context *,
+					   int32_t status, TDB_DATA data, 
+					   const char *errormsg,
+					   void *private_data);
+
+/*
+  structure describing a connected client in the daemon
+ */
+struct ctdb_client {
+	struct ctdb_context *ctdb;
+	int fd;
+	struct ctdb_queue *queue;
+	uint32_t client_id;
+	pid_t pid;
+	struct ctdb_tcp_list *tcp_list;
+};
+
 
 /*
   state associated with one node
@@ -70,16 +115,42 @@ struct ctdb_node {
 	const char *name; /* for debug messages */
 	void *private_data; /* private to transport */
 	uint32_t vnn;
+#define NODE_FLAGS_DISCONNECTED		0x00000001 /* node isn't connected */
+#define NODE_FLAGS_UNHEALTHY  		0x00000002 /* monitoring says node is unhealthy */
+#define NODE_FLAGS_PERMANENTLY_DISABLED	0x00000004 /* administrator has disabled node */
+#define NODE_FLAGS_BANNED		0x00000008 /* recovery daemon has banned the node */
+#define NODE_FLAGS_DISABLED		(NODE_FLAGS_UNHEALTHY|NODE_FLAGS_PERMANENTLY_DISABLED)
+#define NODE_FLAGS_INACTIVE		(NODE_FLAGS_DISCONNECTED|NODE_FLAGS_BANNED)
+	uint32_t flags;
+
+	/* used by the dead node monitoring */
+	uint32_t dead_count;
+	uint32_t rx_cnt;
+	uint32_t tx_cnt;
+
+	/* a list of controls pending to this node, so we can time them out quickly
+	   if the node becomes disconnected */
+	struct daemon_control_state *pending_controls;
+
+	/* the public address of this node, if known */
+	const char *public_address;
+	uint8_t public_netmask_bits;
+
+	/* the node number that has taken over this nodes public address, if any. 
+	   If not taken over, then set to -1 */
+	int32_t takeover_vnn;
 };
 
 /*
   transport specific methods
 */
 struct ctdb_methods {
+	int (*initialise)(struct ctdb_context *); /* initialise transport structures */	
 	int (*start)(struct ctdb_context *); /* start protocol processing */	
 	int (*add_node)(struct ctdb_node *); /* setup a new node */	
 	int (*queue_pkt)(struct ctdb_node *, uint8_t *data, uint32_t length);
 	void *(*allocate_pkt)(TALLOC_CTX *mem_ctx, size_t );
+	void (*shutdown)(struct ctdb_context *); /* shutdown transport */
 };
 
 /*
@@ -101,7 +172,7 @@ struct ctdb_upcalls {
 struct ctdb_message_list {
 	struct ctdb_context *ctdb;
 	struct ctdb_message_list *next, *prev;
-	uint32_t srvid;
+	uint64_t srvid;
 	ctdb_message_fn_t message_handler;
 	void *message_private;
 };
@@ -116,60 +187,126 @@ struct ctdb_daemon_data {
 /*
   ctdb status information
  */
-struct ctdb_status {
+struct ctdb_statistics {
+	uint32_t num_clients;
+	uint32_t frozen;
+	uint32_t recovering;
 	uint32_t client_packets_sent;
 	uint32_t client_packets_recv;
 	uint32_t node_packets_sent;
 	uint32_t node_packets_recv;
+	uint32_t keepalive_packets_sent;
+	uint32_t keepalive_packets_recv;
 	struct {
 		uint32_t req_call;
 		uint32_t reply_call;
-		uint32_t reply_redirect;
 		uint32_t req_dmaster;
 		uint32_t reply_dmaster;
 		uint32_t reply_error;
 		uint32_t req_message;
-		uint32_t req_finished;
-	} count;
+		uint32_t req_control;
+		uint32_t reply_control;
+	} node;
 	struct {
 		uint32_t req_call;
 		uint32_t req_message;
-		uint32_t req_finished;
-		uint32_t req_register;
-		uint32_t req_connect_wait;
-		uint32_t req_shutdown;
-		uint32_t req_status;
+		uint32_t req_control;
 	} client;
+	struct {
+		uint32_t call;
+		uint32_t control;
+		uint32_t traverse;
+	} timeouts;
 	uint32_t total_calls;
 	uint32_t pending_calls;
 	uint32_t lockwait_calls;
 	uint32_t pending_lockwait_calls;
+	uint32_t memory_used;
+	uint32_t __last_counter; /* hack for control_statistics_all */
+	uint32_t max_hop_count;
 	double max_call_latency;
 	double max_lockwait_latency;
 };
 
+/* table that contains the mapping between a hash value and lmaster
+ */
+struct ctdb_vnn_map {
+	uint32_t generation;
+	uint32_t size;
+	uint32_t *map;
+};
+
+/* 
+   a wire representation of the vnn map
+ */
+struct ctdb_vnn_map_wire {
+	uint32_t generation;
+	uint32_t size;
+	uint32_t map[1];
+};
+
+/* a structure that contains the elements required for the write record
+   control
+*/
+struct ctdb_write_record {
+	uint32_t dbid;
+	uint32_t keylen;
+	uint32_t datalen;
+	unsigned char blob[1];
+};
+
+enum ctdb_freeze_mode {CTDB_FREEZE_NONE, CTDB_FREEZE_PENDING, CTDB_FREEZE_FROZEN};
+
+#define CTDB_MONITORING_ACTIVE		0
+#define CTDB_MONITORING_DISABLED	1
+
+/* information about IP takeover */
+struct ctdb_takeover {
+	bool enabled;
+	const char *interface;
+	const char *event_script;
+	TALLOC_CTX *last_ctx;
+};
+
 /* main state of the ctdb daemon */
 struct ctdb_context {
 	struct event_context *ev;
+	uint32_t recovery_mode;
+	uint32_t monitoring_mode;
+	TALLOC_CTX *monitor_context;
+	struct ctdb_tunable tunable;
+	enum ctdb_freeze_mode freeze_mode;
+	struct ctdb_freeze_handle *freeze_handle;
 	struct ctdb_address address;
 	const char *name;
 	const char *db_directory;
+	const char *transport;
+	const char *logfile;
+	char *node_list_file;
+	char *recovery_lock_file;
+	int recovery_lock_fd;
 	uint32_t vnn; /* our own vnn */
 	uint32_t num_nodes;
 	uint32_t num_connected;
-	uint32_t num_finished;
 	unsigned flags;
 	struct idr_context *idr;
+	uint16_t idr_cnt;
 	struct ctdb_node **nodes; /* array of nodes in the cluster - indexed by vnn */
 	char *err_msg;
 	const struct ctdb_methods *methods; /* transport methods */
 	const struct ctdb_upcalls *upcalls; /* transport upcalls */
 	void *private_data; /* private to transport */
-	unsigned max_lacount;
 	struct ctdb_db_context *db_list;
 	struct ctdb_message_list *message_list;
 	struct ctdb_daemon_data daemon;
-	struct ctdb_status status;
+	struct ctdb_statistics statistics;
+	struct ctdb_vnn_map *vnn_map;
+	uint32_t num_clients;
+	uint32_t recovery_master;
+	struct ctdb_call_state *pending_calls;
+	struct ctdb_takeover takeover;
+	struct ctdb_tcp_list *tcp_list;
+	struct ctdb_client_ip *client_ip_list;
 };
 
 struct ctdb_db_context {
@@ -177,33 +314,34 @@ struct ctdb_db_context {
 	struct ctdb_context *ctdb;
 	uint32_t db_id;
 	const char *db_name;
+	const char *db_path;
 	struct tdb_wrap *ltdb;
 	struct ctdb_registered_call *calls; /* list of registered calls */
+	uint32_t seqnum;
+	struct timed_event *te;
 };
 
 
 #define CTDB_NO_MEMORY(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
           ctdb_set_error(ctdb, "Out of memory at %s:%d", __FILE__, __LINE__); \
 	  return -1; }} while (0)
 
+#define CTDB_NO_MEMORY_VOID(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
+          ctdb_set_error(ctdb, "Out of memory at %s:%d", __FILE__, __LINE__); \
+	  }} while (0)
+
 #define CTDB_NO_MEMORY_NULL(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
           ctdb_set_error(ctdb, "Out of memory at %s:%d", __FILE__, __LINE__); \
 	  return NULL; }} while (0)
 
 #define CTDB_NO_MEMORY_FATAL(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
           ctdb_fatal(ctdb, "Out of memory in " __location__ ); \
 	  }} while (0)
 
-/* arbitrary maximum timeout for ctdb operations */
-#define CTDB_REQ_TIMEOUT 10
-
-/* max number of redirects before we ask the lmaster */
-#define CTDB_MAX_REDIRECT 2
-
-/* number of consecutive calls from the same node before we give them
-   the record */
-#define CTDB_DEFAULT_MAX_LACOUNT 7
-
 /*
   the extended header for records in the ltdb
 */
@@ -214,20 +352,144 @@ struct ctdb_ltdb_header {
 	uint32_t lacount;
 };
 
+enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0, 
+		    CTDB_CONTROL_STATISTICS              = 1, 
+		    /* #2 removed */
+		    CTDB_CONTROL_PING                    = 3,
+		    CTDB_CONTROL_GETDBPATH               = 4,
+		    CTDB_CONTROL_GETVNNMAP               = 5,
+		    CTDB_CONTROL_SETVNNMAP               = 6,
+		    CTDB_CONTROL_GET_DEBUG               = 7,
+		    CTDB_CONTROL_SET_DEBUG               = 8,
+		    CTDB_CONTROL_GET_DBMAP               = 9,
+		    CTDB_CONTROL_GET_NODEMAP             = 10,
+		    CTDB_CONTROL_SET_DMASTER             = 11,
+		    /* #12 removed */
+		    CTDB_CONTROL_PULL_DB                 = 13,
+		    CTDB_CONTROL_PUSH_DB                 = 14,
+		    CTDB_CONTROL_GET_RECMODE             = 15,
+		    CTDB_CONTROL_SET_RECMODE             = 16,
+		    CTDB_CONTROL_STATISTICS_RESET        = 17,
+		    CTDB_CONTROL_DB_ATTACH               = 18,
+		    CTDB_CONTROL_SET_CALL                = 19,
+		    CTDB_CONTROL_TRAVERSE_START          = 20,
+		    CTDB_CONTROL_TRAVERSE_ALL            = 21,
+		    CTDB_CONTROL_TRAVERSE_DATA           = 22,
+		    CTDB_CONTROL_REGISTER_SRVID          = 23,
+		    CTDB_CONTROL_DEREGISTER_SRVID        = 24,
+		    CTDB_CONTROL_GET_DBNAME              = 25,
+		    CTDB_CONTROL_ENABLE_SEQNUM           = 26,
+		    CTDB_CONTROL_UPDATE_SEQNUM           = 27,
+		    /* #28 removed */
+		    CTDB_CONTROL_DUMP_MEMORY             = 29,
+		    CTDB_CONTROL_GET_PID                 = 30,
+		    CTDB_CONTROL_GET_RECMASTER           = 31,
+		    CTDB_CONTROL_SET_RECMASTER           = 32,
+		    CTDB_CONTROL_FREEZE                  = 33,
+		    CTDB_CONTROL_THAW                    = 34,
+		    CTDB_CONTROL_GET_VNN                 = 35,
+		    CTDB_CONTROL_SHUTDOWN                = 36,
+		    CTDB_CONTROL_GET_MONMODE             = 37,
+		    CTDB_CONTROL_SET_MONMODE             = 38,
+		    CTDB_CONTROL_MAX_RSN                 = 39,
+		    CTDB_CONTROL_SET_RSN_NONEMPTY        = 40,
+		    CTDB_CONTROL_DELETE_LOW_RSN          = 41,
+		    CTDB_CONTROL_TAKEOVER_IP             = 42,
+		    CTDB_CONTROL_RELEASE_IP              = 43,
+		    CTDB_CONTROL_TCP_CLIENT              = 44,
+		    CTDB_CONTROL_TCP_ADD                 = 45,
+		    CTDB_CONTROL_TCP_REMOVE              = 46,
+		    CTDB_CONTROL_STARTUP                 = 47,
+		    CTDB_CONTROL_SET_TUNABLE             = 48,
+		    CTDB_CONTROL_GET_TUNABLE             = 49,
+		    CTDB_CONTROL_LIST_TUNABLES           = 50,
+		    CTDB_CONTROL_GET_PUBLIC_IPS          = 51,
+		    CTDB_CONTROL_MODIFY_FLAGS            = 52,
+		    CTDB_CONTROL_GET_ALL_TUNABLES        = 53,
+};
+
+/*
+  structure passed in ctdb_control_set_rsn_nonempty
+ */
+struct ctdb_control_set_rsn_nonempty {
+	uint32_t db_id;
+	uint64_t rsn;
+};
+
+/*
+  structure passed in ctdb_control_delete_low_rsn
+ */
+struct ctdb_control_delete_low_rsn {
+	uint32_t db_id;
+	uint64_t rsn;
+};
+
+/*
+  structure passed in set_call control
+ */
+struct ctdb_control_set_call {
+	uint32_t db_id;
+	ctdb_fn_t fn;
+	uint32_t id;
+};
+
+/*
+  struct for tcp_client control
+ */
+struct ctdb_control_tcp {
+	struct sockaddr_in src;
+	struct sockaddr_in dest;
+};
+
+/*
+  struct for tcp_add and tcp_remove controls
+ */
+struct ctdb_control_tcp_vnn {
+	uint32_t vnn;
+	struct sockaddr_in src;
+	struct sockaddr_in dest;
+};
+
+/*
+  structure used for CTDB_SRVID_NODE_FLAGS_CHANGED
+ */
+struct ctdb_node_flag_change {
+	uint32_t vnn;
+	uint32_t flags;
+};
+
+/*
+  structure to change flags on a node
+ */
+struct ctdb_node_modflags {
+	uint32_t set;
+	uint32_t clear;
+};
+
+/*
+  struct for admin setting a ban
+ */
+struct ctdb_ban_info {
+	uint32_t vnn;
+	uint32_t ban_time;
+};
+
 enum call_state {CTDB_CALL_WAIT, CTDB_CALL_DONE, CTDB_CALL_ERROR};
 
+#define CTDB_LMASTER_ANY	0xffffffff
+
 /*
   state of a in-progress ctdb call
 */
 struct ctdb_call_state {
+	struct ctdb_call_state *next, *prev;
 	enum call_state state;
+	uint32_t reqid;
 	struct ctdb_req_call *c;
 	struct ctdb_db_context *ctdb_db;
-	struct ctdb_node *node;
 	const char *errmsg;
 	struct ctdb_call call;
-	int redirect_count;
-	struct ctdb_ltdb_header header;
+	uint32_t generation;
 	struct {
 		void (*fn)(struct ctdb_call_state *);
 		void *private_data;
@@ -249,20 +511,14 @@ struct ctdb_fetch_handle {
 enum ctdb_operation {
 	CTDB_REQ_CALL           = 0,
 	CTDB_REPLY_CALL         = 1,
-	CTDB_REPLY_REDIRECT     = 2,
-	CTDB_REQ_DMASTER        = 3,
-	CTDB_REPLY_DMASTER      = 4,
-	CTDB_REPLY_ERROR        = 5,
-	CTDB_REQ_MESSAGE        = 6,
-	CTDB_REQ_FINISHED       = 7,
-	
-	/* only used on the domain socket */
-	CTDB_REQ_REGISTER       = 1000,     
-	CTDB_REQ_CONNECT_WAIT   = 1001,
-	CTDB_REPLY_CONNECT_WAIT = 1002,
-	CTDB_REQ_SHUTDOWN       = 1003,
-	CTDB_REQ_STATUS         = 1004,
-	CTDB_REPLY_STATUS       = 1005
+	CTDB_REQ_DMASTER        = 2,
+	CTDB_REPLY_DMASTER      = 3,
+	CTDB_REPLY_ERROR        = 4,
+	CTDB_REQ_MESSAGE        = 5,
+	/* #6 removed */
+	CTDB_REQ_CONTROL        = 7,
+	CTDB_REPLY_CONTROL      = 8,
+	CTDB_REQ_KEEPALIVE      = 9,
 };
 
 #define CTDB_MAGIC 0x43544442 /* CTDB */
@@ -275,6 +531,7 @@ struct ctdb_req_header {
 	uint32_t length;
 	uint32_t ctdb_magic;
 	uint32_t ctdb_version;
+	uint32_t generation;
 	uint32_t operation;
 	uint32_t destnode;
 	uint32_t srcnode;
@@ -286,6 +543,7 @@ struct ctdb_req_call {
 	uint32_t flags;
 	uint32_t db_id;
 	uint32_t callid;
+	uint32_t hopcount;
 	uint32_t keylen;
 	uint32_t calldatalen;
 	uint8_t data[1]; /* key[] followed by calldata[] */
@@ -305,14 +563,10 @@ struct ctdb_reply_error {
 	uint8_t  msg[1];
 };
 
-struct ctdb_reply_redirect {
-	struct ctdb_req_header hdr;
-	uint32_t dmaster;
-};
-
 struct ctdb_req_dmaster {
 	struct ctdb_req_header hdr;
 	uint32_t db_id;
+	uint64_t rsn;
 	uint32_t dmaster;
 	uint32_t keylen;
 	uint32_t datalen;
@@ -321,47 +575,52 @@ struct ctdb_req_dmaster {
 
 struct ctdb_reply_dmaster {
 	struct ctdb_req_header hdr;
+	uint32_t db_id;
+	uint64_t rsn;
+	uint32_t keylen;
 	uint32_t datalen;
 	uint8_t  data[1];
 };
 
-struct ctdb_req_register {
-	struct ctdb_req_header hdr;
-	uint32_t srvid;
-};
-
 struct ctdb_req_message {
 	struct ctdb_req_header hdr;
-	uint32_t srvid;
+	uint64_t srvid;
 	uint32_t datalen;
 	uint8_t data[1];
 };
 
-struct ctdb_req_finished {
-	struct ctdb_req_header hdr;
-};
-
-struct ctdb_req_shutdown {
+struct ctdb_req_getdbpath {
 	struct ctdb_req_header hdr;
+	uint32_t db_id;
 };
 
-struct ctdb_req_connect_wait {
+struct ctdb_reply_getdbpath {
 	struct ctdb_req_header hdr;
+	uint32_t datalen;
+	uint8_t data[1];
 };
 
-struct ctdb_reply_connect_wait {
+struct ctdb_req_control {
 	struct ctdb_req_header hdr;
-	uint32_t vnn;
-	uint32_t num_connected;
+	uint32_t opcode;
+	uint64_t srvid;
+	uint32_t client_id;
+#define CTDB_CTRL_FLAG_NOREPLY   1
+	uint32_t flags;
+	uint32_t datalen;
+	uint8_t data[1];
 };
 
-struct ctdb_req_status {
+struct ctdb_reply_control {
 	struct ctdb_req_header hdr;
+	int32_t  status;
+	uint32_t datalen;
+	uint32_t errorlen;
+	uint8_t data[1];
 };
 
-struct ctdb_reply_status {
+struct ctdb_req_keepalive {
 	struct ctdb_req_header hdr;
-	struct ctdb_status status;
 };
 
 /* internal prototypes */
@@ -372,13 +631,13 @@ int ctdb_parse_address(struct ctdb_context *ctdb,
 		       TALLOC_CTX *mem_ctx, const char *str,
 		       struct ctdb_address *address);
 uint32_t ctdb_hash(const TDB_DATA *key);
+uint32_t ctdb_hash_string(const char *str);
 void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 void ctdb_request_message(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 void ctdb_reply_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 void ctdb_reply_error(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
-void ctdb_reply_redirect(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 
 uint32_t ctdb_lmaster(struct ctdb_context *ctdb, const TDB_DATA *key);
 int ctdb_ltdb_fetch(struct ctdb_db_context *ctdb_db, 
@@ -389,14 +648,14 @@ int ctdb_ltdb_store(struct ctdb_db_context *ctdb_db, TDB_DATA key,
 void ctdb_queue_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
 int ctdb_ltdb_lock_requeue(struct ctdb_db_context *ctdb_db, 
 			   TDB_DATA key, struct ctdb_req_header *hdr,
-			   void (*recv_pkt)(void *, uint8_t *, uint32_t ),
-			   void *recv_context);
+			   void (*recv_pkt)(void *, struct ctdb_req_header *),
+			   void *recv_context, bool ignore_generation);
 int ctdb_ltdb_lock_fetch_requeue(struct ctdb_db_context *ctdb_db, 
 				 TDB_DATA key, struct ctdb_ltdb_header *header, 
 				 struct ctdb_req_header *hdr, TDB_DATA *data,
-				 void (*recv_pkt)(void *, uint8_t *, uint32_t ),
-				 void *recv_context);
-void ctdb_recv_pkt(struct ctdb_context *ctdb, uint8_t *data, uint32_t length);
+				 void (*recv_pkt)(void *, struct ctdb_req_header *),
+				 void *recv_context, bool ignore_generation);
+void ctdb_input_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *);
 
 struct ctdb_call_state *ctdb_call_local_send(struct ctdb_db_context *ctdb_db, 
 					     struct ctdb_call *call,
@@ -430,7 +689,21 @@ struct ctdb_queue *ctdb_queue_setup(struct ctdb_context *ctdb,
 /*
   allocate a packet for use in client<->daemon communication
  */
-void *ctdbd_allocate_pkt(TALLOC_CTX *mem_ctx, size_t len);
+struct ctdb_req_header *_ctdbd_allocate_pkt(struct ctdb_context *ctdb,
+					    TALLOC_CTX *mem_ctx, 
+					    enum ctdb_operation operation, 
+					    size_t length, size_t slength,
+					    const char *type);
+#define ctdbd_allocate_pkt(ctdb, mem_ctx, operation, length, type) \
+	(type *)_ctdbd_allocate_pkt(ctdb, mem_ctx, operation, length, sizeof(type), #type)
+
+struct ctdb_req_header *_ctdb_transport_allocate(struct ctdb_context *ctdb,
+						 TALLOC_CTX *mem_ctx, 
+						 enum ctdb_operation operation, 
+						 size_t length, size_t slength,
+						 const char *type);
+#define ctdb_transport_allocate(ctdb, mem_ctx, operation, length, type) \
+	(type *)_ctdb_transport_allocate(ctdb, mem_ctx, operation, length, sizeof(type), #type)
 
 
 /*
@@ -461,24 +734,18 @@ struct ctdb_call_state *ctdb_client_call_send(struct ctdb_db_context *ctdb_db,
 */
 int ctdb_client_call_recv(struct ctdb_call_state *state, struct ctdb_call *call);
 
-int ctdb_daemon_set_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+int ctdb_daemon_set_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 			     ctdb_message_fn_t handler,
 			     void *private_data);
 
 int ctdb_client_send_message(struct ctdb_context *ctdb, uint32_t vnn,
-			     uint32_t srvid, TDB_DATA data);
+			     uint64_t srvid, TDB_DATA data);
 
 /*
   send a ctdb message
 */
 int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t vnn,
-			     uint32_t srvid, TDB_DATA data);
-
-
-/*
-  wait for all nodes to be connected
-*/
-void ctdb_daemon_connect_wait(struct ctdb_context *ctdb);
+			     uint64_t srvid, TDB_DATA data);
 
 
 struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
@@ -494,14 +761,11 @@ struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctd
 						     struct ctdb_call *call, 
 						     struct ctdb_ltdb_header *header);
 
-void ctdb_request_finished(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
-
 int ctdb_call_local(struct ctdb_db_context *ctdb_db, struct ctdb_call *call,
-		    struct ctdb_ltdb_header *header, TDB_DATA *data,
+		    struct ctdb_ltdb_header *header, TALLOC_CTX *mem_ctx, TDB_DATA *data,
 		    uint32_t caller);
 
-void *_idr_find_type(struct idr_context *idp, int id, const char *type, const char *location);
-#define idr_find_type(idp, id, type) (type *)_idr_find_type(idp, id, #type, __location__)
+#define ctdb_reqid_find(ctdb, reqid, type)	(type *)_ctdb_reqid_find(ctdb, reqid, #type, __location__)
 
 void ctdb_recv_raw_pkt(void *p, uint8_t *data, uint32_t length);
 
@@ -509,4 +773,263 @@ int ctdb_socket_connect(struct ctdb_context *ctdb);
 
 void ctdb_latency(double *latency, struct timeval t);
 
+uint32_t ctdb_reqid_new(struct ctdb_context *ctdb, void *state);
+void *_ctdb_reqid_find(struct ctdb_context *ctdb, uint32_t reqid, const char *type, const char *location);
+void ctdb_reqid_remove(struct ctdb_context *ctdb, uint32_t reqid);
+
+void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_reply_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+
+int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
+			     uint64_t srvid, uint32_t opcode, uint32_t client_id, uint32_t flags,
+			     TDB_DATA data,
+			     ctdb_control_callback_fn_t callback,
+			     void *private_data);
+
+int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata, 
+			       TDB_DATA *outdata);
+
+int ctdb_daemon_set_call(struct ctdb_context *ctdb, uint32_t db_id,
+			 ctdb_fn_t fn, int id);
+
+int ctdb_control(struct ctdb_context *ctdb, uint32_t destnode, uint64_t srvid, 
+		 uint32_t opcode, uint32_t flags, TDB_DATA data, 
+		 TALLOC_CTX *mem_ctx, TDB_DATA *outdata, int32_t *status,
+		 struct timeval *timeout, char **errormsg);
+
+
+
+
+#define CHECK_CONTROL_DATA_SIZE(size) do { \
+ if (indata.dsize != size) { \
+	 DEBUG(0,(__location__ " Invalid data size in opcode %u. Got %u expected %u\n", \
+		  opcode, (unsigned)indata.dsize, (unsigned)size));	\
+	 return -1; \
+ } \
+ } while (0)
+
+int ctdb_control_getvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_setvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_getdbmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_writerecord(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+
+
+struct ctdb_traverse_start {
+	uint32_t db_id;
+	uint32_t reqid;
+	uint64_t srvid;
+};
+
+/*
+  structure used to pass record data between the child and parent
+ */
+struct ctdb_rec_data {
+	uint32_t length;
+	uint32_t reqid;
+	uint32_t keylen;
+	uint32_t datalen;
+	uint8_t  data[1];
+};
+				   
+
+/* structure used for pulldb control */
+struct ctdb_control_pulldb {
+	uint32_t db_id;
+	uint32_t lmaster;
+};
+
+/* structure used for pulldb control */
+struct ctdb_control_pulldb_reply {
+	uint32_t db_id;
+	uint32_t count;
+	uint8_t data[1];
+};
+
+/* set dmaster control structure */
+struct ctdb_control_set_dmaster {
+	uint32_t db_id;
+	uint32_t dmaster;
+};
+
+/*
+  structure for setting a tunable
+ */
+struct ctdb_control_set_tunable {
+	uint32_t value;
+	uint32_t length;
+	uint8_t  name[1];
+};
+
+/*
+  structure for getting a tunable
+ */
+struct ctdb_control_get_tunable {
+	uint32_t length;
+	uint8_t  name[1];
+};
+
+/*
+  structure for listing tunables
+ */
+struct ctdb_control_list_tunable {
+	uint32_t length;
+	/* returns a : separated list of tunable names */
+	uint8_t  data[1];
+};
+
+
+/* table that contains a list of all nodes a ctdb knows about and their 
+   status
+ */
+struct ctdb_node_and_flags {
+	uint32_t vnn;
+	uint32_t flags;
+	struct sockaddr_in sin;
+
+};
+
+struct ctdb_node_map {
+	uint32_t num;
+	struct ctdb_node_and_flags nodes[1];
+};
+
+int32_t ctdb_control_traverse_start(struct ctdb_context *ctdb, TDB_DATA indata, 
+				    TDB_DATA *outdata, uint32_t srcnode);
+int32_t ctdb_control_traverse_all(struct ctdb_context *ctdb, TDB_DATA data, TDB_DATA *outdata);
+int32_t ctdb_control_traverse_data(struct ctdb_context *ctdb, TDB_DATA data, TDB_DATA *outdata);
+
+int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA data);
+
+int daemon_register_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid);
+int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data);
+int daemon_deregister_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid);
+
+int32_t ctdb_ltdb_enable_seqnum(struct ctdb_context *ctdb, uint32_t db_id);
+int32_t ctdb_ltdb_update_seqnum(struct ctdb_context *ctdb, uint32_t db_id, uint32_t srcnode);
+int32_t ctdb_ltdb_set_seqnum_frequency(struct ctdb_context *ctdb, uint32_t frequency);
+
+struct ctdb_rec_data *ctdb_marshall_record(TALLOC_CTX *mem_ctx, uint32_t reqid,	TDB_DATA key, TDB_DATA data);
+
+int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_set_dmaster(struct ctdb_context *ctdb, TDB_DATA indata);
+
+int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, bool *async_reply,
+				 const char **errormsg);
+void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_control *c,
+				TDB_DATA *outdata, int32_t status, const char *errormsg);
+
+int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply);
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb);
+
+int ctdb_start_recoverd(struct ctdb_context *ctdb);
+
+uint32_t ctdb_get_num_active_nodes(struct ctdb_context *ctdb);
+
+void ctdb_stop_monitoring(struct ctdb_context *ctdb);
+void ctdb_start_monitoring(struct ctdb_context *ctdb);
+void ctdb_send_keepalive(struct ctdb_context *ctdb, uint32_t destnode);
+
+void ctdb_daemon_cancel_controls(struct ctdb_context *ctdb, struct ctdb_node *node);
+void ctdb_call_resend_all(struct ctdb_context *ctdb);
+void ctdb_node_dead(struct ctdb_node *node);
+void ctdb_node_connected(struct ctdb_node *node);
+bool ctdb_blocking_freeze(struct ctdb_context *ctdb);
+int32_t ctdb_control_max_rsn(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int32_t ctdb_control_set_rsn_nonempty(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int32_t ctdb_control_delete_low_rsn(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_ctrl_get_max_rsn(struct ctdb_context *ctdb, struct timeval timeout, 
+			  uint32_t destnode, uint32_t db_id, uint64_t *max_rsn);
+int ctdb_ctrl_set_rsn_nonempty(struct ctdb_context *ctdb, struct timeval timeout, 
+			       uint32_t destnode, uint32_t db_id, uint64_t rsn);
+int ctdb_ctrl_delete_low_rsn(struct ctdb_context *ctdb, struct timeval timeout, 
+			     uint32_t destnode, uint32_t db_id, uint64_t rsn);
+void ctdb_set_realtime(bool enable);
+int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply);
+int32_t ctdb_control_release_ip(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply);
+
+struct ctdb_public_ip {
+	uint32_t vnn;
+	uint32_t takeover_vnn;
+	struct sockaddr_in sin;
+};
+int ctdb_ctrl_takeover_ip(struct ctdb_context *ctdb, struct timeval timeout, 
+			  uint32_t destnode, struct ctdb_public_ip *ip);
+int ctdb_ctrl_release_ip(struct ctdb_context *ctdb, struct timeval timeout, 
+			 uint32_t destnode, struct ctdb_public_ip *ip);
+
+struct ctdb_all_public_ips {
+	uint32_t num;
+	struct ctdb_public_ip ips[1];
+};
+int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA *outdata);
+int ctdb_ctrl_get_public_ips(struct ctdb_context *ctdb, 
+			struct timeval timeout, uint32_t destnode, 
+			TALLOC_CTX *mem_ctx, struct ctdb_all_public_ips **ips);
+
+
+/* from takeover/system.c */
+int ctdb_sys_send_arp(const struct sockaddr_in *saddr, const char *iface);
+bool ctdb_sys_have_ip(const char *ip);
+int ctdb_sys_send_tcp(const struct sockaddr_in *dest, 
+		      const struct sockaddr_in *src,
+		      uint32_t seq, uint32_t ack, int rst);
+int ctdb_sys_kill_tcp(struct event_context *ev,
+		      const struct sockaddr_in *dest, 
+		      const struct sockaddr_in *src);
+
+int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist);
+int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script);
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap);
+
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, 
+				uint32_t srcnode, TDB_DATA indata);
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn);
+
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client);
+int ctdb_event_script(struct ctdb_context *ctdb, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3);
+int ctdb_event_script_callback(struct ctdb_context *ctdb, 
+			       struct timeval timeout,
+			       TALLOC_CTX *mem_ctx,
+			       void (*callback)(struct ctdb_context *, int, void *),
+			       void *private_data,
+			       const char *fmt, ...) PRINTF_ATTRIBUTE(6,7);
+void ctdb_release_all_ips(struct ctdb_context *ctdb);
+
+void set_nonblocking(int fd);
+void set_close_on_exec(int fd);
+
+bool ctdb_recovery_lock(struct ctdb_context *ctdb, bool keep);
+
+int ctdb_set_recovery_lock_file(struct ctdb_context *ctdb, const char *file);
+
+int32_t ctdb_control_get_tunable(struct ctdb_context *ctdb, TDB_DATA indata, 
+				 TDB_DATA *outdata);
+int32_t ctdb_control_set_tunable(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_list_tunables(struct ctdb_context *ctdb, TDB_DATA *outdata);
+
+void ctdb_tunables_set_defaults(struct ctdb_context *ctdb);
+
+int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata);
+
+int ctdb_ctrl_get_all_tunables(struct ctdb_context *ctdb, 
+			       struct timeval timeout, 
+			       uint32_t destnode,
+			       struct ctdb_tunable *tunables);
+
+void ctdb_start_freeze(struct ctdb_context *ctdb);
+
+bool parse_ip_port(const char *s, struct sockaddr_in *ip);
+
 #endif
diff --git a/source4/cluster/ctdb/include/includes.h b/source4/cluster/ctdb/include/includes.h
index bffc66b358..a345fa2e81 100644
--- a/source4/cluster/ctdb/include/includes.h
+++ b/source4/cluster/ctdb/include/includes.h
@@ -5,7 +5,6 @@
 #include "tdb.h"
 #include "idtree.h"
 #include "ctdb.h"
-#include "lib/util/dlinklist.h"
 #include "lib/util/debug.h"
 
 typedef bool BOOL;
@@ -35,3 +34,7 @@ struct timeval timeval_until(const struct timeval *tv1,
 _PUBLIC_ struct timeval timeval_current_ofs(uint32_t secs, uint32_t usecs);
 double timeval_elapsed(struct timeval *tv);
 char **file_lines_load(const char *fname, int *numlines, TALLOC_CTX *mem_ctx);
+char *hex_encode(TALLOC_CTX *mem_ctx, const unsigned char *buff_in, size_t len);
+_PUBLIC_ const char **str_list_add(const char **list, const char *s);
+_PUBLIC_ int set_blocking(int fd, BOOL set);
+
diff --git a/source4/cluster/ctdb/packaging/RHEL/ctdb.spec b/source4/cluster/ctdb/packaging/RHEL/ctdb.spec
new file mode 100644
index 0000000000..9bba5683b1
--- /dev/null
+++ b/source4/cluster/ctdb/packaging/RHEL/ctdb.spec
@@ -0,0 +1,117 @@
+%define initdir %{_sysconfdir}/rc.d/init.d
+
+Summary: Clustered TDB
+Vendor: Samba Team
+Packager: Samba Team <samba@samba.org>
+Name: ctdb
+Version: 1.0
+Release: 1
+Epoch: 0
+License: GNU GPL version 2
+Group: System Environment/Daemons
+URL: bzr://www.samba.org/~tridge/ctdb/
+
+Source: ctdb-%{version}.tar.bz2
+Source999: ctdb-setup.tar.bz2
+
+Prereq: /sbin/chkconfig /bin/mktemp /usr/bin/killall
+Prereq: fileutils sed /etc/init.d
+
+Requires: initscripts >= 5.54-1
+Provides: ctdb = %{version}
+
+Prefix: /usr
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+
+%description
+ctdb is the clustered database used by samba
+
+
+#######################################################################
+
+%prep
+%setup -q
+# setup the init script and sysconfig file
+%setup -T -D -a 999 -n ctdb-%{version} -q
+
+%build
+
+CC="gcc"
+
+## always run autogen.sh
+./autogen.sh
+
+CFLAGS="$RPM_OPT_FLAGS $EXTRA -D_GNU_SOURCE" ./configure \
+	--prefix=%{_prefix} \
+	--sysconfdir=%{_sysconfdir} \
+	--localstatedir="/var"
+
+make showflags
+make   
+
+%install
+# Clean up in case there is trash left from a previous build
+rm -rf $RPM_BUILD_ROOT
+
+# Create the target build directory hierarchy
+mkdir -p $RPM_BUILD_ROOT%{_includedir}
+mkdir -p $RPM_BUILD_ROOT{%{_libdir},%{_includedir}}
+mkdir -p $RPM_BUILD_ROOT%{_prefix}/{bin,sbin}
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ctdb
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d
+
+make DESTDIR=$RPM_BUILD_ROOT install
+
+install -m644 setup/ctdb.sysconfig $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/ctdb
+install -m755 setup/ctdb.init $RPM_BUILD_ROOT%{initdir}/ctdb
+install -m755 tools/events $RPM_BUILD_ROOT%{_sysconfdir}/ctdb/events
+install -m755 tools/onnode.ssh $RPM_BUILD_ROOT%{_bindir}
+install -m755 tools/onnode.rsh $RPM_BUILD_ROOT%{_bindir}
+ln -sf %{_bindir}/onnode.ssh $RPM_BUILD_ROOT%{_bindir}/onnode
+
+# unfortunately samba3 needs ctdb_private.h too
+install -m644 include/ctdb_private.h $RPM_BUILD_ROOT%{_includedir}/ctdb_private.h
+
+
+# Remove "*.old" files
+find $RPM_BUILD_ROOT -name "*.old" -exec rm -f {} \;
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%post
+/sbin/chkconfig --add ctdb
+
+%preun
+if [ $1 = 0 ] ; then
+    /sbin/chkconfig --del ctdb
+    /sbin/service ctdb stop >/dev/null 2>&1
+fi
+exit 0
+
+%postun
+if [ "$1" -ge "1" ]; then
+	%{initdir}/ctdb restart >/dev/null 2>&1
+fi	
+
+
+#######################################################################
+## Files section                                                     ##
+#######################################################################
+
+%files
+%defattr(-,root,root)
+
+%config(noreplace) %{_sysconfdir}/sysconfig/ctdb
+%attr(755,root,root) %config %{initdir}/ctdb
+
+%{_sysconfdir}/ctdb/events
+%{_sbindir}/ctdbd
+%{_bindir}/ctdb
+%{_bindir}/onnode.ssh
+%{_bindir}/onnode.rsh
+%{_bindir}/onnode
+%{_includedir}/ctdb.h
+%{_includedir}/ctdb_private.h
+
diff --git a/source4/cluster/ctdb/packaging/RHEL/makerpms.sh b/source4/cluster/ctdb/packaging/RHEL/makerpms.sh
new file mode 100755
index 0000000000..e43d53683c
--- /dev/null
+++ b/source4/cluster/ctdb/packaging/RHEL/makerpms.sh
@@ -0,0 +1,90 @@
+#!/bin/sh
+# Copyright (C) John H Terpstra 1998-2002
+#               Gerald (Jerry) Carter 2003
+#		Jim McDonough 2007
+#		Andrew Tridgell 2007
+
+# The following allows environment variables to override the target directories
+#   the alternative is to have a file in your home directory calles .rpmmacros
+#   containing the following:
+#   %_topdir  /home/mylogin/redhat
+#
+# Note: Under this directory rpm expects to find the same directories that are under the
+#   /usr/src/redhat directory
+#
+
+EXTRA_OPTIONS="$1"
+
+RHEL="packaging/RHEL"
+
+[ -d ${RHEL} ] || {
+    echo "Must run this from the ctdb directory"
+    exit 1
+}
+
+
+SPECDIR=`rpm --eval %_specdir`
+SRCDIR=`rpm --eval %_sourcedir`
+
+# At this point the SPECDIR and SRCDIR vaiables must have a value!
+
+USERID=`id -u`
+GRPID=`id -g`
+VERSION='1.0'
+REVISION=''
+SPECFILE="ctdb.spec"
+RPMVER=`rpm --version | awk '{print $3}'`
+RPMBUILD="rpmbuild"
+
+##
+## Check the RPM version (paranoid)
+##
+case $RPMVER in
+    4*)
+       echo "Supported RPM version [$RPMVER]"
+       ;;
+    *)
+       echo "Unknown RPM version: `rpm --version`"
+       exit 1
+       ;;
+esac
+
+if [ -f Makefile ]; then 
+	make distclean
+fi
+
+pushd .
+BASEDIR=`basename $PWD`
+cd ..
+chown -R ${USERID}.${GRPID} $BASEDIR
+if [ ! -d ctdb-${VERSION} ]; then
+	ln -s $BASEDIR ctdb-${VERSION} || exit 1
+	REMOVE_LN=$PWD/ctdb-$VERSION
+fi
+echo -n "Creating ctdb-${VERSION}.tar.bz2 ... "
+tar --exclude=.bzr --exclude .bzrignore --exclude packaging --exclude="*~" -cf - ctdb-${VERSION}/. | bzip2 > ${SRCDIR}/ctdb-${VERSION}.tar.bz2
+echo "Done."
+if [ $? -ne 0 ]; then
+        echo "Build failed!"
+	[ ${REMOVE_LN} ] && rm $REMOVE_LN
+        exit 1
+fi
+
+popd
+
+
+##
+## copy additional source files
+##
+(cd packaging/RHEL && tar --exclude=.bzr --exclude="*~" -jcvf - setup) > ${SRCDIR}/ctdb-setup.tar.bz2
+cp -p ${RHEL}/${SPECFILE} ${SPECDIR}
+
+##
+## Build
+##
+echo "$(basename $0): Getting Ready to build release package"
+cd ${SPECDIR}
+${RPMBUILD} -ba --clean --rmsource $EXTRA_OPTIONS $SPECFILE
+
+echo "$(basename $0): Done."
+[ ${REMOVE_LN} ] && rm $REMOVE_LN
diff --git a/source4/cluster/ctdb/packaging/RHEL/setup/ctdb.init b/source4/cluster/ctdb/packaging/RHEL/setup/ctdb.init
new file mode 100755
index 0000000000..e34107e6b3
--- /dev/null
+++ b/source4/cluster/ctdb/packaging/RHEL/setup/ctdb.init
@@ -0,0 +1,104 @@
+#!/bin/sh
+#
+# chkconfig: - 90 36
+# description: Starts and stops the clustered tdb daemon
+#
+# pidfile: /var/run/ctdbd/ctdbd.pid
+
+# Source function library.
+if [ -f /etc/init.d/functions ] ; then
+  . /etc/init.d/functions
+elif [ -f /etc/rc.d/init.d/functions ] ; then
+  . /etc/rc.d/init.d/functions
+else
+  exit 0
+fi
+
+# Avoid using root's TMPDIR
+unset TMPDIR
+
+# Source networking configuration.
+. /etc/sysconfig/network
+
+CTDB_OPTIONS=""
+
+# pull in admin specified config 
+if [ -f /etc/sysconfig/ctdb ]; then
+   . /etc/sysconfig/ctdb 
+fi
+
+# Check that networking is up.
+[ ${NETWORKING} = "no" ] && exit 0
+
+# build up CTDB_OPTIONS variable
+[ -z "$LOGFILE" ]          || CTDB_OPTIONS="$CTDB_OPTIONS --logfile=$LOGFILE"
+[ -z "$NODES" ]            || CTDB_OPTIONS="$CTDB_OPTIONS --nlist=$NODES"
+[ -z "$CTDB_SOCKET" ]      || CTDB_OPTIONS="$CTDB_OPTIONS --socket=$CTDB_SOCKET"
+[ -z "$PUBLIC_ADDRESSES" ] || CTDB_OPTIONS="$CTDB_OPTIONS --public-addresses=$PUBLIC_ADDRESSES"
+[ -z "$PUBLIC_INTERFACE" ] || CTDB_OPTIONS="$CTDB_OPTIONS --public-interface=$PUBLIC_INTERFACE"
+[ -z "$DBDIR" ]            || CTDB_OPTIONS="$CTDB_OPTIONS --dbdir=$DBDIR"
+[ -z "$EVENT_SCRIPT" ]     || CTDB_OPTIONS="$CTDB_OPTIONS --event-script $EVENT_SCRIPT"
+[ -z "$TRANSPORT" ]        || CTDB_OPTIONS="$CTDB_OPTIONS --transport $TRANSPORT"
+[ -z "$DEBUGLEVEL" ]       || CTDB_OPTIONS="$CTDB_OPTIONS -d $DEBUGLEVEL"
+
+start() {
+	echo -n $"Starting ctdbd services: "
+	daemon ctdbd $CTDB_OPTIONS
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/ctdb || \
+	   RETVAL=1
+	return $RETVAL
+}	
+
+stop() {
+	echo -n $"Shutting down ctdbd services: "
+	ctdb shutdown
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/ctdb
+	echo ""
+	return $RETVAL
+}	
+
+restart() {
+	stop
+	start
+}	
+
+rhstatus() {
+	ctdb status
+	if [ $? -ne 0 ] ; then
+		return 1
+	fi
+}	
+
+
+# Allow status as non-root.
+if [ "$1" = status ]; then
+       rhstatus
+       exit $?
+fi
+
+case "$1" in
+  start)
+  	start
+	;;
+  stop)
+  	stop
+	;;
+  restart)
+  	restart
+	;;
+  status)
+  	rhstatus
+	;;
+  condrestart)
+  	[ -f /var/lock/subsys/ctdb ] && restart || :
+	;;
+  *)
+	echo $"Usage: $0 {start|stop|restart|status|condrestart}"
+	exit 1
+esac
+
+exit $?
diff --git a/source4/cluster/ctdb/packaging/RHEL/setup/ctdb.sysconfig b/source4/cluster/ctdb/packaging/RHEL/setup/ctdb.sysconfig
new file mode 100644
index 0000000000..13d83ce1ab
--- /dev/null
+++ b/source4/cluster/ctdb/packaging/RHEL/setup/ctdb.sysconfig
@@ -0,0 +1,52 @@
+# Options to ctdbd. This is read by /etc/init.d/ctdb
+
+# the NODES file must be specified or ctdb won't start
+# it should contain a list of IPs that ctdb will use
+# it must be exactly the same on all cluster nodes
+# defaults to /etc/ctdb/nodes
+# NODES=/etc/ctdb/nodes
+
+# the directory to put the local ctdb database files in
+# defaults to /var/ctdb
+# DBDIR=/var/ctdb
+
+# the script to run when ctdb needs to ask the OS for help,
+# such as when a IP address needs to be taken or released
+# defaults to /etc/ctdb/events
+# EVENT_SCRIPT=/etc/ctdb/events
+
+# the location of the local ctdb socket
+# defaults to /tmp/ctdb.socket
+# CTDB_SOCKET=/tmp/ctdb.socket
+
+# what transport to use. Only tcp is currently supported
+# defaults to tcp
+# TRANSPORT="tcp"
+
+# should ctdb do IP takeover? If it should, then specify a file
+# containing the list of public IP addresses that ctdb will manage
+# Note that these IPs must be different from those in $NODES above
+# there is no default
+# PUBLIC_ADDRESSES=/etc/ctdb/public_addresses
+
+# when doing IP takeover you also must specify what network interface
+# to use for the public addresses
+# there is no default
+# PUBLIC_INTERFACE=eth0
+
+# where to log messages
+# the default is /var/log/log.ctdb
+# LOGFILE=/var/log/log.ctdb
+
+# what debug level to run at. Higher means more verbose
+# the default is 0
+# DEBUGLEVEL=0
+
+# use this to specify any local tcp ports to wait on before starting
+# ctdb. Use 445 and 139 for Samba
+# the default is not to wait for any local services
+# CTDB_WAIT_TCP_PORTS="445 139"
+
+# any other options you might want. Run ctdbd --help for a list
+# CTDB_OPTIONS=
+
diff --git a/source4/cluster/ctdb/packaging/RPM/ctdb.spec b/source4/cluster/ctdb/packaging/RPM/ctdb.spec
new file mode 100644
index 0000000000..6920e643ae
--- /dev/null
+++ b/source4/cluster/ctdb/packaging/RPM/ctdb.spec
@@ -0,0 +1,112 @@
+%define initdir %{_sysconfdir}/init.d
+
+Summary: Clustered TDB
+Vendor: Samba Team
+Packager: Samba Team <samba@samba.org>
+Name: ctdb
+Version: 1.0
+Release: 7
+Epoch: 0
+License: GNU GPL version 3
+Group: System Environment/Daemons
+URL: http://ctdb.samba.org/
+
+Source: ctdb-%{version}.tar.gz
+
+Prereq: /sbin/chkconfig /bin/mktemp /usr/bin/killall
+Prereq: fileutils sed /etc/init.d
+
+Provides: ctdb = %{version}
+
+Prefix: /usr
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+
+%description
+ctdb is the clustered database used by samba
+
+
+#######################################################################
+
+%prep
+%setup -q
+# setup the init script and sysconfig file
+%setup -T -D -n ctdb-%{version} -q
+
+%build
+
+CC="gcc"
+
+## always run autogen.sh
+./autogen.sh
+
+CFLAGS="$RPM_OPT_FLAGS $EXTRA -D_GNU_SOURCE" ./configure \
+	--prefix=%{_prefix} \
+	--sysconfdir=%{_sysconfdir} \
+	--mandir=%{_mandir} \
+	--localstatedir="/var"
+
+make showflags
+make   
+
+%install
+# Clean up in case there is trash left from a previous build
+rm -rf $RPM_BUILD_ROOT
+
+# Create the target build directory hierarchy
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/init.d
+
+make DESTDIR=$RPM_BUILD_ROOT install
+
+install -m644 config/ctdb.sysconfig $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/ctdb
+install -m755 config/ctdb.init $RPM_BUILD_ROOT%{initdir}/ctdb
+
+# Remove "*.old" files
+find $RPM_BUILD_ROOT -name "*.old" -exec rm -f {} \;
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%post
+[ -x /sbin/chkconfig ] && /sbin/chkconfig --add ctdb
+
+%preun
+if [ $1 = 0 ] ; then
+    [ -x /sbin/chkconfig ] && /sbin/chkconfig --del ctdb
+fi
+exit 0
+
+%postun
+if [ "$1" -ge "1" ]; then
+	%{initdir}/ctdb restart >/dev/null 2>&1
+fi	
+
+
+#######################################################################
+## Files section                                                     ##
+#######################################################################
+
+%files
+%defattr(-,root,root)
+
+%config(noreplace) %{_sysconfdir}/sysconfig/ctdb
+%attr(755,root,root) %config %{initdir}/ctdb
+
+%{_sysconfdir}/ctdb/events
+%{_sysconfdir}/ctdb/functions
+%{_sysconfdir}/ctdb/events.d/10.interface
+%{_sysconfdir}/ctdb/events.d/40.vsftpd
+%{_sysconfdir}/ctdb/events.d/50.samba
+%{_sysconfdir}/ctdb/events.d/60.nfs
+%{_sysconfdir}/ctdb/events.d/61.nfstickle
+%{_sysconfdir}/ctdb/statd-callout
+%{_sbindir}/ctdbd
+%{_bindir}/ctdb
+%{_bindir}/onnode.ssh
+%{_bindir}/onnode.rsh
+%{_bindir}/onnode
+%{_mandir}/man1/ctdb.1.gz
+%{_mandir}/man1/ctdbd.1.gz
+%{_mandir}/man1/onnode.1.gz
+%{_includedir}/ctdb.h
+%{_includedir}/ctdb_private.h
diff --git a/source4/cluster/ctdb/packaging/RPM/makerpms.sh b/source4/cluster/ctdb/packaging/RPM/makerpms.sh
new file mode 100755
index 0000000000..c267ed17e6
--- /dev/null
+++ b/source4/cluster/ctdb/packaging/RPM/makerpms.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+# Copyright (C) John H Terpstra 1998-2002
+#               Gerald (Jerry) Carter 2003
+#		Jim McDonough 2007
+#		Andrew Tridgell 2007
+
+# The following allows environment variables to override the target directories
+#   the alternative is to have a file in your home directory calles .rpmmacros
+#   containing the following:
+#   %_topdir  /home/mylogin/redhat
+#
+# Note: Under this directory rpm expects to find the same directories that are under the
+#   /usr/src/redhat directory
+#
+
+EXTRA_OPTIONS="$1"
+
+[ -d packaging ] || {
+    echo "Must run this from the ctdb directory"
+    exit 1
+}
+
+
+SPECDIR=`rpm --eval %_specdir`
+SRCDIR=`rpm --eval %_sourcedir`
+
+# At this point the SPECDIR and SRCDIR vaiables must have a value!
+
+USERID=`id -u`
+GRPID=`id -g`
+VERSION='1.0'
+REVISION=''
+SPECFILE="ctdb.spec"
+RPMVER=`rpm --version | awk '{print $3}'`
+RPMBUILD="rpmbuild"
+
+##
+## Check the RPM version (paranoid)
+##
+case $RPMVER in
+    4*)
+       echo "Supported RPM version [$RPMVER]"
+       ;;
+    *)
+       echo "Unknown RPM version: `rpm --version`"
+       exit 1
+       ;;
+esac
+
+if [ -f Makefile ]; then 
+	make distclean
+fi
+
+pushd .
+BASEDIR=`basename $PWD`
+cd ..
+chown -R ${USERID}.${GRPID} $BASEDIR
+if [ ! -d ctdb-${VERSION} ]; then
+	ln -s $BASEDIR ctdb-${VERSION} || exit 1
+	REMOVE_LN=$PWD/ctdb-$VERSION
+fi
+echo -n "Creating ctdb-${VERSION}.tar.gz ... "
+tar --exclude=.bzr --exclude .bzrignore --exclude="*~" -cf - ctdb-${VERSION}/. | gzip -9 --rsyncable > ${SRCDIR}/ctdb-${VERSION}.tar.gz
+echo "Done."
+if [ $? -ne 0 ]; then
+        echo "Build failed!"
+	[ ${REMOVE_LN} ] && rm $REMOVE_LN
+        exit 1
+fi
+
+popd
+
+
+##
+## copy additional source files
+##
+cp -p packaging/RPM/ctdb.spec ${SPECDIR}
+
+##
+## Build
+##
+echo "$(basename $0): Getting Ready to build release package"
+cd ${SPECDIR}
+${RPMBUILD} -ba --clean --rmsource $EXTRA_OPTIONS $SPECFILE || exit 1
+
+echo "$(basename $0): Done."
+[ ${REMOVE_LN} ] && /bin/rm -f $REMOVE_LN
+
+exit 0
diff --git a/source4/cluster/ctdb/common/ctdb_call.c b/source4/cluster/ctdb/server/ctdb_call.c
index 67ab076308..bbe07717ed 100644
--- a/source4/cluster/ctdb/common/ctdb_call.c
+++ b/source4/cluster/ctdb/server/ctdb_call.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 /*
   see http://wiki.samba.org/index.php/Samba_%26_Clustering for
@@ -23,6 +23,7 @@
 #include "includes.h"
 #include "lib/events/events.h"
 #include "lib/tdb/include/tdb.h"
+#include "lib/util/dlinklist.h"
 #include "system/network.h"
 #include "system/filesys.h"
 #include "../include/ctdb_private.h"
@@ -44,78 +45,15 @@
 
 
 /*
-  local version of ctdb_call
+  a varient of input packet that can be used in lock requeue
 */
-int ctdb_call_local(struct ctdb_db_context *ctdb_db, struct ctdb_call *call,
-		    struct ctdb_ltdb_header *header, TDB_DATA *data,
-		    uint32_t caller)
+static void ctdb_call_input_pkt(void *p, struct ctdb_req_header *hdr)
 {
-	struct ctdb_call_info *c;
-	struct ctdb_registered_call *fn;
-	struct ctdb_context *ctdb = ctdb_db->ctdb;
-	
-	c = talloc(ctdb, struct ctdb_call_info);
-	CTDB_NO_MEMORY(ctdb, c);
-
-	c->key = call->key;
-	c->call_data = &call->call_data;
-	c->record_data.dptr = talloc_memdup(c, data->dptr, data->dsize);
-	c->record_data.dsize = data->dsize;
-	CTDB_NO_MEMORY(ctdb, c->record_data.dptr);
-	c->new_data = NULL;
-	c->reply_data = NULL;
-	c->status = 0;
-
-	for (fn=ctdb_db->calls;fn;fn=fn->next) {
-		if (fn->id == call->call_id) break;
-	}
-	if (fn == NULL) {
-		ctdb_set_error(ctdb, "Unknown call id %u\n", call->call_id);
-		talloc_free(c);
-		return -1;
-	}
-
-	if (fn->fn(c) != 0) {
-		ctdb_set_error(ctdb, "ctdb_call %u failed\n", call->call_id);
-		talloc_free(c);
-		return -1;
-	}
-
-	if (header->laccessor != caller) {
-		header->lacount = 0;
-	}
-	header->laccessor = caller;
-	header->lacount++;
-
-	/* we need to force the record to be written out if this was a remote access,
-	   so that the lacount is updated */
-	if (c->new_data == NULL && header->laccessor != ctdb->vnn) {
-		c->new_data = &c->record_data;
-	}
-
-	if (c->new_data) {
-		if (ctdb_ltdb_store(ctdb_db, call->key, header, *c->new_data) != 0) {
-			ctdb_set_error(ctdb, "ctdb_call tdb_store failed\n");
-			talloc_free(c);
-			return -1;
-		}
-	}
-
-	if (c->reply_data) {
-		call->reply_data = *c->reply_data;
-		talloc_steal(ctdb, call->reply_data.dptr);
-		talloc_set_name_const(call->reply_data.dptr, __location__);
-	} else {
-		call->reply_data.dptr = NULL;
-		call->reply_data.dsize = 0;
-	}
-	call->status = c->status;
-
-	talloc_free(c);
-
-	return 0;
+	struct ctdb_context *ctdb = talloc_get_type(p, struct ctdb_context);
+	ctdb_input_pkt(ctdb, hdr);
 }
 
+
 /*
   send an error reply
 */
@@ -140,16 +78,11 @@ static void ctdb_send_error(struct ctdb_context *ctdb,
 
 	msglen = strlen(msg)+1;
 	len = offsetof(struct ctdb_reply_error, msg);
-	r = ctdb->methods->allocate_pkt(msg, len + msglen);
+	r = ctdb_transport_allocate(ctdb, msg, CTDB_REPLY_ERROR, len + msglen, 
+				    struct ctdb_reply_error);
 	CTDB_NO_MEMORY_FATAL(ctdb, r);
-	talloc_set_name_const(r, "send_error packet");
 
-	r->hdr.length    = len + msglen;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REPLY_ERROR;
 	r->hdr.destnode  = hdr->srcnode;
-	r->hdr.srcnode   = ctdb->vnn;
 	r->hdr.reqid     = hdr->reqid;
 	r->status        = status;
 	r->msglen        = msglen;
@@ -165,26 +98,75 @@ static void ctdb_send_error(struct ctdb_context *ctdb,
   send a redirect reply
 */
 static void ctdb_call_send_redirect(struct ctdb_context *ctdb, 
+				    TDB_DATA key,
 				    struct ctdb_req_call *c, 
 				    struct ctdb_ltdb_header *header)
 {
-	struct ctdb_reply_redirect *r;
+	
+	uint32_t lmaster = ctdb_lmaster(ctdb, &key);
+	if (ctdb->vnn == lmaster) {
+		c->hdr.destnode = header->dmaster;
+	} else if ((c->hopcount % ctdb->tunable.max_redirect_count) == 0) {
+		c->hdr.destnode = lmaster;
+	} else {
+		c->hdr.destnode = header->dmaster;
+	}
+	c->hopcount++;
+	ctdb_queue_packet(ctdb, &c->hdr);
+}
+
+
+/*
+  send a dmaster reply
+
+  caller must have the chainlock before calling this routine. Caller must be
+  the lmaster
+*/
+static void ctdb_send_dmaster_reply(struct ctdb_db_context *ctdb_db,
+				    struct ctdb_ltdb_header *header,
+				    TDB_DATA key, TDB_DATA data,
+				    uint32_t new_dmaster,
+				    uint32_t reqid)
+{
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	struct ctdb_reply_dmaster *r;
+	int ret, len;
+	TALLOC_CTX *tmp_ctx;
+
+	if (ctdb->vnn != ctdb_lmaster(ctdb, &key)) {
+		DEBUG(0,(__location__ " Caller is not lmaster!\n"));
+		return;
+	}
+
+	header->dmaster = new_dmaster;
+	ret = ctdb_ltdb_store(ctdb_db, key, header, data);
+	if (ret != 0) {
+		ctdb_fatal(ctdb, "ctdb_req_dmaster unable to update dmaster");
+		return;
+	}
+
+	/* put the packet on a temporary context, allowing us to safely free
+	   it below even if ctdb_reply_dmaster() has freed it already */
+	tmp_ctx = talloc_new(ctdb);
 
-	r = ctdb->methods->allocate_pkt(ctdb, sizeof(*r));
+	/* send the CTDB_REPLY_DMASTER */
+	len = offsetof(struct ctdb_reply_dmaster, data) + key.dsize + data.dsize;
+	r = ctdb_transport_allocate(ctdb, tmp_ctx, CTDB_REPLY_DMASTER, len,
+				    struct ctdb_reply_dmaster);
 	CTDB_NO_MEMORY_FATAL(ctdb, r);
-	talloc_set_name_const(r, "send_redirect packet");
-	r->hdr.length = sizeof(*r);
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REPLY_REDIRECT;
-	r->hdr.destnode  = c->hdr.srcnode;
-	r->hdr.srcnode   = ctdb->vnn;
-	r->hdr.reqid     = c->hdr.reqid;
-	r->dmaster       = header->dmaster;
+
+	r->hdr.destnode  = new_dmaster;
+	r->hdr.reqid     = reqid;
+	r->rsn           = header->rsn;
+	r->keylen        = key.dsize;
+	r->datalen       = data.dsize;
+	r->db_id         = ctdb_db->db_id;
+	memcpy(&r->data[0], key.dptr, key.dsize);
+	memcpy(&r->data[key.dsize], data.dptr, data.dsize);
 
 	ctdb_queue_packet(ctdb, &r->hdr);
 
-	talloc_free(r);
+	talloc_free(tmp_ctx);
 }
 
 /*
@@ -202,35 +184,91 @@ static void ctdb_call_send_dmaster(struct ctdb_db_context *ctdb_db,
 	struct ctdb_req_dmaster *r;
 	struct ctdb_context *ctdb = ctdb_db->ctdb;
 	int len;
+	uint32_t lmaster = ctdb_lmaster(ctdb, key);
+
+	if (lmaster == ctdb->vnn) {
+		ctdb_send_dmaster_reply(ctdb_db, header, *key, *data, 
+					c->hdr.srcnode, c->hdr.reqid);
+		return;
+	}
 	
 	len = offsetof(struct ctdb_req_dmaster, data) + key->dsize + data->dsize;
-	r = ctdb->methods->allocate_pkt(ctdb, len);
+	r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REQ_DMASTER, len, 
+				    struct ctdb_req_dmaster);
 	CTDB_NO_MEMORY_FATAL(ctdb, r);
-	talloc_set_name_const(r, "send_dmaster packet");
-	r->hdr.length    = len;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REQ_DMASTER;
-	r->hdr.destnode  = ctdb_lmaster(ctdb, key);
-	r->hdr.srcnode   = ctdb->vnn;
+	r->hdr.destnode  = lmaster;
 	r->hdr.reqid     = c->hdr.reqid;
 	r->db_id         = c->db_id;
+	r->rsn           = header->rsn;
 	r->dmaster       = c->hdr.srcnode;
 	r->keylen        = key->dsize;
 	r->datalen       = data->dsize;
 	memcpy(&r->data[0], key->dptr, key->dsize);
 	memcpy(&r->data[key->dsize], data->dptr, data->dsize);
 
-	/* XXX - probably not necessary when lmaster==dmaster
-	   update the ltdb to record the new dmaster */
-	header->dmaster = r->hdr.destnode;
-	ctdb_ltdb_store(ctdb_db, *key, header, *data);
+	header->dmaster = c->hdr.srcnode;
+	if (ctdb_ltdb_store(ctdb_db, *key, header, *data) != 0) {
+		ctdb_fatal(ctdb, "Failed to store record in ctdb_call_send_dmaster");
+	}
 	
 	ctdb_queue_packet(ctdb, &r->hdr);
 
 	talloc_free(r);
 }
 
+/*
+  called when a CTDB_REPLY_DMASTER packet comes in, or when the lmaster
+  gets a CTDB_REQUEST_DMASTER for itself. We become the dmaster.
+
+  must be called with the chainlock held. This function releases the chainlock
+*/
+static void ctdb_become_dmaster(struct ctdb_db_context *ctdb_db, 
+				uint32_t reqid, TDB_DATA key, TDB_DATA data,
+				uint64_t rsn)
+{
+	struct ctdb_call_state *state;
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	struct ctdb_ltdb_header header;
+
+	DEBUG(2,("vnn %u dmaster response %08x\n", ctdb->vnn, ctdb_hash(&key)));
+
+	ZERO_STRUCT(header);
+	header.rsn = rsn + 1;
+	header.dmaster = ctdb->vnn;
+
+	if (ctdb_ltdb_store(ctdb_db, key, &header, data) != 0) {
+		ctdb_fatal(ctdb, "ctdb_reply_dmaster store failed\n");
+		ctdb_ltdb_unlock(ctdb_db, key);
+		return;
+	}
+
+	state = ctdb_reqid_find(ctdb, reqid, struct ctdb_call_state);
+
+	if (state == NULL) {
+		DEBUG(0,("vnn %u Invalid reqid %u in ctdb_become_dmaster\n",
+			 ctdb->vnn, reqid));
+		ctdb_ltdb_unlock(ctdb_db, key);
+		return;
+	}
+
+	if (reqid != state->reqid) {
+		/* we found a record  but it was the wrong one */
+		DEBUG(0, ("Dropped orphan in ctdb_become_dmaster with reqid:%u\n",reqid));
+		ctdb_ltdb_unlock(ctdb_db, key);
+		return;
+	}
+
+	ctdb_call_local(ctdb_db, &state->call, &header, state, &data, ctdb->vnn);
+
+	ctdb_ltdb_unlock(ctdb_db, state->call.key);
+
+	state->state = CTDB_CALL_DONE;
+	if (state->async.fn) {
+		state->async.fn(state);
+	}
+}
+
+
 
 /*
   called when a CTDB_REQ_DMASTER packet comes in
@@ -241,12 +279,10 @@ static void ctdb_call_send_dmaster(struct ctdb_db_context *ctdb_db,
 void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 {
 	struct ctdb_req_dmaster *c = (struct ctdb_req_dmaster *)hdr;
-	struct ctdb_reply_dmaster *r;
 	TDB_DATA key, data, data2;
 	struct ctdb_ltdb_header header;
 	struct ctdb_db_context *ctdb_db;
-	int ret, len;
-	TALLOC_CTX *tmp_ctx;
+	int ret;
 
 	key.dptr = c->data;
 	key.dsize = c->keylen;
@@ -263,7 +299,7 @@ void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr
 	
 	/* fetch the current record */
 	ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, key, &header, hdr, &data2,
-					   ctdb_recv_raw_pkt, ctdb);
+					   ctdb_call_input_pkt, ctdb, False);
 	if (ret == -1) {
 		ctdb_fatal(ctdb, "ctdb_req_dmaster failed to fetch record");
 		return;
@@ -272,45 +308,37 @@ void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr
 		DEBUG(2,(__location__ " deferring ctdb_request_dmaster\n"));
 		return;
 	}
-	
-	/* its a protocol error if the sending node is not the current dmaster */
-	if (header.dmaster != hdr->srcnode && 
-	    hdr->srcnode != ctdb_lmaster(ctdb_db->ctdb, &key)) {
-		ctdb_fatal(ctdb, "dmaster request from non-master");
-		return;
-	}
-	
-	header.dmaster = c->dmaster;
-	ret = ctdb_ltdb_store(ctdb_db, key, &header, data);
-	ctdb_ltdb_unlock(ctdb_db, key);
-	if (ret != 0) {
-		ctdb_fatal(ctdb, "ctdb_req_dmaster unable to update dmaster");
-		return;
-	}
 
-	/* put the packet on a temporary context, allowing us to safely free
-	   it below even if ctdb_reply_dmaster() has freed it already */
-	tmp_ctx = talloc_new(ctdb);
+	if (ctdb_lmaster(ctdb, &key) != ctdb->vnn) {
+		DEBUG(0,("vnn %u dmaster request to non-lmaster lmaster=%u gen=%u curgen=%u\n",
+			 ctdb->vnn, ctdb_lmaster(ctdb, &key), 
+			 hdr->generation, ctdb->vnn_map->generation));
+		ctdb_fatal(ctdb, "ctdb_req_dmaster to non-lmaster");
+	}
 
-	/* send the CTDB_REPLY_DMASTER */
-	len = offsetof(struct ctdb_reply_dmaster, data) + data.dsize;
-	r = ctdb->methods->allocate_pkt(tmp_ctx, len);
-	CTDB_NO_MEMORY_FATAL(ctdb, r);
+	DEBUG(2,("vnn %u dmaster request on %08x for %u from %u\n", 
+		 ctdb->vnn, ctdb_hash(&key), c->dmaster, c->hdr.srcnode));
 
-	talloc_set_name_const(r, "reply_dmaster packet");
-	r->hdr.length    = len;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REPLY_DMASTER;
-	r->hdr.destnode  = c->dmaster;
-	r->hdr.srcnode   = ctdb->vnn;
-	r->hdr.reqid     = hdr->reqid;
-	r->datalen       = data.dsize;
-	memcpy(&r->data[0], data.dptr, data.dsize);
+	/* its a protocol error if the sending node is not the current dmaster */
+	if (header.dmaster != hdr->srcnode) {
+		DEBUG(0,("vnn %u dmaster request non-master %u dmaster=%u key %08x dbid 0x%08x gen=%u curgen=%u\n",
+			 ctdb->vnn, hdr->srcnode, header.dmaster, ctdb_hash(&key),
+			 ctdb_db->db_id, hdr->generation, ctdb->vnn_map->generation));
+		ctdb_fatal(ctdb, "ctdb_req_dmaster from non-master");
+		return;
+	}
 
-	ctdb_queue_packet(ctdb, &r->hdr);
+	/* use the rsn from the sending node */
+	header.rsn = c->rsn;
 
-	talloc_free(tmp_ctx);
+	/* check if the new dmaster is the lmaster, in which case we
+	   skip the dmaster reply */
+	if (c->dmaster == ctdb->vnn) {
+		ctdb_become_dmaster(ctdb_db, hdr->reqid, key, data, c->rsn);
+	} else {
+		ctdb_send_dmaster_reply(ctdb_db, &header, key, data, c->dmaster, hdr->reqid);
+		ctdb_ltdb_unlock(ctdb_db, key);
+	}
 }
 
 
@@ -346,7 +374,7 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	   if the call will be answered locally */
 
 	ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, call.key, &header, hdr, &data,
-					   ctdb_recv_raw_pkt, ctdb);
+					   ctdb_call_input_pkt, ctdb, False);
 	if (ret == -1) {
 		ctdb_send_error(ctdb, hdr, ret, "ltdb fetch failed in ctdb_request_call");
 		return;
@@ -359,45 +387,46 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	/* if we are not the dmaster, then send a redirect to the
 	   requesting node */
 	if (header.dmaster != ctdb->vnn) {
-		ctdb_call_send_redirect(ctdb, c, &header);
 		talloc_free(data.dptr);
+		ctdb_call_send_redirect(ctdb, call.key, c, &header);
 		ctdb_ltdb_unlock(ctdb_db, call.key);
 		return;
 	}
 
+	if (c->hopcount > ctdb->statistics.max_hop_count) {
+		ctdb->statistics.max_hop_count = c->hopcount;
+	}
+
 	/* if this nodes has done enough consecutive calls on the same record
 	   then give them the record
 	   or if the node requested an immediate migration
 	*/
-	if ( (header.laccessor == c->hdr.srcnode
-	      && header.lacount >= ctdb->max_lacount)
-	   || c->flags&CTDB_IMMEDIATE_MIGRATION ) {
+	if ( c->hdr.srcnode != ctdb->vnn &&
+	     ((header.laccessor == c->hdr.srcnode
+	       && header.lacount >= ctdb->tunable.max_lacount)
+	      || (c->flags & CTDB_IMMEDIATE_MIGRATION)) ) {
+		DEBUG(2,("vnn %u starting migration of %08x to %u\n", 
+			 ctdb->vnn, ctdb_hash(&call.key), c->hdr.srcnode));
 		ctdb_call_send_dmaster(ctdb_db, c, &header, &call.key, &data);
 		talloc_free(data.dptr);
 		ctdb_ltdb_unlock(ctdb_db, call.key);
 		return;
 	}
 
-	ctdb_call_local(ctdb_db, &call, &header, &data, c->hdr.srcnode);
+	ctdb_call_local(ctdb_db, &call, &header, hdr, &data, c->hdr.srcnode);
 
 	ctdb_ltdb_unlock(ctdb_db, call.key);
 
 	len = offsetof(struct ctdb_reply_call, data) + call.reply_data.dsize;
-	r = ctdb->methods->allocate_pkt(ctdb, len);
+	r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REPLY_CALL, len, 
+				    struct ctdb_reply_call);
 	CTDB_NO_MEMORY_FATAL(ctdb, r);
-	talloc_set_name_const(r, "reply_call packet");
-	r->hdr.length    = len;
-	r->hdr.ctdb_magic = CTDB_MAGIC;
-	r->hdr.ctdb_version = CTDB_VERSION;
-	r->hdr.operation = CTDB_REPLY_CALL;
 	r->hdr.destnode  = hdr->srcnode;
-	r->hdr.srcnode   = hdr->destnode;
 	r->hdr.reqid     = hdr->reqid;
 	r->status        = call.status;
 	r->datalen       = call.reply_data.dsize;
 	if (call.reply_data.dsize) {
 		memcpy(&r->data[0], call.reply_data.dptr, call.reply_data.dsize);
-		talloc_free(call.reply_data.dptr);
 	}
 
 	ctdb_queue_packet(ctdb, &r->hdr);
@@ -416,9 +445,15 @@ void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	struct ctdb_reply_call *c = (struct ctdb_reply_call *)hdr;
 	struct ctdb_call_state *state;
 
-	state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_call_state);
+	state = ctdb_reqid_find(ctdb, hdr->reqid, struct ctdb_call_state);
 	if (state == NULL) {
-		DEBUG(0, ("reqid %d not found\n", hdr->reqid));
+		DEBUG(0, (__location__ " reqid %u not found\n", hdr->reqid));
+		return;
+	}
+
+	if (hdr->reqid != state->reqid) {
+		/* we found a record  but it was the wrong one */
+		DEBUG(0, ("Dropped orphaned call reply with reqid:%u\n",hdr->reqid));
 		return;
 	}
 
@@ -434,6 +469,7 @@ void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	}
 }
 
+
 /*
   called when a CTDB_REPLY_DMASTER packet comes in
 
@@ -444,20 +480,23 @@ void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 void ctdb_reply_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 {
 	struct ctdb_reply_dmaster *c = (struct ctdb_reply_dmaster *)hdr;
-	struct ctdb_call_state *state;
 	struct ctdb_db_context *ctdb_db;
-	TDB_DATA data;
+	TDB_DATA key, data;
 	int ret;
 
-	state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_call_state);
-	if (state == NULL) {
+	ctdb_db = find_ctdb_db(ctdb, c->db_id);
+	if (ctdb_db == NULL) {
+		DEBUG(0,("Unknown db_id 0x%x in ctdb_reply_dmaster\n", c->db_id));
 		return;
 	}
+	
+	key.dptr = c->data;
+	key.dsize = c->keylen;
+	data.dptr = &c->data[key.dsize];
+	data.dsize = c->datalen;
 
-	ctdb_db = state->ctdb_db;
-
-	ret = ctdb_ltdb_lock_requeue(ctdb_db, state->call.key, hdr,
-				     ctdb_recv_raw_pkt, ctdb);
+	ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr,
+				     ctdb_call_input_pkt, ctdb, False);
 	if (ret == -2) {
 		return;
 	}
@@ -466,31 +505,7 @@ void ctdb_reply_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 		return;
 	}
 
-	data.dptr = c->data;
-	data.dsize = c->datalen;
-
-	talloc_steal(state, c);
-
-	/* we're now the dmaster - update our local ltdb with new header
-	   and data */
-	state->header.dmaster = ctdb->vnn;
-
-	if (ctdb_ltdb_store(ctdb_db, state->call.key, &state->header, data) != 0) {
-		ctdb_ltdb_unlock(ctdb_db, state->call.key);
-		ctdb_fatal(ctdb, "ctdb_reply_dmaster store failed\n");
-		return;
-	}
-
-	ctdb_call_local(ctdb_db, &state->call, &state->header, &data, ctdb->vnn);
-
-	ctdb_ltdb_unlock(ctdb_db, state->call.key);
-
-	talloc_steal(state, state->call.reply_data.dptr);
-
-	state->state = CTDB_CALL_DONE;
-	if (state->async.fn) {
-		state->async.fn(state);
-	}
+	ctdb_become_dmaster(ctdb_db, hdr->reqid, key, data, c->rsn);
 }
 
 
@@ -502,8 +517,18 @@ void ctdb_reply_error(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	struct ctdb_reply_error *c = (struct ctdb_reply_error *)hdr;
 	struct ctdb_call_state *state;
 
-	state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_call_state);
-	if (state == NULL) return;
+	state = ctdb_reqid_find(ctdb, hdr->reqid, struct ctdb_call_state);
+	if (state == NULL) {
+		DEBUG(0,("vnn %u Invalid reqid %u in ctdb_reply_error\n",
+			 ctdb->vnn, hdr->reqid));
+		return;
+	}
+
+	if (hdr->reqid != state->reqid) {
+		/* we found a record  but it was the wrong one */
+		DEBUG(0, ("Dropped orphaned error reply with reqid:%u\n",hdr->reqid));
+		return;
+	}
 
 	talloc_steal(state, c);
 
@@ -516,56 +541,49 @@ void ctdb_reply_error(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 
 
 /*
-  called when a CTDB_REPLY_REDIRECT packet comes in
+  destroy a ctdb_call
+*/
+static int ctdb_call_destructor(struct ctdb_call_state *state)
+{
+	DLIST_REMOVE(state->ctdb_db->ctdb->pending_calls, state);
+	ctdb_reqid_remove(state->ctdb_db->ctdb, state->reqid);
+	return 0;
+}
+
 
-  This packet arrives when we have sent a CTDB_REQ_CALL request and
-  the node that received it is not the dmaster for the given key. We
-  are given a hint as to what node to try next.
+/*
+  called when a ctdb_call needs to be resent after a reconfigure event
 */
-void ctdb_reply_redirect(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+static void ctdb_call_resend(struct ctdb_call_state *state)
 {
-	struct ctdb_reply_redirect *c = (struct ctdb_reply_redirect *)hdr;
-	struct ctdb_call_state *state;
+	struct ctdb_context *ctdb = state->ctdb_db->ctdb;
 
-	state = idr_find_type(ctdb->idr, hdr->reqid, struct ctdb_call_state);
-	if (state == NULL) return;
+	state->generation = ctdb->vnn_map->generation;
 
-	talloc_steal(state, c);
-	
-	/* don't allow for too many redirects */
-	if (state->redirect_count++ == CTDB_MAX_REDIRECT) {
-		c->dmaster = ctdb_lmaster(ctdb, &state->call.key);
-	}
+	/* use a new reqid, in case the old reply does eventually come in */
+	ctdb_reqid_remove(ctdb, state->reqid);
+	state->reqid = ctdb_reqid_new(ctdb, state);
+	state->c->hdr.reqid = state->reqid;
 
-	/* send it off again */
-	state->node = ctdb->nodes[c->dmaster];
-	state->c->hdr.destnode = c->dmaster;
+	/* update the generation count for this request, so its valid with the new vnn_map */
+	state->c->hdr.generation = state->generation;
 
-	ctdb_queue_packet(ctdb, &state->c->hdr);
-}
+	/* send the packet to ourselves, it will be redirected appropriately */
+	state->c->hdr.destnode = ctdb->vnn;
 
-/*
-  destroy a ctdb_call
-*/
-static int ctdb_call_destructor(struct ctdb_call_state *state)
-{
-	idr_remove(state->node->ctdb->idr, state->c->hdr.reqid);
-	return 0;
+	ctdb_queue_packet(ctdb, &state->c->hdr);
+	DEBUG(0,("resent ctdb_call\n"));
 }
 
-
 /*
-  called when a ctdb_call times out
-*/
-void ctdb_call_timeout(struct event_context *ev, struct timed_event *te, 
-		       struct timeval t, void *private_data)
+  resend all pending calls on recovery
+ */
+void ctdb_call_resend_all(struct ctdb_context *ctdb)
 {
-	struct ctdb_call_state *state = talloc_get_type(private_data, struct ctdb_call_state);
-	state->state = CTDB_CALL_ERROR;
-	ctdb_set_error(state->node->ctdb, "ctdb_call %u timed out",
-		       state->c->hdr.reqid);
-	if (state->async.fn) {
-		state->async.fn(state);
+	struct ctdb_call_state *state, *next;
+	for (state=ctdb->pending_calls;state;state=next) {
+		next = state->next;
+		ctdb_call_resend(state);
 	}
 }
 
@@ -603,12 +621,10 @@ struct ctdb_call_state *ctdb_call_local_send(struct ctdb_db_context *ctdb_db,
 	talloc_steal(state, data->dptr);
 
 	state->state = CTDB_CALL_DONE;
-	state->node = ctdb->nodes[ctdb->vnn];
 	state->call = *call;
 	state->ctdb_db = ctdb_db;
 
-	ret = ctdb_call_local(ctdb_db, &state->call, header, data, ctdb->vnn);
-	talloc_steal(state, state->call.reply_data.dptr);
+	ret = ctdb_call_local(ctdb_db, &state->call, header, state, data, ctdb->vnn);
 
 	event_add_timed(ctdb->ev, state, timeval_zero(), call_local_trigger, state);
 
@@ -632,23 +648,22 @@ struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctd
 
 	state = talloc_zero(ctdb_db, struct ctdb_call_state);
 	CTDB_NO_MEMORY_NULL(ctdb, state);
+	state->reqid = ctdb_reqid_new(ctdb, state);
+	state->ctdb_db = ctdb_db;
+	talloc_set_destructor(state, ctdb_call_destructor);
 
 	len = offsetof(struct ctdb_req_call, data) + call->key.dsize + call->call_data.dsize;
-	state->c = ctdb->methods->allocate_pkt(state, len);
+	state->c = ctdb_transport_allocate(ctdb, state, CTDB_REQ_CALL, len, 
+					   struct ctdb_req_call);
 	CTDB_NO_MEMORY_NULL(ctdb, state->c);
-	talloc_set_name_const(state->c, "req_call packet");
-
-	state->c->hdr.length    = len;
-	state->c->hdr.ctdb_magic = CTDB_MAGIC;
-	state->c->hdr.ctdb_version = CTDB_VERSION;
-	state->c->hdr.operation = CTDB_REQ_CALL;
 	state->c->hdr.destnode  = header->dmaster;
-	state->c->hdr.srcnode   = ctdb->vnn;
+
 	/* this limits us to 16k outstanding messages - not unreasonable */
-	state->c->hdr.reqid     = idr_get_new(ctdb->idr, state, 0xFFFF);
+	state->c->hdr.reqid     = state->reqid;
 	state->c->flags         = call->flags;
 	state->c->db_id         = ctdb_db->db_id;
 	state->c->callid        = call->call_id;
+	state->c->hopcount      = 0;
 	state->c->keylen        = call->key.dsize;
 	state->c->calldatalen   = call->call_data.dsize;
 	memcpy(&state->c->data[0], call->key.dptr, call->key.dsize);
@@ -658,17 +673,13 @@ struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctd
 	state->call.call_data.dptr = &state->c->data[call->key.dsize];
 	state->call.key.dptr       = &state->c->data[0];
 
-	state->node   = ctdb->nodes[header->dmaster];
 	state->state  = CTDB_CALL_WAIT;
-	state->header = *header;
-	state->ctdb_db = ctdb_db;
+	state->generation = ctdb->vnn_map->generation;
 
-	talloc_set_destructor(state, ctdb_call_destructor);
+	DLIST_ADD(ctdb->pending_calls, state);
 
 	ctdb_queue_packet(ctdb, &state->c->hdr);
 
-	event_add_timed(ctdb->ev, state, timeval_current_ofs(CTDB_REQ_TIMEOUT, 0), 
-			ctdb_call_timeout, state);
 	return state;
 }
 
@@ -681,16 +692,16 @@ struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctd
 int ctdb_daemon_call_recv(struct ctdb_call_state *state, struct ctdb_call *call)
 {
 	while (state->state < CTDB_CALL_DONE) {
-		event_loop_once(state->node->ctdb->ev);
+		event_loop_once(state->ctdb_db->ctdb->ev);
 	}
 	if (state->state != CTDB_CALL_DONE) {
-		ctdb_set_error(state->node->ctdb, "%s", state->errmsg);
+		ctdb_set_error(state->ctdb_db->ctdb, "%s", state->errmsg);
 		talloc_free(state);
 		return -1;
 	}
 
 	if (state->call.reply_data.dsize) {
-		call->reply_data.dptr = talloc_memdup(state->node->ctdb,
+		call->reply_data.dptr = talloc_memdup(state->ctdb_db->ctdb,
 						      state->call.reply_data.dptr,
 						      state->call.reply_data.dsize);
 		call->reply_data.dsize = state->call.reply_data.dsize;
@@ -704,3 +715,23 @@ int ctdb_daemon_call_recv(struct ctdb_call_state *state, struct ctdb_call *call)
 }
 
 
+/* 
+   send a keepalive packet to the other node
+*/
+void ctdb_send_keepalive(struct ctdb_context *ctdb, uint32_t destnode)
+{
+	struct ctdb_req_keepalive *r;
+	
+	r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REQ_KEEPALIVE,
+				    sizeof(struct ctdb_req_keepalive), 
+				    struct ctdb_req_keepalive);
+	CTDB_NO_MEMORY_FATAL(ctdb, r);
+	r->hdr.destnode  = destnode;
+	r->hdr.reqid     = 0;
+	
+	ctdb->statistics.keepalive_packets_sent++;
+
+	ctdb_queue_packet(ctdb, &r->hdr);
+
+	talloc_free(r);
+}
diff --git a/source4/cluster/ctdb/server/ctdb_control.c b/source4/cluster/ctdb/server/ctdb_control.c
new file mode 100644
index 0000000000..69848bb15c
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_control.c
@@ -0,0 +1,499 @@
+/* 
+   ctdb_control protocol code
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+#include "lib/util/dlinklist.h"
+#include "db_wrap.h"
+
+struct ctdb_control_state {
+	struct ctdb_context *ctdb;
+	uint32_t reqid;
+	ctdb_control_callback_fn_t callback;
+	void *private_data;
+	unsigned flags;
+};
+
+/*
+  process a control request
+ */
+static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb, 
+				     struct ctdb_req_control *c,
+				     TDB_DATA indata,
+				     TDB_DATA *outdata, uint32_t srcnode,
+				     const char **errormsg,
+				     bool *async_reply)
+{
+	uint32_t opcode = c->opcode;
+	uint64_t srvid = c->srvid;
+	uint32_t client_id = c->client_id;
+
+	switch (opcode) {
+	case CTDB_CONTROL_PROCESS_EXISTS: {
+		CHECK_CONTROL_DATA_SIZE(sizeof(pid_t));
+		return kill(*(pid_t *)indata.dptr, 0);
+	}
+
+	case CTDB_CONTROL_SET_DEBUG: {
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
+		LogLevel = *(uint32_t *)indata.dptr;
+		return 0;
+	}
+
+	case CTDB_CONTROL_GET_DEBUG: {
+		CHECK_CONTROL_DATA_SIZE(0);
+		outdata->dptr = (uint8_t *)&LogLevel;
+		outdata->dsize = sizeof(LogLevel);
+		return 0;
+	}
+
+	case CTDB_CONTROL_STATISTICS: {
+		CHECK_CONTROL_DATA_SIZE(0);
+		ctdb->statistics.memory_used = talloc_total_size(ctdb);
+		ctdb->statistics.frozen = (ctdb->freeze_mode == CTDB_FREEZE_FROZEN);
+		ctdb->statistics.recovering = (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE);
+		outdata->dptr = (uint8_t *)&ctdb->statistics;
+		outdata->dsize = sizeof(ctdb->statistics);
+		return 0;
+	}
+
+	case CTDB_CONTROL_GET_ALL_TUNABLES: {
+		CHECK_CONTROL_DATA_SIZE(0);
+		outdata->dptr = (uint8_t *)&ctdb->tunable;
+		outdata->dsize = sizeof(ctdb->tunable);
+		return 0;
+	}
+
+	case CTDB_CONTROL_DUMP_MEMORY: {
+		CHECK_CONTROL_DATA_SIZE(0);
+		talloc_report_full(ctdb, stdout);
+		return 0;
+	}
+
+	case CTDB_CONTROL_STATISTICS_RESET: {
+		CHECK_CONTROL_DATA_SIZE(0);
+		ZERO_STRUCT(ctdb->statistics);
+		return 0;
+	}
+
+	case CTDB_CONTROL_GETVNNMAP:
+		return ctdb_control_getvnnmap(ctdb, opcode, indata, outdata);
+
+	case CTDB_CONTROL_GET_DBMAP:
+		return ctdb_control_getdbmap(ctdb, opcode, indata, outdata);
+
+	case CTDB_CONTROL_GET_NODEMAP:
+		return ctdb_control_getnodemap(ctdb, opcode, indata, outdata);
+
+	case CTDB_CONTROL_SETVNNMAP:
+		return ctdb_control_setvnnmap(ctdb, opcode, indata, outdata);
+
+	case CTDB_CONTROL_PULL_DB: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_pulldb));
+		return ctdb_control_pull_db(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_SET_DMASTER: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_set_dmaster));
+		return ctdb_control_set_dmaster(ctdb, indata);
+
+	case CTDB_CONTROL_PUSH_DB:
+		return ctdb_control_push_db(ctdb, indata);
+
+	case CTDB_CONTROL_GET_RECMODE: {
+		return ctdb->recovery_mode;
+	}
+
+	case CTDB_CONTROL_SET_RECMASTER: {
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
+		if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+			DEBUG(0,("Attempt to set recmaster when not frozen\n"));
+			return -1;
+		}
+		ctdb->recovery_master = ((uint32_t *)(&indata.dptr[0]))[0];
+		return 0;
+	}
+
+	case CTDB_CONTROL_GET_RECMASTER:
+		return ctdb->recovery_master;
+
+	case CTDB_CONTROL_GET_PID:
+		return getpid();
+
+	case CTDB_CONTROL_GET_VNN:
+		return ctdb->vnn;
+
+	case CTDB_CONTROL_PING:
+		CHECK_CONTROL_DATA_SIZE(0);
+		return ctdb->statistics.num_clients;
+
+	case CTDB_CONTROL_GET_DBNAME: {
+		uint32_t db_id;
+		struct ctdb_db_context *ctdb_db;
+
+		CHECK_CONTROL_DATA_SIZE(sizeof(db_id));
+		db_id = *(uint32_t *)indata.dptr;
+		ctdb_db = find_ctdb_db(ctdb, db_id);
+		if (ctdb_db == NULL) return -1;
+		outdata->dptr = discard_const(ctdb_db->db_name);
+		outdata->dsize = strlen(ctdb_db->db_name)+1;
+		return 0;
+	}
+
+	case CTDB_CONTROL_GETDBPATH: {
+		uint32_t db_id;
+		struct ctdb_db_context *ctdb_db;
+
+		CHECK_CONTROL_DATA_SIZE(sizeof(db_id));
+		db_id = *(uint32_t *)indata.dptr;
+		ctdb_db = find_ctdb_db(ctdb, db_id);
+		if (ctdb_db == NULL) return -1;
+		outdata->dptr = discard_const(ctdb_db->db_path);
+		outdata->dsize = strlen(ctdb_db->db_path)+1;
+		return 0;
+	}
+
+	case CTDB_CONTROL_DB_ATTACH:
+		return ctdb_control_db_attach(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_SET_CALL: {
+		struct ctdb_control_set_call *sc = 
+			(struct ctdb_control_set_call *)indata.dptr;
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_set_call));
+		return ctdb_daemon_set_call(ctdb, sc->db_id, sc->fn, sc->id);
+	}
+
+	case CTDB_CONTROL_TRAVERSE_START:
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_traverse_start));
+		return ctdb_control_traverse_start(ctdb, indata, outdata, srcnode);
+
+	case CTDB_CONTROL_TRAVERSE_ALL:
+		return ctdb_control_traverse_all(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_TRAVERSE_DATA:
+		return ctdb_control_traverse_data(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_REGISTER_SRVID:
+		return daemon_register_message_handler(ctdb, client_id, srvid);
+
+	case CTDB_CONTROL_DEREGISTER_SRVID:
+		return daemon_deregister_message_handler(ctdb, client_id, srvid);
+
+	case CTDB_CONTROL_ENABLE_SEQNUM:
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
+		return ctdb_ltdb_enable_seqnum(ctdb, *(uint32_t *)indata.dptr);
+
+	case CTDB_CONTROL_UPDATE_SEQNUM:
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));		
+		return ctdb_ltdb_update_seqnum(ctdb, *(uint32_t *)indata.dptr, srcnode);
+
+	case CTDB_CONTROL_FREEZE:
+		CHECK_CONTROL_DATA_SIZE(0);
+		return ctdb_control_freeze(ctdb, c, async_reply);
+
+	case CTDB_CONTROL_THAW:
+		CHECK_CONTROL_DATA_SIZE(0);
+		return ctdb_control_thaw(ctdb);
+
+	case CTDB_CONTROL_SET_RECMODE:
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));		
+		return ctdb_control_set_recmode(ctdb, c, indata, async_reply, errormsg);
+
+	case CTDB_CONTROL_SET_MONMODE:
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));		
+		ctdb->monitoring_mode = *(uint32_t *)indata.dptr;
+		return 0;
+
+	case CTDB_CONTROL_GET_MONMODE: 
+		return ctdb->monitoring_mode;
+
+	case CTDB_CONTROL_SHUTDOWN:
+		ctdb_release_all_ips(ctdb);
+		ctdb->methods->shutdown(ctdb);
+		ctdb_event_script(ctdb, "shutdown");
+		DEBUG(0,("shutting down\n"));
+		exit(0);
+
+	case CTDB_CONTROL_MAX_RSN: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
+		return ctdb_control_max_rsn(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_SET_RSN_NONEMPTY: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_set_rsn_nonempty));
+		return ctdb_control_set_rsn_nonempty(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_TAKEOVER_IP:
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_public_ip));
+		return ctdb_control_takeover_ip(ctdb, c, indata, async_reply);
+
+	case CTDB_CONTROL_RELEASE_IP:
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_public_ip));
+		return ctdb_control_release_ip(ctdb, c, indata, async_reply);
+
+	case CTDB_CONTROL_GET_PUBLIC_IPS:
+		CHECK_CONTROL_DATA_SIZE(0);
+		return ctdb_control_get_public_ips(ctdb, c, outdata);
+
+	case CTDB_CONTROL_DELETE_LOW_RSN: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_delete_low_rsn));
+		return ctdb_control_delete_low_rsn(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_TCP_CLIENT: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+		return ctdb_control_tcp_client(ctdb, client_id, srcnode, indata);
+
+	case CTDB_CONTROL_STARTUP: 
+		CHECK_CONTROL_DATA_SIZE(0);
+		return ctdb_control_startup(ctdb, srcnode);
+
+	case CTDB_CONTROL_TCP_ADD: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp_vnn));
+		return ctdb_control_tcp_add(ctdb, indata);
+
+	case CTDB_CONTROL_TCP_REMOVE: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp_vnn));
+		return ctdb_control_tcp_remove(ctdb, indata);
+
+	case CTDB_CONTROL_SET_TUNABLE:
+		return ctdb_control_set_tunable(ctdb, indata);
+
+	case CTDB_CONTROL_GET_TUNABLE:
+		return ctdb_control_get_tunable(ctdb, indata, outdata);
+
+	case CTDB_CONTROL_LIST_TUNABLES:
+		return ctdb_control_list_tunables(ctdb, outdata);
+
+	case CTDB_CONTROL_MODIFY_FLAGS:
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_node_modflags));
+		return ctdb_control_modflags(ctdb, indata);
+
+	default:
+		DEBUG(0,(__location__ " Unknown CTDB control opcode %u\n", opcode));
+		return -1;
+	}
+}
+
+
+/*
+  send a reply for a ctdb control
+ */
+void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_control *c,
+				TDB_DATA *outdata, int32_t status, const char *errormsg)
+{
+	struct ctdb_reply_control *r;
+	size_t len;
+	
+	/* some controls send no reply */
+	if (c->flags & CTDB_CTRL_FLAG_NOREPLY) {
+		return;
+	}
+
+	len = offsetof(struct ctdb_reply_control, data) + (outdata?outdata->dsize:0);
+	if (errormsg) {
+		len += strlen(errormsg);
+	}
+	r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REPLY_CONTROL, len, struct ctdb_reply_control);
+	CTDB_NO_MEMORY_VOID(ctdb, r);
+
+	r->hdr.destnode     = c->hdr.srcnode;
+	r->hdr.reqid        = c->hdr.reqid;
+	r->status           = status;
+	r->datalen          = outdata?outdata->dsize:0;
+	if (outdata && outdata->dsize) {
+		memcpy(&r->data[0], outdata->dptr, outdata->dsize);
+	}
+	if (errormsg) {
+		r->errorlen = strlen(errormsg);
+		memcpy(&r->data[r->datalen], errormsg, r->errorlen);
+	}
+	
+	ctdb_queue_packet(ctdb, &r->hdr);	
+
+	talloc_free(r);
+}
+
+/*
+  called when a CTDB_REQ_CONTROL packet comes in
+*/
+void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	struct ctdb_req_control *c = (struct ctdb_req_control *)hdr;
+	TDB_DATA data, *outdata;
+	int32_t status;
+	bool async_reply = False;
+	const char *errormsg = NULL;
+
+	data.dptr = &c->data[0];
+	data.dsize = c->datalen;
+
+	outdata = talloc_zero(c, TDB_DATA);
+
+	status = ctdb_control_dispatch(ctdb, c, data, outdata, hdr->srcnode, 
+				       &errormsg, &async_reply);
+
+	if (!async_reply) {
+		ctdb_request_control_reply(ctdb, c, outdata, status, errormsg);
+	}
+}
+
+/*
+  called when a CTDB_REPLY_CONTROL packet comes in
+*/
+void ctdb_reply_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	struct ctdb_reply_control *c = (struct ctdb_reply_control *)hdr;
+	TDB_DATA data;
+	struct ctdb_control_state *state;
+	const char *errormsg = NULL;
+
+	state = ctdb_reqid_find(ctdb, hdr->reqid, struct ctdb_control_state);
+	if (state == NULL) {
+		DEBUG(0,("vnn %u Invalid reqid %u in ctdb_reply_control\n",
+			 ctdb->vnn, hdr->reqid));
+		return;
+	}
+
+	if (hdr->reqid != state->reqid) {
+		/* we found a record  but it was the wrong one */
+		DEBUG(0, ("Dropped orphaned control reply with reqid:%u\n", hdr->reqid));
+		return;
+	}
+
+	data.dptr = &c->data[0];
+	data.dsize = c->datalen;
+	if (c->errorlen) {
+		errormsg = talloc_strndup(state, 
+					  (char *)&c->data[c->datalen], c->errorlen);
+	}
+
+	/* make state a child of the packet, so it goes away when the packet
+	   is freed. */
+	talloc_steal(hdr, state);
+
+	state->callback(ctdb, c->status, data, errormsg, state->private_data);
+}
+
+static int ctdb_control_destructor(struct ctdb_control_state *state)
+{
+	ctdb_reqid_remove(state->ctdb, state->reqid);
+	return 0;
+}
+
+/*
+  handle a timeout of a control
+ */
+static void ctdb_control_timeout(struct event_context *ev, struct timed_event *te, 
+		       struct timeval t, void *private_data)
+{
+	struct ctdb_control_state *state = talloc_get_type(private_data, struct ctdb_control_state);
+	TALLOC_CTX *tmp_ctx = talloc_new(ev);
+
+	state->ctdb->statistics.timeouts.control++;
+
+	talloc_steal(tmp_ctx, state);
+
+	state->callback(state->ctdb, -1, tdb_null,
+			"ctdb_control timed out", 
+			state->private_data);
+	talloc_free(tmp_ctx);
+}
+
+
+/*
+  send a control message to a node
+ */
+int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
+			     uint64_t srvid, uint32_t opcode, uint32_t client_id,
+			     uint32_t flags,
+			     TDB_DATA data,
+			     ctdb_control_callback_fn_t callback,
+			     void *private_data)
+{
+	struct ctdb_req_control *c;
+	struct ctdb_control_state *state;
+	size_t len;
+
+	if (((destnode == CTDB_BROADCAST_VNNMAP) || 
+	     (destnode == CTDB_BROADCAST_ALL) ||
+	     (destnode == CTDB_BROADCAST_CONNECTED)) && 
+	    !(flags & CTDB_CTRL_FLAG_NOREPLY)) {
+		DEBUG(0,("Attempt to broadcast control without NOREPLY\n"));
+		return -1;
+	}
+
+	if (destnode != CTDB_BROADCAST_VNNMAP && 
+	    destnode != CTDB_BROADCAST_ALL && 
+	    destnode != CTDB_BROADCAST_CONNECTED && 
+	    (!ctdb_validate_vnn(ctdb, destnode) || 
+	     (ctdb->nodes[destnode]->flags & NODE_FLAGS_DISCONNECTED))) {
+		if (!(flags & CTDB_CTRL_FLAG_NOREPLY)) {
+			callback(ctdb, -1, tdb_null, "ctdb_control to disconnected node", private_data);
+		}
+		return 0;
+	}
+
+	/* the state is made a child of private_data if possible. This means any reply
+	   will be discarded if the private_data goes away */
+	state = talloc(private_data?private_data:ctdb, struct ctdb_control_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	state->reqid = ctdb_reqid_new(ctdb, state);
+	state->callback = callback;
+	state->private_data = private_data;
+	state->ctdb = ctdb;
+	state->flags = flags;
+
+	talloc_set_destructor(state, ctdb_control_destructor);
+
+	len = offsetof(struct ctdb_req_control, data) + data.dsize;
+	c = ctdb_transport_allocate(ctdb, state, CTDB_REQ_CONTROL, len, 
+				    struct ctdb_req_control);
+	CTDB_NO_MEMORY(ctdb, c);
+	talloc_set_name_const(c, "ctdb_req_control packet");
+
+	c->hdr.destnode     = destnode;
+	c->hdr.reqid        = state->reqid;
+	c->opcode           = opcode;
+	c->client_id        = client_id;
+	c->flags            = flags;
+	c->srvid            = srvid;
+	c->datalen          = data.dsize;
+	if (data.dsize) {
+		memcpy(&c->data[0], data.dptr, data.dsize);
+	}
+
+	ctdb_queue_packet(ctdb, &c->hdr);	
+
+	if (flags & CTDB_CTRL_FLAG_NOREPLY) {
+		talloc_free(state);
+		return 0;
+	}
+
+	if (ctdb->tunable.control_timeout) {
+		event_add_timed(ctdb->ev, state, 
+				timeval_current_ofs(ctdb->tunable.control_timeout, 0), 
+				ctdb_control_timeout, state);
+	}
+
+	talloc_free(c);
+	return 0;
+}
diff --git a/source4/cluster/ctdb/server/ctdb_daemon.c b/source4/cluster/ctdb/server/ctdb_daemon.c
new file mode 100644
index 0000000000..2577970075
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_daemon.c
@@ -0,0 +1,927 @@
+/* 
+   ctdb daemon code
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "db_wrap.h"
+#include "lib/tdb/include/tdb.h"
+#include "lib/events/events.h"
+#include "lib/util/dlinklist.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb.h"
+#include "../include/ctdb_private.h"
+
+static void daemon_incoming_packet(void *, struct ctdb_req_header *);
+
+/*
+  handler for when a node changes its flags
+*/
+static void flag_change_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+				TDB_DATA data, void *private_data)
+{
+	struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)data.dptr;
+
+	if (data.dsize != sizeof(*c) || !ctdb_validate_vnn(ctdb, c->vnn)) {
+		DEBUG(0,(__location__ "Invalid data in ctdb_node_flag_change\n"));
+		return;
+	}
+
+	if (!ctdb_validate_vnn(ctdb, c->vnn)) {
+		DEBUG(0,("Bad vnn %u in flag_change_handler\n", c->vnn));
+		return;
+	}
+
+	/* don't get the disconnected flag from the other node */
+	ctdb->nodes[c->vnn]->flags = 
+		(ctdb->nodes[c->vnn]->flags&NODE_FLAGS_DISCONNECTED) 
+		| (c->flags & ~NODE_FLAGS_DISCONNECTED);	
+	DEBUG(2,("Node flags for node %u are now 0x%x\n", c->vnn, ctdb->nodes[c->vnn]->flags));
+
+	/* make sure we don't hold any IPs when we shouldn't */
+	if (c->vnn == ctdb->vnn &&
+	    (ctdb->nodes[c->vnn]->flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_BANNED))) {
+		ctdb_release_all_ips(ctdb);
+	}
+}
+
+/* called when the "startup" event script has finished */
+static void ctdb_start_transport(struct ctdb_context *ctdb, int status, void *p)
+{
+	if (status != 0) {
+		DEBUG(0,("startup event failed!\n"));
+		ctdb_fatal(ctdb, "startup event script failed");		
+	}
+
+	/* start the transport running */
+	if (ctdb->methods->start(ctdb) != 0) {
+		DEBUG(0,("transport failed to start!\n"));
+		ctdb_fatal(ctdb, "transport failed to start");
+	}
+
+	/* start the recovery daemon process */
+	if (ctdb_start_recoverd(ctdb) != 0) {
+		DEBUG(0,("Failed to start recovery daemon\n"));
+		exit(11);
+	}
+
+	/* a handler for when nodes are disabled/enabled */
+	ctdb_register_message_handler(ctdb, ctdb, CTDB_SRVID_NODE_FLAGS_CHANGED, 
+				      flag_change_handler, NULL);
+
+	/* start monitoring for dead nodes */
+	ctdb_start_monitoring(ctdb);
+}
+
+/* go into main ctdb loop */
+static void ctdb_main_loop(struct ctdb_context *ctdb)
+{
+	int ret = -1;
+
+	if (strcmp(ctdb->transport, "tcp") == 0) {
+		int ctdb_tcp_init(struct ctdb_context *);
+		ret = ctdb_tcp_init(ctdb);
+	}
+#ifdef USE_INFINIBAND
+	if (strcmp(ctdb->transport, "ib") == 0) {
+		int ctdb_ibw_init(struct ctdb_context *);
+		ret = ctdb_ibw_init(ctdb);
+	}
+#endif
+	if (ret != 0) {
+		DEBUG(0,("Failed to initialise transport '%s'\n", ctdb->transport));
+		return;
+	}
+
+	/* initialise the transport  */
+	if (ctdb->methods->initialise(ctdb) != 0) {
+		DEBUG(0,("transport failed to initialise!\n"));
+		ctdb_fatal(ctdb, "transport failed to initialise");
+	}
+
+	/* tell all other nodes we've just started up */
+	ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL,
+				 0, CTDB_CONTROL_STARTUP, 0,
+				 CTDB_CTRL_FLAG_NOREPLY,
+				 tdb_null, NULL, NULL);
+
+	/* release any IPs we hold from previous runs of the daemon */
+	ctdb_release_all_ips(ctdb);
+
+	ret = ctdb_event_script_callback(ctdb, timeval_zero(), ctdb, 
+					 ctdb_start_transport, NULL, "startup");
+	if (ret != 0) {
+		DEBUG(0,("Failed startup event script\n"));
+		return;
+	}
+
+	/* go into a wait loop to allow other nodes to complete */
+	event_loop_wait(ctdb->ev);
+
+	DEBUG(0,("event_loop_wait() returned. this should not happen\n"));
+	exit(1);
+}
+
+
+static void block_signal(int signum)
+{
+	struct sigaction act;
+
+	memset(&act, 0, sizeof(act));
+
+	act.sa_handler = SIG_IGN;
+	sigemptyset(&act.sa_mask);
+	sigaddset(&act.sa_mask, signum);
+	sigaction(signum, &act, NULL);
+}
+
+
+/*
+  send a packet to a client
+ */
+static int daemon_queue_send(struct ctdb_client *client, struct ctdb_req_header *hdr)
+{
+	client->ctdb->statistics.client_packets_sent++;
+	return ctdb_queue_send(client->queue, (uint8_t *)hdr, hdr->length);
+}
+
+/*
+  message handler for when we are in daemon mode. This redirects the message
+  to the right client
+ */
+static void daemon_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+				    TDB_DATA data, void *private_data)
+{
+	struct ctdb_client *client = talloc_get_type(private_data, struct ctdb_client);
+	struct ctdb_req_message *r;
+	int len;
+
+	/* construct a message to send to the client containing the data */
+	len = offsetof(struct ctdb_req_message, data) + data.dsize;
+	r = ctdbd_allocate_pkt(ctdb, ctdb, CTDB_REQ_MESSAGE, 
+			       len, struct ctdb_req_message);
+	CTDB_NO_MEMORY_VOID(ctdb, r);
+
+	talloc_set_name_const(r, "req_message packet");
+
+	r->srvid         = srvid;
+	r->datalen       = data.dsize;
+	memcpy(&r->data[0], data.dptr, data.dsize);
+
+	daemon_queue_send(client, &r->hdr);
+
+	talloc_free(r);
+}
+					   
+
+/*
+  this is called when the ctdb daemon received a ctdb request to 
+  set the srvid from the client
+ */
+int daemon_register_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid)
+{
+	struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+	int res;
+	if (client == NULL) {
+		DEBUG(0,("Bad client_id in daemon_request_register_message_handler\n"));
+		return -1;
+	}
+	res = ctdb_register_message_handler(ctdb, client, srvid, daemon_message_handler, client);
+	if (res != 0) {
+		DEBUG(0,(__location__ " Failed to register handler %llu in daemon\n", 
+			 (unsigned long long)srvid));
+	} else {
+		DEBUG(2,(__location__ " Registered message handler for srvid=%llu\n", 
+			 (unsigned long long)srvid));
+	}
+
+	/* this is a hack for Samba - we now know the pid of the Samba client */
+	if ((srvid & 0xFFFFFFFF) == srvid &&
+	    kill(srvid, 0) == 0) {
+		client->pid = srvid;
+		DEBUG(0,(__location__ " Registered PID %u for client %u\n",
+			 (unsigned)client->pid, client_id));
+	}
+	return res;
+}
+
+/*
+  this is called when the ctdb daemon received a ctdb request to 
+  remove a srvid from the client
+ */
+int daemon_deregister_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid)
+{
+	struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+	if (client == NULL) {
+		DEBUG(0,("Bad client_id in daemon_request_deregister_message_handler\n"));
+		return -1;
+	}
+	return ctdb_deregister_message_handler(ctdb, srvid, client);
+}
+
+
+/*
+  destroy a ctdb_client
+*/
+static int ctdb_client_destructor(struct ctdb_client *client)
+{
+	ctdb_takeover_client_destructor_hook(client);
+	ctdb_reqid_remove(client->ctdb, client->client_id);
+	client->ctdb->statistics.num_clients--;
+	return 0;
+}
+
+
+/*
+  this is called when the ctdb daemon received a ctdb request message
+  from a local client over the unix domain socket
+ */
+static void daemon_request_message_from_client(struct ctdb_client *client, 
+					       struct ctdb_req_message *c)
+{
+	TDB_DATA data;
+	int res;
+
+	/* maybe the message is for another client on this node */
+	if (ctdb_get_vnn(client->ctdb)==c->hdr.destnode) {
+		ctdb_request_message(client->ctdb, (struct ctdb_req_header *)c);
+		return;
+	}
+
+	/* its for a remote node */
+	data.dptr = &c->data[0];
+	data.dsize = c->datalen;
+	res = ctdb_daemon_send_message(client->ctdb, c->hdr.destnode,
+				       c->srvid, data);
+	if (res != 0) {
+		DEBUG(0,(__location__ " Failed to send message to remote node %u\n",
+			 c->hdr.destnode));
+	}
+}
+
+
+struct daemon_call_state {
+	struct ctdb_client *client;
+	uint32_t reqid;
+	struct ctdb_call *call;
+	struct timeval start_time;
+};
+
+/* 
+   complete a call from a client 
+*/
+static void daemon_call_from_client_callback(struct ctdb_call_state *state)
+{
+	struct daemon_call_state *dstate = talloc_get_type(state->async.private_data, 
+							   struct daemon_call_state);
+	struct ctdb_reply_call *r;
+	int res;
+	uint32_t length;
+	struct ctdb_client *client = dstate->client;
+
+	talloc_steal(client, dstate);
+	talloc_steal(dstate, dstate->call);
+
+	res = ctdb_daemon_call_recv(state, dstate->call);
+	if (res != 0) {
+		DEBUG(0, (__location__ " ctdbd_call_recv() returned error\n"));
+		client->ctdb->statistics.pending_calls--;
+		ctdb_latency(&client->ctdb->statistics.max_call_latency, dstate->start_time);
+		return;
+	}
+
+	length = offsetof(struct ctdb_reply_call, data) + dstate->call->reply_data.dsize;
+	r = ctdbd_allocate_pkt(client->ctdb, dstate, CTDB_REPLY_CALL, 
+			       length, struct ctdb_reply_call);
+	if (r == NULL) {
+		DEBUG(0, (__location__ " Failed to allocate reply_call in ctdb daemon\n"));
+		client->ctdb->statistics.pending_calls--;
+		ctdb_latency(&client->ctdb->statistics.max_call_latency, dstate->start_time);
+		return;
+	}
+	r->hdr.reqid        = dstate->reqid;
+	r->datalen          = dstate->call->reply_data.dsize;
+	memcpy(&r->data[0], dstate->call->reply_data.dptr, r->datalen);
+
+	res = daemon_queue_send(client, &r->hdr);
+	if (res != 0) {
+		DEBUG(0, (__location__ " Failed to queue packet from daemon to client\n"));
+	}
+	ctdb_latency(&client->ctdb->statistics.max_call_latency, dstate->start_time);
+	talloc_free(dstate);
+	client->ctdb->statistics.pending_calls--;
+}
+
+
+static void daemon_request_call_from_client(struct ctdb_client *client, 
+					    struct ctdb_req_call *c);
+
+/*
+  this is called when the ctdb daemon received a ctdb request call
+  from a local client over the unix domain socket
+ */
+static void daemon_request_call_from_client(struct ctdb_client *client, 
+					    struct ctdb_req_call *c)
+{
+	struct ctdb_call_state *state;
+	struct ctdb_db_context *ctdb_db;
+	struct daemon_call_state *dstate;
+	struct ctdb_call *call;
+	struct ctdb_ltdb_header header;
+	TDB_DATA key, data;
+	int ret;
+	struct ctdb_context *ctdb = client->ctdb;
+
+	ctdb->statistics.total_calls++;
+	ctdb->statistics.pending_calls++;
+
+	ctdb_db = find_ctdb_db(client->ctdb, c->db_id);
+	if (!ctdb_db) {
+		DEBUG(0, (__location__ " Unknown database in request. db_id==0x%08x",
+			  c->db_id));
+		ctdb->statistics.pending_calls--;
+		return;
+	}
+
+	key.dptr = c->data;
+	key.dsize = c->keylen;
+
+	ret = ctdb_ltdb_lock_fetch_requeue(ctdb_db, key, &header, 
+					   (struct ctdb_req_header *)c, &data,
+					   daemon_incoming_packet, client, True);
+	if (ret == -2) {
+		/* will retry later */
+		ctdb->statistics.pending_calls--;
+		return;
+	}
+
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Unable to fetch record\n"));
+		ctdb->statistics.pending_calls--;
+		return;
+	}
+
+	dstate = talloc(client, struct daemon_call_state);
+	if (dstate == NULL) {
+		ctdb_ltdb_unlock(ctdb_db, key);
+		DEBUG(0,(__location__ " Unable to allocate dstate\n"));
+		ctdb->statistics.pending_calls--;
+		return;
+	}
+	dstate->start_time = timeval_current();
+	dstate->client = client;
+	dstate->reqid  = c->hdr.reqid;
+	talloc_steal(dstate, data.dptr);
+
+	call = dstate->call = talloc_zero(dstate, struct ctdb_call);
+	if (call == NULL) {
+		ctdb_ltdb_unlock(ctdb_db, key);
+		DEBUG(0,(__location__ " Unable to allocate call\n"));
+		ctdb->statistics.pending_calls--;
+		ctdb_latency(&ctdb->statistics.max_call_latency, dstate->start_time);
+		return;
+	}
+
+	call->call_id = c->callid;
+	call->key = key;
+	call->call_data.dptr = c->data + c->keylen;
+	call->call_data.dsize = c->calldatalen;
+	call->flags = c->flags;
+
+	if (header.dmaster == ctdb->vnn) {
+		state = ctdb_call_local_send(ctdb_db, call, &header, &data);
+	} else {
+		state = ctdb_daemon_call_send_remote(ctdb_db, call, &header);
+	}
+
+	ctdb_ltdb_unlock(ctdb_db, key);
+
+	if (state == NULL) {
+		DEBUG(0,(__location__ " Unable to setup call send\n"));
+		ctdb->statistics.pending_calls--;
+		ctdb_latency(&ctdb->statistics.max_call_latency, dstate->start_time);
+		return;
+	}
+	talloc_steal(state, dstate);
+	talloc_steal(client, state);
+
+	state->async.fn = daemon_call_from_client_callback;
+	state->async.private_data = dstate;
+}
+
+
+static void daemon_request_control_from_client(struct ctdb_client *client, 
+					       struct ctdb_req_control *c);
+
+/* data contains a packet from the client */
+static void daemon_incoming_packet(void *p, struct ctdb_req_header *hdr)
+{
+	struct ctdb_client *client = talloc_get_type(p, struct ctdb_client);
+	TALLOC_CTX *tmp_ctx;
+	struct ctdb_context *ctdb = client->ctdb;
+
+	/* place the packet as a child of a tmp_ctx. We then use
+	   talloc_free() below to free it. If any of the calls want
+	   to keep it, then they will steal it somewhere else, and the
+	   talloc_free() will be a no-op */
+	tmp_ctx = talloc_new(client);
+	talloc_steal(tmp_ctx, hdr);
+
+	if (hdr->ctdb_magic != CTDB_MAGIC) {
+		ctdb_set_error(client->ctdb, "Non CTDB packet rejected in daemon\n");
+		goto done;
+	}
+
+	if (hdr->ctdb_version != CTDB_VERSION) {
+		ctdb_set_error(client->ctdb, "Bad CTDB version 0x%x rejected in daemon\n", hdr->ctdb_version);
+		goto done;
+	}
+
+	switch (hdr->operation) {
+	case CTDB_REQ_CALL:
+		ctdb->statistics.client.req_call++;
+		daemon_request_call_from_client(client, (struct ctdb_req_call *)hdr);
+		break;
+
+	case CTDB_REQ_MESSAGE:
+		ctdb->statistics.client.req_message++;
+		daemon_request_message_from_client(client, (struct ctdb_req_message *)hdr);
+		break;
+
+	case CTDB_REQ_CONTROL:
+		ctdb->statistics.client.req_control++;
+		daemon_request_control_from_client(client, (struct ctdb_req_control *)hdr);
+		break;
+
+	default:
+		DEBUG(0,(__location__ " daemon: unrecognized operation %u\n",
+			 hdr->operation));
+	}
+
+done:
+	talloc_free(tmp_ctx);
+}
+
+/*
+  called when the daemon gets a incoming packet
+ */
+static void ctdb_daemon_read_cb(uint8_t *data, size_t cnt, void *args)
+{
+	struct ctdb_client *client = talloc_get_type(args, struct ctdb_client);
+	struct ctdb_req_header *hdr;
+
+	if (cnt == 0) {
+		talloc_free(client);
+		return;
+	}
+
+	client->ctdb->statistics.client_packets_recv++;
+
+	if (cnt < sizeof(*hdr)) {
+		ctdb_set_error(client->ctdb, "Bad packet length %u in daemon\n", 
+			       (unsigned)cnt);
+		return;
+	}
+	hdr = (struct ctdb_req_header *)data;
+	if (cnt != hdr->length) {
+		ctdb_set_error(client->ctdb, "Bad header length %u expected %u\n in daemon", 
+			       (unsigned)hdr->length, (unsigned)cnt);
+		return;
+	}
+
+	if (hdr->ctdb_magic != CTDB_MAGIC) {
+		ctdb_set_error(client->ctdb, "Non CTDB packet rejected\n");
+		return;
+	}
+
+	if (hdr->ctdb_version != CTDB_VERSION) {
+		ctdb_set_error(client->ctdb, "Bad CTDB version 0x%x rejected in daemon\n", hdr->ctdb_version);
+		return;
+	}
+
+	DEBUG(3,(__location__ " client request %u of type %u length %u from "
+		 "node %u to %u\n", hdr->reqid, hdr->operation, hdr->length,
+		 hdr->srcnode, hdr->destnode));
+
+	/* it is the responsibility of the incoming packet function to free 'data' */
+	daemon_incoming_packet(client, hdr);
+}
+
+static void ctdb_accept_client(struct event_context *ev, struct fd_event *fde, 
+			 uint16_t flags, void *private_data)
+{
+	struct sockaddr_in addr;
+	socklen_t len;
+	int fd;
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+	struct ctdb_client *client;
+
+	memset(&addr, 0, sizeof(addr));
+	len = sizeof(addr);
+	fd = accept(ctdb->daemon.sd, (struct sockaddr *)&addr, &len);
+	if (fd == -1) {
+		return;
+	}
+
+	set_nonblocking(fd);
+	set_close_on_exec(fd);
+
+	client = talloc_zero(ctdb, struct ctdb_client);
+	client->ctdb = ctdb;
+	client->fd = fd;
+	client->client_id = ctdb_reqid_new(ctdb, client);
+	ctdb->statistics.num_clients++;
+
+	client->queue = ctdb_queue_setup(ctdb, client, fd, CTDB_DS_ALIGNMENT, 
+					 ctdb_daemon_read_cb, client);
+
+	talloc_set_destructor(client, ctdb_client_destructor);
+}
+
+
+
+/*
+  create a unix domain socket and bind it
+  return a file descriptor open on the socket 
+*/
+static int ux_socket_bind(struct ctdb_context *ctdb)
+{
+	struct sockaddr_un addr;
+
+	ctdb->daemon.sd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (ctdb->daemon.sd == -1) {
+		return -1;
+	}
+
+	set_nonblocking(ctdb->daemon.sd);
+	set_close_on_exec(ctdb->daemon.sd);
+
+#if 0
+	/* AIX doesn't like this :( */
+	if (fchown(ctdb->daemon.sd, geteuid(), getegid()) != 0 ||
+	    fchmod(ctdb->daemon.sd, 0700) != 0) {
+		DEBUG(0,("Unable to secure ctdb socket '%s', ctdb->daemon.name\n"));
+		goto failed;
+	}
+#endif
+
+	set_nonblocking(ctdb->daemon.sd);
+
+	memset(&addr, 0, sizeof(addr));
+	addr.sun_family = AF_UNIX;
+	strncpy(addr.sun_path, ctdb->daemon.name, sizeof(addr.sun_path));
+
+	if (bind(ctdb->daemon.sd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
+		DEBUG(0,("Unable to bind on ctdb socket '%s'\n", ctdb->daemon.name));
+		goto failed;
+	}	
+	if (listen(ctdb->daemon.sd, 10) != 0) {
+		DEBUG(0,("Unable to listen on ctdb socket '%s'\n", ctdb->daemon.name));
+		goto failed;
+	}
+
+	return 0;
+
+failed:
+	close(ctdb->daemon.sd);
+	ctdb->daemon.sd = -1;
+	return -1;	
+}
+
+/*
+  delete the socket on exit - called on destruction of autofree context
+ */
+static int unlink_destructor(const char *name)
+{
+	unlink(name);
+	return 0;
+}
+
+
+/*
+  start the protocol going as a daemon
+*/
+int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork)
+{
+	int res;
+	struct fd_event *fde;
+	const char *domain_socket_name;
+
+	/* get rid of any old sockets */
+	unlink(ctdb->daemon.name);
+
+	/* create a unix domain stream socket to listen to */
+	res = ux_socket_bind(ctdb);
+	if (res!=0) {
+		DEBUG(0,(__location__ " Failed to open CTDB unix domain socket\n"));
+		exit(10);
+	}
+
+	if (do_fork && fork()) {
+		return 0;
+	}
+
+	tdb_reopen_all(False);
+
+	if (do_fork) {
+		setsid();
+	}
+	block_signal(SIGPIPE);
+
+	/* try to set us up as realtime */
+	ctdb_set_realtime(true);
+
+	/* ensure the socket is deleted on exit of the daemon */
+	domain_socket_name = talloc_strdup(talloc_autofree_context(), ctdb->daemon.name);
+	talloc_set_destructor(domain_socket_name, unlink_destructor);	
+
+	ctdb->ev = event_context_init(NULL);
+
+	/* start frozen, then let the first election sort things out */
+	if (!ctdb_blocking_freeze(ctdb)) {
+		DEBUG(0,("Failed to get initial freeze\n"));
+		exit(12);
+	}
+
+	/* force initial recovery for election */
+	ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
+
+	/* now start accepting clients, only can do this once frozen */
+	fde = event_add_fd(ctdb->ev, ctdb, ctdb->daemon.sd, 
+			   EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+			   ctdb_accept_client, ctdb);
+
+	ctdb_main_loop(ctdb);
+
+	return 0;
+}
+
+/*
+  allocate a packet for use in daemon<->daemon communication
+ */
+struct ctdb_req_header *_ctdb_transport_allocate(struct ctdb_context *ctdb,
+						 TALLOC_CTX *mem_ctx, 
+						 enum ctdb_operation operation, 
+						 size_t length, size_t slength,
+						 const char *type)
+{
+	int size;
+	struct ctdb_req_header *hdr;
+
+	length = MAX(length, slength);
+	size = (length+(CTDB_DS_ALIGNMENT-1)) & ~(CTDB_DS_ALIGNMENT-1);
+
+	hdr = (struct ctdb_req_header *)ctdb->methods->allocate_pkt(mem_ctx, size);
+	if (hdr == NULL) {
+		DEBUG(0,("Unable to allocate transport packet for operation %u of length %u\n",
+			 operation, (unsigned)length));
+		return NULL;
+	}
+	talloc_set_name_const(hdr, type);
+	memset(hdr, 0, slength);
+	hdr->length       = length;
+	hdr->operation    = operation;
+	hdr->ctdb_magic   = CTDB_MAGIC;
+	hdr->ctdb_version = CTDB_VERSION;
+	hdr->generation   = ctdb->vnn_map->generation;
+	hdr->srcnode      = ctdb->vnn;
+
+	return hdr;	
+}
+
+struct daemon_control_state {
+	struct daemon_control_state *next, *prev;
+	struct ctdb_client *client;
+	struct ctdb_req_control *c;
+	uint32_t reqid;
+	struct ctdb_node *node;
+};
+
+/*
+  callback when a control reply comes in
+ */
+static void daemon_control_callback(struct ctdb_context *ctdb,
+				    int32_t status, TDB_DATA data, 
+				    const char *errormsg,
+				    void *private_data)
+{
+	struct daemon_control_state *state = talloc_get_type(private_data, 
+							     struct daemon_control_state);
+	struct ctdb_client *client = state->client;
+	struct ctdb_reply_control *r;
+	size_t len;
+
+	/* construct a message to send to the client containing the data */
+	len = offsetof(struct ctdb_reply_control, data) + data.dsize;
+	if (errormsg) {
+		len += strlen(errormsg);
+	}
+	r = ctdbd_allocate_pkt(ctdb, state, CTDB_REPLY_CONTROL, len, 
+			       struct ctdb_reply_control);
+	CTDB_NO_MEMORY_VOID(ctdb, r);
+
+	r->hdr.reqid     = state->reqid;
+	r->status        = status;
+	r->datalen       = data.dsize;
+	r->errorlen = 0;
+	memcpy(&r->data[0], data.dptr, data.dsize);
+	if (errormsg) {
+		r->errorlen = strlen(errormsg);
+		memcpy(&r->data[r->datalen], errormsg, r->errorlen);
+	}
+
+	daemon_queue_send(client, &r->hdr);
+
+	talloc_free(state);
+}
+
+/*
+  fail all pending controls to a disconnected node
+ */
+void ctdb_daemon_cancel_controls(struct ctdb_context *ctdb, struct ctdb_node *node)
+{
+	struct daemon_control_state *state;
+	while ((state = node->pending_controls)) {
+		DLIST_REMOVE(node->pending_controls, state);
+		daemon_control_callback(ctdb, (uint32_t)-1, tdb_null, 
+					"node is disconnected", state);
+	}
+}
+
+/*
+  destroy a daemon_control_state
+ */
+static int daemon_control_destructor(struct daemon_control_state *state)
+{
+	if (state->node) {
+		DLIST_REMOVE(state->node->pending_controls, state);
+	}
+	return 0;
+}
+
+/*
+  this is called when the ctdb daemon received a ctdb request control
+  from a local client over the unix domain socket
+ */
+static void daemon_request_control_from_client(struct ctdb_client *client, 
+					       struct ctdb_req_control *c)
+{
+	TDB_DATA data;
+	int res;
+	struct daemon_control_state *state;
+	TALLOC_CTX *tmp_ctx = talloc_new(client);
+
+	if (c->hdr.destnode == CTDB_CURRENT_NODE) {
+		c->hdr.destnode = client->ctdb->vnn;
+	}
+
+	state = talloc(client, struct daemon_control_state);
+	CTDB_NO_MEMORY_VOID(client->ctdb, state);
+
+	state->client = client;
+	state->c = talloc_steal(state, c);
+	state->reqid = c->hdr.reqid;
+	if (ctdb_validate_vnn(client->ctdb, c->hdr.destnode)) {
+		state->node = client->ctdb->nodes[c->hdr.destnode];
+		DLIST_ADD(state->node->pending_controls, state);
+	} else {
+		state->node = NULL;
+	}
+
+	talloc_set_destructor(state, daemon_control_destructor);
+
+	if (c->flags & CTDB_CTRL_FLAG_NOREPLY) {
+		talloc_steal(tmp_ctx, state);
+	}
+	
+	data.dptr = &c->data[0];
+	data.dsize = c->datalen;
+	res = ctdb_daemon_send_control(client->ctdb, c->hdr.destnode,
+				       c->srvid, c->opcode, client->client_id,
+				       c->flags,
+				       data, daemon_control_callback,
+				       state);
+	if (res != 0) {
+		DEBUG(0,(__location__ " Failed to send control to remote node %u\n",
+			 c->hdr.destnode));
+	}
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+  register a call function
+*/
+int ctdb_daemon_set_call(struct ctdb_context *ctdb, uint32_t db_id,
+			 ctdb_fn_t fn, int id)
+{
+	struct ctdb_registered_call *call;
+	struct ctdb_db_context *ctdb_db;
+
+	ctdb_db = find_ctdb_db(ctdb, db_id);
+	if (ctdb_db == NULL) {
+		return -1;
+	}
+
+	call = talloc(ctdb_db, struct ctdb_registered_call);
+	call->fn = fn;
+	call->id = id;
+
+	DLIST_ADD(ctdb_db->calls, call);	
+	return 0;
+}
+
+
+
+/*
+  this local messaging handler is ugly, but is needed to prevent
+  recursion in ctdb_send_message() when the destination node is the
+  same as the source node
+ */
+struct ctdb_local_message {
+	struct ctdb_context *ctdb;
+	uint64_t srvid;
+	TDB_DATA data;
+};
+
+static void ctdb_local_message_trigger(struct event_context *ev, struct timed_event *te, 
+				       struct timeval t, void *private_data)
+{
+	struct ctdb_local_message *m = talloc_get_type(private_data, 
+						       struct ctdb_local_message);
+	int res;
+
+	res = ctdb_dispatch_message(m->ctdb, m->srvid, m->data);
+	if (res != 0) {
+		DEBUG(0, (__location__ " Failed to dispatch message for srvid=%llu\n", 
+			  (unsigned long long)m->srvid));
+	}
+	talloc_free(m);
+}
+
+static int ctdb_local_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA data)
+{
+	struct ctdb_local_message *m;
+	m = talloc(ctdb, struct ctdb_local_message);
+	CTDB_NO_MEMORY(ctdb, m);
+
+	m->ctdb = ctdb;
+	m->srvid = srvid;
+	m->data  = data;
+	m->data.dptr = talloc_memdup(m, m->data.dptr, m->data.dsize);
+	if (m->data.dptr == NULL) {
+		talloc_free(m);
+		return -1;
+	}
+
+	/* this needs to be done as an event to prevent recursion */
+	event_add_timed(ctdb->ev, m, timeval_zero(), ctdb_local_message_trigger, m);
+	return 0;
+}
+
+/*
+  send a ctdb message
+*/
+int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t vnn,
+			     uint64_t srvid, TDB_DATA data)
+{
+	struct ctdb_req_message *r;
+	int len;
+
+	/* see if this is a message to ourselves */
+	if (vnn == ctdb->vnn) {
+		return ctdb_local_message(ctdb, srvid, data);
+	}
+
+	len = offsetof(struct ctdb_req_message, data) + data.dsize;
+	r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REQ_MESSAGE, len,
+				    struct ctdb_req_message);
+	CTDB_NO_MEMORY(ctdb, r);
+
+	r->hdr.destnode  = vnn;
+	r->srvid         = srvid;
+	r->datalen       = data.dsize;
+	memcpy(&r->data[0], data.dptr, data.dsize);
+
+	ctdb_queue_packet(ctdb, &r->hdr);
+
+	talloc_free(r);
+	return 0;
+}
+
diff --git a/source4/cluster/ctdb/server/ctdb_freeze.c b/source4/cluster/ctdb/server/ctdb_freeze.c
new file mode 100644
index 0000000000..42ad975b53
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_freeze.c
@@ -0,0 +1,256 @@
+/* 
+   ctdb freeze handling
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+#include "lib/util/dlinklist.h"
+#include "db_wrap.h"
+
+
+/*
+  lock all databases
+ */
+static int ctdb_lock_all_databases(struct ctdb_context *ctdb)
+{
+	struct ctdb_db_context *ctdb_db;
+	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+		if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/*
+  a list of control requests waiting for a freeze lock child to get
+  the database locks
+ */
+struct ctdb_freeze_waiter {
+	struct ctdb_freeze_waiter *next, *prev;
+	struct ctdb_context *ctdb;
+	struct ctdb_req_control *c;
+	int32_t status;
+};
+
+/* a handle to a freeze lock child process */
+struct ctdb_freeze_handle {
+	struct ctdb_context *ctdb;
+	pid_t child;
+	int fd;
+	struct ctdb_freeze_waiter *waiters;
+};
+
+/*
+  destroy a freeze handle
+ */	
+static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
+{
+	h->ctdb->freeze_mode = CTDB_FREEZE_NONE;
+	kill(h->child, SIGKILL);
+	waitpid(h->child, NULL, 0);
+	return 0;
+}
+
+/*
+  called when the child writes its status to us
+ */
+static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *fde, 
+				       uint16_t flags, void *private_data)
+{
+	struct ctdb_freeze_handle *h = talloc_get_type(private_data, struct ctdb_freeze_handle);
+	int32_t status;
+	struct ctdb_freeze_waiter *w;
+
+	if (h->ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("freeze child died - unfreezing\n"));
+		talloc_free(h);
+		return;
+	}
+
+	if (read(h->fd, &status, sizeof(status)) != sizeof(status)) {
+		DEBUG(0,("read error from freeze lock child\n"));
+		status = -1;
+	}
+
+	if (status == -1) {
+		DEBUG(0,("Failed to get locks in ctdb_freeze_child\n"));
+		/* we didn't get the locks - destroy the handle */
+		talloc_free(h);
+		return;
+	}
+
+	h->ctdb->freeze_mode = CTDB_FREEZE_FROZEN;
+
+	/* notify the waiters */
+	while ((w = h->ctdb->freeze_handle->waiters)) {
+		w->status = status;
+		DLIST_REMOVE(h->ctdb->freeze_handle->waiters, w);
+		talloc_free(w);
+	}
+}
+
+/*
+  create a child which gets locks on all the open databases, then calls the callback telling the parent
+  that it is done
+ */
+static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
+{
+	struct ctdb_freeze_handle *h;
+	int fd[2];
+	struct fd_event *fde;
+
+	h = talloc_zero(ctdb, struct ctdb_freeze_handle);
+	CTDB_NO_MEMORY_VOID(ctdb, h);
+
+	h->ctdb = ctdb;
+
+	/* use socketpair() instead of pipe() so we have bi-directional fds */
+	if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != 0) {
+		DEBUG(0,("Failed to create pipe for ctdb_freeze_lock\n"));
+		talloc_free(h);
+		return NULL;
+	}
+	
+	h->child = fork();
+	if (h->child == -1) {
+		DEBUG(0,("Failed to fork child for ctdb_freeze_lock\n"));
+		talloc_free(h);
+		return NULL;
+	}
+
+	if (h->child == 0) {
+		int ret;
+		/* in the child */
+		close(fd[0]);
+		ret = ctdb_lock_all_databases(ctdb);
+		if (ret != 0) {
+			_exit(0);
+		}
+		write(fd[1], &ret, sizeof(ret));
+		/* the read here means we will die if the parent exits */
+		read(fd[1], &ret, sizeof(ret));
+		_exit(0);
+	}
+
+	talloc_set_destructor(h, ctdb_freeze_handle_destructor);
+
+	close(fd[1]);
+
+	h->fd = fd[0];
+
+	fde = event_add_fd(ctdb->ev, h, h->fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+			   ctdb_freeze_lock_handler, h);
+	if (fde == NULL) {
+		DEBUG(0,("Failed to setup fd event for ctdb_freeze_lock\n"));
+		close(fd[0]);
+		talloc_free(h);
+		return NULL;
+	}
+
+	return h;
+}
+
+/*
+  destroy a waiter for a freeze mode change
+ */
+static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
+{
+	DLIST_REMOVE(w->ctdb->freeze_handle->waiters, w);
+	ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
+	return 0;
+}
+
+/*
+  start the freeze process
+ */
+void ctdb_start_freeze(struct ctdb_context *ctdb)
+{
+	if (ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+		/* we're already frozen */
+		return;
+	}
+
+	/* if there isn't a freeze lock child then create one */
+	if (!ctdb->freeze_handle) {
+		ctdb->freeze_handle = ctdb_freeze_lock(ctdb);
+		CTDB_NO_MEMORY_VOID(ctdb, ctdb->freeze_handle);
+		ctdb->freeze_mode = CTDB_FREEZE_PENDING;
+	}
+}
+
+/*
+  freeze the databases
+ */
+int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply)
+{
+	struct ctdb_freeze_waiter *w;
+
+	if (ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+		/* we're already frozen */
+		return 0;
+	}
+
+	ctdb_start_freeze(ctdb);
+
+	/* add ourselves to list of waiters */
+	w = talloc(ctdb->freeze_handle, struct ctdb_freeze_waiter);
+	CTDB_NO_MEMORY(ctdb, w);
+	w->ctdb   = ctdb;
+	w->c      = talloc_steal(w, c);
+	w->status = -1;
+	talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
+	DLIST_ADD(ctdb->freeze_handle->waiters, w);
+
+	/* we won't reply till later */
+	*async_reply = True;
+	return 0;
+}
+
+
+/*
+  block until we are frozen, used during daemon startup
+ */
+bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
+{
+	ctdb_start_freeze(ctdb);
+
+	/* block until frozen */
+	while (ctdb->freeze_mode == CTDB_FREEZE_PENDING) {
+		event_loop_once(ctdb->ev);
+	}
+
+	return ctdb->freeze_mode == CTDB_FREEZE_FROZEN;
+}
+
+
+
+/*
+  thaw the databases
+ */
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb)
+{
+	talloc_free(ctdb->freeze_handle);
+	ctdb->freeze_handle = NULL;
+	ctdb_call_resend_all(ctdb);
+	return 0;
+}
diff --git a/source4/cluster/ctdb/common/ctdb_lockwait.c b/source4/cluster/ctdb/server/ctdb_lockwait.c
index 2134cb95e9..5b0019836e 100644
--- a/source4/cluster/ctdb/common/ctdb_lockwait.c
+++ b/source4/cluster/ctdb/server/ctdb_lockwait.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -28,11 +28,13 @@
 
 struct lockwait_handle {
 	struct ctdb_context *ctdb;
+	struct ctdb_db_context *ctdb_db;
 	struct fd_event *fde;
 	int fd[2];
 	pid_t child;
 	void *private_data;
 	void (*callback)(void *);
+	TDB_DATA key;
 	struct timeval start_time;
 };
 
@@ -44,19 +46,39 @@ static void lockwait_handler(struct event_context *ev, struct fd_event *fde,
 	void (*callback)(void *) = h->callback;
 	void *p = h->private_data;
 	pid_t child = h->child;
+	TDB_DATA key = h->key;
+	struct tdb_context *tdb = h->ctdb_db->ltdb->tdb;
+	TALLOC_CTX *tmp_ctx = talloc_new(ev);
+
+	key.dptr = talloc_memdup(tmp_ctx, key.dptr, key.dsize);
+
 	talloc_set_destructor(h, NULL);
-	close(h->fd[0]);
-	ctdb_latency(&h->ctdb->status.max_lockwait_latency, h->start_time);
-	h->ctdb->status.pending_lockwait_calls--;
-	talloc_free(h);	
+	ctdb_latency(&h->ctdb->statistics.max_lockwait_latency, h->start_time);
+	h->ctdb->statistics.pending_lockwait_calls--;
+
+	/* the handle needs to go away when the context is gone - when
+	   the handle goes away this implicitly closes the pipe, which
+	   kills the child holding the lock */
+	talloc_steal(tmp_ctx, h);
+
+	if (h->ctdb->flags & CTDB_FLAG_TORTURE) {
+		if (tdb_chainlock_nonblock(tdb, key) == 0) {
+			ctdb_fatal(h->ctdb, "got chain lock while lockwait child active");
+		}
+	}
+
+	tdb_chainlock_mark(tdb, key);
 	callback(p);
+	tdb_chainlock_unmark(tdb, key);
+
+	kill(child, SIGKILL);
 	waitpid(child, NULL, 0);
+	talloc_free(tmp_ctx);
 }
 
 static int lockwait_destructor(struct lockwait_handle *h)
 {
-	h->ctdb->status.pending_lockwait_calls--;
-	close(h->fd[0]);
+	h->ctdb->statistics.pending_lockwait_calls--;
 	kill(h->child, SIGKILL);
 	waitpid(h->child, NULL, 0);
 	return 0;
@@ -79,12 +101,13 @@ struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
 {
 	struct lockwait_handle *result;
 	int ret;
+	pid_t parent = getpid();
 
-	ctdb_db->ctdb->status.lockwait_calls++;
-	ctdb_db->ctdb->status.pending_lockwait_calls++;
+	ctdb_db->ctdb->statistics.lockwait_calls++;
+	ctdb_db->ctdb->statistics.pending_lockwait_calls++;
 
-	if (!(result = talloc_zero(ctdb_db, struct lockwait_handle))) {
-		ctdb_db->ctdb->status.pending_lockwait_calls--;
+	if (!(result = talloc_zero(private_data, struct lockwait_handle))) {
+		ctdb_db->ctdb->statistics.pending_lockwait_calls--;
 		return NULL;
 	}
 
@@ -92,7 +115,7 @@ struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
 
 	if (ret != 0) {
 		talloc_free(result);
-		ctdb_db->ctdb->status.pending_lockwait_calls--;
+		ctdb_db->ctdb->statistics.pending_lockwait_calls--;
 		return NULL;
 	}
 
@@ -102,20 +125,25 @@ struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
 		close(result->fd[0]);
 		close(result->fd[1]);
 		talloc_free(result);
-		ctdb_db->ctdb->status.pending_lockwait_calls--;
+		ctdb_db->ctdb->statistics.pending_lockwait_calls--;
 		return NULL;
 	}
 
 	result->callback = callback;
 	result->private_data = private_data;
 	result->ctdb = ctdb_db->ctdb;
+	result->ctdb_db = ctdb_db;
+	result->key = key;
 
 	if (result->child == 0) {
+		char c = 0;
 		close(result->fd[0]);
-		/*
-		 * Do we need a tdb_reopen here?
-		 */
 		tdb_chainlock(ctdb_db->ltdb->tdb, key);
+		write(result->fd[1], &c, 1);
+		/* make sure we die when our parent dies */
+		while (kill(parent, 0) == 0 || errno != ESRCH) {
+			sleep(5);
+		}
 		_exit(0);
 	}
 
@@ -123,11 +151,11 @@ struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
 	talloc_set_destructor(result, lockwait_destructor);
 
 	result->fde = event_add_fd(ctdb_db->ctdb->ev, result, result->fd[0],
-				   EVENT_FD_READ, lockwait_handler,
+				   EVENT_FD_READ|EVENT_FD_AUTOCLOSE, lockwait_handler,
 				   (void *)result);
 	if (result->fde == NULL) {
 		talloc_free(result);
-		ctdb_db->ctdb->status.pending_lockwait_calls--;
+		ctdb_db->ctdb->statistics.pending_lockwait_calls--;
 		return NULL;
 	}
 
diff --git a/source4/cluster/ctdb/server/ctdb_ltdb_server.c b/source4/cluster/ctdb/server/ctdb_ltdb_server.c
new file mode 100644
index 0000000000..bd07f674db
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_ltdb_server.c
@@ -0,0 +1,366 @@
+/* 
+   ctdb ltdb code - server side
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "../include/ctdb_private.h"
+#include "db_wrap.h"
+#include "lib/util/dlinklist.h"
+
+/*
+  this is the dummy null procedure that all databases support
+*/
+static int ctdb_null_func(struct ctdb_call_info *call)
+{
+	return 0;
+}
+
+/*
+  this is a plain fetch procedure that all databases support
+*/
+static int ctdb_fetch_func(struct ctdb_call_info *call)
+{
+	call->reply_data = &call->record_data;
+	return 0;
+}
+
+
+
+struct lock_fetch_state {
+	struct ctdb_context *ctdb;
+	void (*recv_pkt)(void *, struct ctdb_req_header *);
+	void *recv_context;
+	struct ctdb_req_header *hdr;
+	uint32_t generation;
+	bool ignore_generation;
+};
+
+/*
+  called when we should retry the operation
+ */
+static void lock_fetch_callback(void *p)
+{
+	struct lock_fetch_state *state = talloc_get_type(p, struct lock_fetch_state);
+	if (!state->ignore_generation &&
+	    state->generation != state->ctdb->vnn_map->generation) {
+		DEBUG(0,("Discarding previous generation lockwait packet\n"));
+		talloc_free(state->hdr);
+		return;
+	}
+	state->recv_pkt(state->recv_context, state->hdr);
+	DEBUG(2,(__location__ " PACKET REQUEUED\n"));
+}
+
+
+/*
+  do a non-blocking ltdb_lock, deferring this ctdb request until we
+  have the chainlock
+
+  It does the following:
+
+   1) tries to get the chainlock. If it succeeds, then it returns 0
+
+   2) if it fails to get a chainlock immediately then it sets up a
+   non-blocking chainlock via ctdb_lockwait, and when it gets the
+   chainlock it re-submits this ctdb request to the main packet
+   receive function
+
+   This effectively queues all ctdb requests that cannot be
+   immediately satisfied until it can get the lock. This means that
+   the main ctdb daemon will not block waiting for a chainlock held by
+   a client
+
+   There are 3 possible return values:
+
+       0:    means that it got the lock immediately.
+      -1:    means that it failed to get the lock, and won't retry
+      -2:    means that it failed to get the lock immediately, but will retry
+ */
+int ctdb_ltdb_lock_requeue(struct ctdb_db_context *ctdb_db, 
+			   TDB_DATA key, struct ctdb_req_header *hdr,
+			   void (*recv_pkt)(void *, struct ctdb_req_header *),
+			   void *recv_context, bool ignore_generation)
+{
+	int ret;
+	struct tdb_context *tdb = ctdb_db->ltdb->tdb;
+	struct lockwait_handle *h;
+	struct lock_fetch_state *state;
+	
+	ret = tdb_chainlock_nonblock(tdb, key);
+
+	if (ret != 0 &&
+	    !(errno == EACCES || errno == EAGAIN || errno == EDEADLK)) {
+		/* a hard failure - don't try again */
+		return -1;
+	}
+
+	/* when torturing, ensure we test the contended path */
+	if ((ctdb_db->ctdb->flags & CTDB_FLAG_TORTURE) &&
+	    random() % 5 == 0) {
+		ret = -1;
+		tdb_chainunlock(tdb, key);
+	}
+
+	/* first the non-contended path */
+	if (ret == 0) {
+		return 0;
+	}
+
+	state = talloc(hdr, struct lock_fetch_state);
+	state->ctdb = ctdb_db->ctdb;
+	state->hdr = hdr;
+	state->recv_pkt = recv_pkt;
+	state->recv_context = recv_context;
+	state->generation = ctdb_db->ctdb->vnn_map->generation;
+	state->ignore_generation = ignore_generation;
+
+	/* now the contended path */
+	h = ctdb_lockwait(ctdb_db, key, lock_fetch_callback, state);
+	if (h == NULL) {
+		tdb_chainunlock(tdb, key);
+		return -1;
+	}
+
+	/* we need to move the packet off the temporary context in ctdb_input_pkt(),
+	   so it won't be freed yet */
+	talloc_steal(state, hdr);
+	talloc_steal(state, h);
+
+	/* now tell the caller than we will retry asynchronously */
+	return -2;
+}
+
+/*
+  a varient of ctdb_ltdb_lock_requeue that also fetches the record
+ */
+int ctdb_ltdb_lock_fetch_requeue(struct ctdb_db_context *ctdb_db, 
+				 TDB_DATA key, struct ctdb_ltdb_header *header, 
+				 struct ctdb_req_header *hdr, TDB_DATA *data,
+				 void (*recv_pkt)(void *, struct ctdb_req_header *),
+				 void *recv_context, bool ignore_generation)
+{
+	int ret;
+
+	ret = ctdb_ltdb_lock_requeue(ctdb_db, key, hdr, recv_pkt, 
+				     recv_context, ignore_generation);
+	if (ret == 0) {
+		ret = ctdb_ltdb_fetch(ctdb_db, key, header, hdr, data);
+		if (ret != 0) {
+			ctdb_ltdb_unlock(ctdb_db, key);
+		}
+	}
+	return ret;
+}
+
+
+/*
+  paraoid check to see if the db is empty
+ */
+static void ctdb_check_db_empty(struct ctdb_db_context *ctdb_db)
+{
+	struct tdb_context *tdb = ctdb_db->ltdb->tdb;
+	int count = tdb_traverse_read(tdb, NULL, NULL);
+	if (count != 0) {
+		DEBUG(0,(__location__ " tdb '%s' not empty on attach! aborting\n",
+			 ctdb_db->db_path));
+		ctdb_fatal(ctdb_db->ctdb, "database not empty on attach");
+	}
+}
+
+/*
+  a client has asked to attach a new database
+ */
+int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
+			       TDB_DATA *outdata)
+{
+	const char *db_name = (const char *)indata.dptr;
+	struct ctdb_db_context *ctdb_db, *tmp_db;
+	int ret;
+
+	/* see if we already have this name */
+	for (tmp_db=ctdb->db_list;tmp_db;tmp_db=tmp_db->next) {
+		if (strcmp(db_name, tmp_db->db_name) == 0) {
+			/* this is not an error */
+			outdata->dptr  = (uint8_t *)&tmp_db->db_id;
+			outdata->dsize = sizeof(tmp_db->db_id);
+			return 0;
+		}
+	}
+
+	ctdb_db = talloc_zero(ctdb, struct ctdb_db_context);
+	CTDB_NO_MEMORY(ctdb, ctdb_db);
+
+	ctdb_db->ctdb = ctdb;
+	ctdb_db->db_name = talloc_strdup(ctdb_db, db_name);
+	CTDB_NO_MEMORY(ctdb, ctdb_db->db_name);
+
+	ctdb_db->db_id = ctdb_hash(&indata);
+
+	outdata->dptr  = (uint8_t *)&ctdb_db->db_id;
+	outdata->dsize = sizeof(ctdb_db->db_id);
+
+	/* check for hash collisions */
+	for (tmp_db=ctdb->db_list;tmp_db;tmp_db=tmp_db->next) {
+		if (tmp_db->db_id == ctdb_db->db_id) {
+			DEBUG(0,("db_id 0x%x hash collision. name1='%s' name2='%s'\n",
+				 tmp_db->db_id, db_name, tmp_db->db_name));
+			talloc_free(ctdb_db);
+			return -1;
+		}
+	}
+
+	if (ctdb->db_directory == NULL) {
+		ctdb->db_directory = VARDIR "/ctdb";
+	}
+
+	/* make sure the db directory exists */
+	if (mkdir(ctdb->db_directory, 0700) == -1 && errno != EEXIST) {
+		DEBUG(0,(__location__ " Unable to create ctdb directory '%s'\n", 
+			 ctdb->db_directory));
+		talloc_free(ctdb_db);
+		return -1;
+	}
+
+	/* open the database */
+	ctdb_db->db_path = talloc_asprintf(ctdb_db, "%s/%s.%u", 
+					   ctdb->db_directory, 
+					   db_name, ctdb->vnn);
+
+	ctdb_db->ltdb = tdb_wrap_open(ctdb, ctdb_db->db_path, 
+				      ctdb->tunable.database_hash_size, 
+				      TDB_CLEAR_IF_FIRST, O_CREAT|O_RDWR, 0666);
+	if (ctdb_db->ltdb == NULL) {
+		DEBUG(0,("Failed to open tdb '%s'\n", ctdb_db->db_path));
+		talloc_free(ctdb_db);
+		return -1;
+	}
+
+	ctdb_check_db_empty(ctdb_db);
+
+	DLIST_ADD(ctdb->db_list, ctdb_db);
+
+	/* 
+	   all databases support the "null" function. we need this in
+	   order to do forced migration of records
+	*/
+	ret = ctdb_daemon_set_call(ctdb, ctdb_db->db_id, ctdb_null_func, CTDB_NULL_FUNC);
+	if (ret != 0) {
+		DEBUG(0,("Failed to setup null function for '%s'\n", ctdb_db->db_name));
+		talloc_free(ctdb_db);
+		return -1;
+	}
+
+	/* 
+	   all databases support the "fetch" function. we need this
+	   for efficient Samba3 ctdb fetch
+	*/
+	ret = ctdb_daemon_set_call(ctdb, ctdb_db->db_id, ctdb_fetch_func, CTDB_FETCH_FUNC);
+	if (ret != 0) {
+		DEBUG(0,("Failed to setup fetch function for '%s'\n", ctdb_db->db_name));
+		talloc_free(ctdb_db);
+		return -1;
+	}
+	
+	/* tell all the other nodes about this database */
+	ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL, 0,
+				 CTDB_CONTROL_DB_ATTACH, 0, CTDB_CTRL_FLAG_NOREPLY,
+				 indata, NULL, NULL);
+
+	DEBUG(1,("Attached to database '%s'\n", ctdb_db->db_path));
+
+	/* success */
+	return 0;
+}
+
+/*
+  called when a broadcast seqnum update comes in
+ */
+int32_t ctdb_ltdb_update_seqnum(struct ctdb_context *ctdb, uint32_t db_id, uint32_t srcnode)
+{
+	struct ctdb_db_context *ctdb_db;
+	if (srcnode == ctdb->vnn) {
+		/* don't update ourselves! */
+		return 0;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, db_id);
+	if (!ctdb_db) {
+		DEBUG(0,("Unknown db_id 0x%x in ctdb_ltdb_update_seqnum\n", db_id));
+		return -1;
+	}
+
+	tdb_increment_seqnum_nonblock(ctdb_db->ltdb->tdb);
+	ctdb_db->seqnum = tdb_get_seqnum(ctdb_db->ltdb->tdb);
+	return 0;
+}
+
+/*
+  timer to check for seqnum changes in a ltdb and propogate them
+ */
+static void ctdb_ltdb_seqnum_check(struct event_context *ev, struct timed_event *te, 
+				   struct timeval t, void *p)
+{
+	struct ctdb_db_context *ctdb_db = talloc_get_type(p, struct ctdb_db_context);
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	uint32_t new_seqnum = tdb_get_seqnum(ctdb_db->ltdb->tdb);
+	if (new_seqnum != ctdb_db->seqnum) {
+		/* something has changed - propogate it */
+		TDB_DATA data;
+		data.dptr = (uint8_t *)&ctdb_db->db_id;
+		data.dsize = sizeof(uint32_t);
+		ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNNMAP, 0,
+					 CTDB_CONTROL_UPDATE_SEQNUM, 0, CTDB_CTRL_FLAG_NOREPLY,
+					 data, NULL, NULL);		
+	}
+	ctdb_db->seqnum = new_seqnum;
+
+	/* setup a new timer */
+	ctdb_db->te = 
+		event_add_timed(ctdb->ev, ctdb_db, 
+				timeval_current_ofs(ctdb->tunable.seqnum_frequency, 0),
+				ctdb_ltdb_seqnum_check, ctdb_db);
+}
+
+/*
+  enable seqnum handling on this db
+ */
+int32_t ctdb_ltdb_enable_seqnum(struct ctdb_context *ctdb, uint32_t db_id)
+{
+	struct ctdb_db_context *ctdb_db;
+	ctdb_db = find_ctdb_db(ctdb, db_id);
+	if (!ctdb_db) {
+		DEBUG(0,("Unknown db_id 0x%x in ctdb_ltdb_enable_seqnum\n", db_id));
+		return -1;
+	}
+
+	if (ctdb_db->te == NULL) {
+		ctdb_db->te = 
+			event_add_timed(ctdb->ev, ctdb_db, 
+					timeval_current_ofs(ctdb->tunable.seqnum_frequency, 0),
+					ctdb_ltdb_seqnum_check, ctdb_db);
+	}
+
+	tdb_enable_seqnum(ctdb_db->ltdb->tdb);
+	ctdb_db->seqnum = tdb_get_seqnum(ctdb_db->ltdb->tdb);
+	return 0;
+}
+
diff --git a/source4/cluster/ctdb/server/ctdb_monitor.c b/source4/cluster/ctdb/server/ctdb_monitor.c
new file mode 100644
index 0000000000..ec5244703c
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_monitor.c
@@ -0,0 +1,227 @@
+/* 
+   monitoring links to all other nodes to detect dead nodes
+
+
+   Copyright (C) Ronnie Sahlberg 2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+
+/*
+  see if any nodes are dead
+ */
+static void ctdb_check_for_dead_nodes(struct event_context *ev, struct timed_event *te, 
+				      struct timeval t, void *private_data)
+{
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+	int i;
+
+	if (ctdb->monitoring_mode == CTDB_MONITORING_DISABLED) {
+		event_add_timed(ctdb->ev, ctdb->monitor_context, 
+			timeval_current_ofs(ctdb->tunable.keepalive_interval, 0), 
+			ctdb_check_for_dead_nodes, ctdb);
+		return;
+	}
+
+	/* send a keepalive to all other nodes, unless */
+	for (i=0;i<ctdb->num_nodes;i++) {
+		struct ctdb_node *node = ctdb->nodes[i];
+		if (node->vnn == ctdb->vnn) {
+			continue;
+		}
+		
+		if (node->flags & NODE_FLAGS_DISCONNECTED) {
+			/* it might have come alive again */
+			if (node->rx_cnt != 0) {
+				ctdb_node_connected(node);
+			}
+			continue;
+		}
+
+
+		if (node->rx_cnt == 0) {
+			node->dead_count++;
+		} else {
+			node->dead_count = 0;
+		}
+
+		node->rx_cnt = 0;
+
+		if (node->dead_count >= ctdb->tunable.keepalive_limit) {
+			DEBUG(0,("dead count reached for node %u\n", node->vnn));
+			ctdb_node_dead(node);
+			ctdb_send_keepalive(ctdb, node->vnn);
+			/* maybe tell the transport layer to kill the
+			   sockets as well?
+			*/
+			continue;
+		}
+		
+		if (node->tx_cnt == 0) {
+			DEBUG(5,("sending keepalive to %u\n", node->vnn));
+			ctdb_send_keepalive(ctdb, node->vnn);
+		}
+
+		node->tx_cnt = 0;
+	}
+	
+	event_add_timed(ctdb->ev, ctdb->monitor_context, 
+			timeval_current_ofs(ctdb->tunable.keepalive_interval, 0), 
+			ctdb_check_for_dead_nodes, ctdb);
+}
+
+static void ctdb_check_health(struct event_context *ev, struct timed_event *te, 
+			      struct timeval t, void *private_data);
+
+/*
+  called when a health monitoring event script finishes
+ */
+static void ctdb_health_callback(struct ctdb_context *ctdb, int status, void *p)
+{
+	struct ctdb_node *node = ctdb->nodes[ctdb->vnn];
+	TDB_DATA data;
+	struct ctdb_node_flag_change c;
+
+	event_add_timed(ctdb->ev, ctdb->monitor_context, 
+			timeval_current_ofs(ctdb->tunable.monitor_interval, 0), 
+			ctdb_check_health, ctdb);
+
+	if (status != 0 && !(node->flags & NODE_FLAGS_UNHEALTHY)) {
+		DEBUG(0,("monitor event failed - disabling node\n"));
+		node->flags |= NODE_FLAGS_UNHEALTHY;
+	} else if (status == 0 && (node->flags & NODE_FLAGS_UNHEALTHY)) {
+		DEBUG(0,("monitor event OK - node re-enabled\n"));
+		ctdb->nodes[ctdb->vnn]->flags &= ~NODE_FLAGS_UNHEALTHY;
+	} else {
+		/* no change */
+		return;
+	}
+
+	c.vnn = ctdb->vnn;
+	c.flags = node->flags;
+
+	data.dptr = (uint8_t *)&c;
+	data.dsize = sizeof(c);
+
+	/* tell the other nodes that something has changed */
+	ctdb_daemon_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
+				 CTDB_SRVID_NODE_FLAGS_CHANGED, data);
+
+}
+
+
+/*
+  see if the event scripts think we are healthy
+ */
+static void ctdb_check_health(struct event_context *ev, struct timed_event *te, 
+			      struct timeval t, void *private_data)
+{
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+	int ret;
+
+	if (ctdb->monitoring_mode == CTDB_MONITORING_DISABLED) {
+		event_add_timed(ctdb->ev, ctdb->monitor_context,
+				timeval_current_ofs(ctdb->tunable.monitor_interval, 0), 
+				ctdb_check_health, ctdb);
+		return;
+	}
+	
+	ret = ctdb_event_script_callback(ctdb, 
+					 timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+					 ctdb->monitor_context, ctdb_health_callback, ctdb, "monitor");
+	if (ret != 0) {
+		DEBUG(0,("Unable to launch monitor event script\n"));
+		event_add_timed(ctdb->ev, ctdb->monitor_context, 
+				timeval_current_ofs(ctdb->tunable.monitor_interval, 0), 
+				ctdb_check_health, ctdb);
+	}	
+}
+
+/* stop any monitoring */
+void ctdb_stop_monitoring(struct ctdb_context *ctdb)
+{
+	talloc_free(ctdb->monitor_context);
+	ctdb->monitor_context = talloc_new(ctdb);
+	CTDB_NO_MEMORY_FATAL(ctdb, ctdb->monitor_context);
+}
+
+/*
+  start watching for nodes that might be dead
+ */
+void ctdb_start_monitoring(struct ctdb_context *ctdb)
+{
+	struct timed_event *te;
+
+	ctdb_stop_monitoring(ctdb);
+
+	te = event_add_timed(ctdb->ev, ctdb->monitor_context,
+			     timeval_current_ofs(ctdb->tunable.keepalive_interval, 0), 
+			     ctdb_check_for_dead_nodes, ctdb);
+	CTDB_NO_MEMORY_FATAL(ctdb, te);
+
+	te = event_add_timed(ctdb->ev, ctdb->monitor_context,
+			     timeval_current_ofs(ctdb->tunable.monitor_interval, 0), 
+			     ctdb_check_health, ctdb);
+	CTDB_NO_MEMORY_FATAL(ctdb, te);
+}
+
+
+/*
+  modify flags on a node
+ */
+int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_node_modflags *m = (struct ctdb_node_modflags *)indata.dptr;
+	TDB_DATA data;
+	struct ctdb_node_flag_change c;
+	struct ctdb_node *node = ctdb->nodes[ctdb->vnn];
+	uint32_t old_flags = node->flags;
+
+	node->flags |= m->set;
+	node->flags &= ~m->clear;
+
+	if (node->flags == old_flags) {
+		/* no change */
+		return 0;
+	}
+
+	DEBUG(0, ("Control modflags on node %u - flags now 0x%x\n", ctdb->vnn, node->flags));
+
+	/* if we have been banned, go into recovery mode */
+	c.vnn = ctdb->vnn;
+	c.flags = node->flags;
+
+	data.dptr = (uint8_t *)&c;
+	data.dsize = sizeof(c);
+
+	/* tell the other nodes that something has changed */
+	ctdb_daemon_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
+				 CTDB_SRVID_NODE_FLAGS_CHANGED, data);
+
+	if ((node->flags & NODE_FLAGS_BANNED) && !(old_flags & NODE_FLAGS_BANNED)) {
+		/* make sure we are frozen */
+		DEBUG(0,("This node has been banned - forcing freeze and recovery\n"));
+		ctdb_start_freeze(ctdb);
+		ctdb_release_all_ips(ctdb);
+		ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
+	}
+	
+	return 0;
+}
diff --git a/source4/cluster/ctdb/server/ctdb_recover.c b/source4/cluster/ctdb/server/ctdb_recover.c
new file mode 100644
index 0000000000..82338d48ce
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_recover.c
@@ -0,0 +1,680 @@
+/* 
+   ctdb recovery code
+
+   Copyright (C) Andrew Tridgell  2007
+   Copyright (C) Ronnie Sahlberg  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+#include "lib/util/dlinklist.h"
+#include "db_wrap.h"
+
+/*
+  lock all databases - mark only
+ */
+static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb)
+{
+	struct ctdb_db_context *ctdb_db;
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("Attempt to mark all databases locked when not frozen\n"));
+		return -1;
+	}
+	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+		if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/*
+  lock all databases - unmark only
+ */
+static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb)
+{
+	struct ctdb_db_context *ctdb_db;
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("Attempt to unmark all databases locked when not frozen\n"));
+		return -1;
+	}
+	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+		if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+
+int 
+ctdb_control_getvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
+{
+	CHECK_CONTROL_DATA_SIZE(0);
+	struct ctdb_vnn_map_wire *map;
+	size_t len;
+
+	len = offsetof(struct ctdb_vnn_map_wire, map) + sizeof(uint32_t)*ctdb->vnn_map->size;
+	map = talloc_size(outdata, len);
+	CTDB_NO_MEMORY_VOID(ctdb, map);
+
+	map->generation = ctdb->vnn_map->generation;
+	map->size = ctdb->vnn_map->size;
+	memcpy(map->map, ctdb->vnn_map->map, sizeof(uint32_t)*map->size);
+
+	outdata->dsize = len;
+	outdata->dptr  = (uint8_t *)map;
+
+	return 0;
+}
+
+int 
+ctdb_control_setvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
+{
+	struct ctdb_vnn_map_wire *map = (struct ctdb_vnn_map_wire *)indata.dptr;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("Attempt to set vnnmap when not frozen\n"));
+		return -1;
+	}
+
+	talloc_free(ctdb->vnn_map);
+
+	ctdb->vnn_map = talloc(ctdb, struct ctdb_vnn_map);
+	CTDB_NO_MEMORY(ctdb, ctdb->vnn_map);
+
+	ctdb->vnn_map->generation = map->generation;
+	ctdb->vnn_map->size       = map->size;
+	ctdb->vnn_map->map = talloc_array(ctdb->vnn_map, uint32_t, map->size);
+	CTDB_NO_MEMORY(ctdb, ctdb->vnn_map->map);
+
+	memcpy(ctdb->vnn_map->map, map->map, sizeof(uint32_t)*map->size);
+
+	return 0;
+}
+
+int 
+ctdb_control_getdbmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
+{
+	uint32_t i, len;
+	struct ctdb_db_context *ctdb_db;
+	struct ctdb_dbid_map *dbid_map;
+
+	CHECK_CONTROL_DATA_SIZE(0);
+
+	len = 0;
+	for(ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next){
+		len++;
+	}
+
+
+	outdata->dsize = offsetof(struct ctdb_dbid_map, dbids) + 4*len;
+	outdata->dptr  = (unsigned char *)talloc_zero_size(outdata, outdata->dsize);
+	if (!outdata->dptr) {
+		DEBUG(0, (__location__ " Failed to allocate dbmap array\n"));
+		exit(1);
+	}
+
+	dbid_map = (struct ctdb_dbid_map *)outdata->dptr;
+	dbid_map->num = len;
+	for(i=0,ctdb_db=ctdb->db_list;ctdb_db;i++,ctdb_db=ctdb_db->next){
+		dbid_map->dbids[i] = ctdb_db->db_id;
+	}
+
+	return 0;
+}
+
+int 
+ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
+{
+	uint32_t i, num_nodes;
+	struct ctdb_node_map *node_map;
+
+	CHECK_CONTROL_DATA_SIZE(0);
+
+	num_nodes = ctdb->num_nodes;
+
+	outdata->dsize = offsetof(struct ctdb_node_map, nodes) + num_nodes*sizeof(struct ctdb_node_and_flags);
+	outdata->dptr  = (unsigned char *)talloc_zero_size(outdata, outdata->dsize);
+	if (!outdata->dptr) {
+		DEBUG(0, (__location__ " Failed to allocate nodemap array\n"));
+		exit(1);
+	}
+
+	node_map = (struct ctdb_node_map *)outdata->dptr;
+	node_map->num = num_nodes;
+	for (i=0; i<num_nodes; i++) {
+		inet_aton(ctdb->nodes[i]->address.address, &node_map->nodes[i].sin.sin_addr);
+		node_map->nodes[i].vnn   = ctdb->nodes[i]->vnn;
+		node_map->nodes[i].flags = ctdb->nodes[i]->flags;
+	}
+
+	return 0;
+}
+
+struct getkeys_params {
+	struct ctdb_context *ctdb;
+	uint32_t lmaster;
+	uint32_t rec_count;
+	struct getkeys_rec {
+		TDB_DATA key;
+		TDB_DATA data;
+	} *recs;
+};
+
+static int traverse_getkeys(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	struct getkeys_params *params = (struct getkeys_params *)p;
+	uint32_t lmaster;
+
+	lmaster = ctdb_lmaster(params->ctdb, &key);
+
+	/* only include this record if the lmaster matches or if
+	   the wildcard lmaster (-1) was specified.
+	*/
+	if ((params->lmaster != CTDB_LMASTER_ANY) && (params->lmaster != lmaster)) {
+		return 0;
+	}
+
+	params->recs = talloc_realloc(NULL, params->recs, struct getkeys_rec, params->rec_count+1);
+	key.dptr = talloc_memdup(params->recs, key.dptr, key.dsize);
+	data.dptr = talloc_memdup(params->recs, data.dptr, data.dsize);
+	params->recs[params->rec_count].key = key;
+	params->recs[params->rec_count].data = data;
+	params->rec_count++;
+
+	return 0;
+}
+
+/*
+  pul a bunch of records from a ltdb, filtering by lmaster
+ */
+int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
+{
+	struct ctdb_control_pulldb *pull;
+	struct ctdb_db_context *ctdb_db;
+	struct getkeys_params params;
+	struct ctdb_control_pulldb_reply *reply;
+	int i;
+	size_t len = 0;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("rejecting ctdb_control_pull_db when not frozen\n"));
+		return -1;
+	}
+
+	pull = (struct ctdb_control_pulldb *)indata.dptr;
+	
+	ctdb_db = find_ctdb_db(ctdb, pull->db_id);
+	if (!ctdb_db) {
+		DEBUG(0,(__location__ " Unknown db\n"));
+		return -1;
+	}
+
+	params.ctdb = ctdb;
+	params.lmaster = pull->lmaster;
+
+	params.rec_count = 0;
+	params.recs = talloc_array(outdata, struct getkeys_rec, 0);
+	CTDB_NO_MEMORY(ctdb, params.recs);
+
+	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+		DEBUG(0,(__location__ " Failed to get lock on entired db - failing\n"));
+		return -1;
+	}
+
+	tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_getkeys, &params);
+
+	ctdb_lock_all_databases_unmark(ctdb);
+
+	reply = talloc(outdata, struct ctdb_control_pulldb_reply);
+	CTDB_NO_MEMORY(ctdb, reply);
+
+	reply->db_id = pull->db_id;
+	reply->count = params.rec_count;
+
+	len = offsetof(struct ctdb_control_pulldb_reply, data);
+
+	for (i=0;i<reply->count;i++) {
+		struct ctdb_rec_data *rec;
+		rec = ctdb_marshall_record(outdata, 0, params.recs[i].key, params.recs[i].data);
+		reply = talloc_realloc_size(outdata, reply, rec->length + len);
+		memcpy(len+(uint8_t *)reply, rec, rec->length);
+		len += rec->length;
+		talloc_free(rec);
+	}
+
+	talloc_free(params.recs);
+
+	outdata->dptr = (uint8_t *)reply;
+	outdata->dsize = len;
+
+	return 0;
+}
+
+/*
+  push a bunch of records into a ltdb, filtering by rsn
+ */
+int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_pulldb_reply *reply = (struct ctdb_control_pulldb_reply *)indata.dptr;
+	struct ctdb_db_context *ctdb_db;
+	int i, ret;
+	struct ctdb_rec_data *rec;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("rejecting ctdb_control_push_db when not frozen\n"));
+		return -1;
+	}
+
+	if (indata.dsize < offsetof(struct ctdb_control_pulldb_reply, data)) {
+		DEBUG(0,(__location__ " invalid data in pulldb reply\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, reply->db_id);
+	if (!ctdb_db) {
+		DEBUG(0,(__location__ " Unknown db 0x%08x\n", reply->db_id));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+		DEBUG(0,(__location__ " Failed to get lock on entired db - failing\n"));
+		return -1;
+	}
+
+	rec = (struct ctdb_rec_data *)&reply->data[0];
+
+	DEBUG(3,("starting push of %u records for dbid 0x%x\n",
+		 reply->count, reply->db_id));
+
+	for (i=0;i<reply->count;i++) {
+		TDB_DATA key, data;
+		struct ctdb_ltdb_header *hdr, header;
+
+		key.dptr = &rec->data[0];
+		key.dsize = rec->keylen;
+		data.dptr = &rec->data[key.dsize];
+		data.dsize = rec->datalen;
+
+		if (data.dsize < sizeof(struct ctdb_ltdb_header)) {
+			DEBUG(0,(__location__ " bad ltdb record\n"));
+			goto failed;
+		}
+		hdr = (struct ctdb_ltdb_header *)data.dptr;
+		data.dptr += sizeof(*hdr);
+		data.dsize -= sizeof(*hdr);
+
+		ret = ctdb_ltdb_fetch(ctdb_db, key, &header, NULL, NULL);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to fetch record\n"));
+			goto failed;
+		}
+		/* The check for dmaster gives priority to the dmaster
+		   if the rsn values are equal */
+		if (header.rsn < hdr->rsn ||
+		    (header.dmaster != ctdb->vnn && header.rsn == hdr->rsn)) {
+			ret = ctdb_ltdb_store(ctdb_db, key, hdr, data);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to store record\n"));
+				goto failed;
+			}
+		}
+
+		rec = (struct ctdb_rec_data *)(rec->length + (uint8_t *)rec);
+	}	    
+
+	DEBUG(3,("finished push of %u records for dbid 0x%x\n",
+		 reply->count, reply->db_id));
+
+	ctdb_lock_all_databases_unmark(ctdb);
+	return 0;
+
+failed:
+	ctdb_lock_all_databases_unmark(ctdb);
+	return -1;
+}
+
+
+static int traverse_setdmaster(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	uint32_t *dmaster = (uint32_t *)p;
+	struct ctdb_ltdb_header *header = (struct ctdb_ltdb_header *)data.dptr;
+	int ret;
+
+	header->dmaster = *dmaster;
+
+	ret = tdb_store(tdb, key, data, TDB_REPLACE);
+	if (ret) {
+		DEBUG(0,(__location__ " failed to write tdb data back  ret:%d\n",ret));
+		return ret;
+	}
+	return 0;
+}
+
+int32_t ctdb_control_set_dmaster(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_set_dmaster *p = (struct ctdb_control_set_dmaster *)indata.dptr;
+	struct ctdb_db_context *ctdb_db;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("rejecting ctdb_control_set_dmaster when not frozen\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, p->db_id);
+	if (!ctdb_db) {
+		DEBUG(0,(__location__ " Unknown db 0x%08x\n", p->db_id));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+		DEBUG(0,(__location__ " Failed to get lock on entired db - failing\n"));
+		return -1;
+	}
+
+	tdb_traverse(ctdb_db->ltdb->tdb, traverse_setdmaster, &p->dmaster);
+
+	ctdb_lock_all_databases_unmark(ctdb);
+	
+	return 0;
+}
+
+struct ctdb_set_recmode_state {
+	struct ctdb_req_control *c;
+	uint32_t recmode;
+};
+
+/*
+  called when the 'recovered' event script has finished
+ */
+static void ctdb_recovered_callback(struct ctdb_context *ctdb, int status, void *p)
+{
+	struct ctdb_set_recmode_state *state = talloc_get_type(p, struct ctdb_set_recmode_state);
+
+	ctdb_start_monitoring(ctdb);
+
+	if (status == 0) {
+		ctdb->recovery_mode = state->recmode;
+	} else {
+		DEBUG(0,(__location__ " recovered event script failed (status %d)\n", status));
+	}
+
+	ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
+	talloc_free(state);
+}
+
+/*
+  set the recovery mode
+ */
+int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, bool *async_reply,
+				 const char **errormsg)
+{
+	uint32_t recmode = *(uint32_t *)indata.dptr;
+	int ret;
+	struct ctdb_set_recmode_state *state;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("Attempt to change recovery mode to %u when not frozen\n", 
+			 recmode));
+		(*errormsg) = "Cannot change recovery mode while not frozen";
+		return -1;
+	}
+
+	if (recmode != CTDB_RECOVERY_NORMAL ||
+	    ctdb->recovery_mode != CTDB_RECOVERY_ACTIVE) {
+		ctdb->recovery_mode = recmode;
+		return 0;
+	}
+
+	/* some special handling when ending recovery mode */
+	state = talloc(ctdb, struct ctdb_set_recmode_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	/* we should not be able to get the lock on the nodes list, as it should be
+	   held by the recovery master */
+	if (ctdb_recovery_lock(ctdb, false)) {
+		DEBUG(0,("ERROR: recovery lock file %s not locked when recovering!\n",
+			 ctdb->recovery_lock_file));
+		return -1;
+	}	
+
+	state->c = talloc_steal(state, c);
+	state->recmode = recmode;
+	
+	ctdb_stop_monitoring(ctdb);
+
+	/* call the events script to tell all subsystems that we have recovered */
+	ret = ctdb_event_script_callback(ctdb, 
+					 timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+					 state, 
+					 ctdb_recovered_callback, 
+					 state, "recovered");
+	if (ret != 0) {
+		return ret;
+	}
+	*async_reply = true;
+
+	return 0;
+}
+
+/*
+  callback for ctdb_control_max_rsn
+ */
+static int traverse_max_rsn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	struct ctdb_ltdb_header *h = (struct ctdb_ltdb_header *)data.dptr;
+	uint64_t *max_rsn = (uint64_t *)p;
+
+	if (data.dsize >= sizeof(*h)) {
+		(*max_rsn) = MAX(*max_rsn, h->rsn);
+	}
+	return 0;
+}
+
+/*
+  get max rsn across an entire db
+ */
+int32_t ctdb_control_max_rsn(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
+{
+	struct ctdb_db_context *ctdb_db;
+	uint32_t db_id = *(uint32_t *)indata.dptr;
+	uint64_t max_rsn = 0;
+	int ret;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("rejecting ctdb_control_max_rsn when not frozen\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, db_id);
+	if (!ctdb_db) {
+		DEBUG(0,(__location__ " Unknown db\n"));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+		DEBUG(0,(__location__ " Failed to get lock on entired db - failing\n"));
+		return -1;
+	}
+
+	ret = tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_max_rsn, &max_rsn);
+	if (ret < 0) {
+		DEBUG(0,(__location__ " traverse failed in ctdb_control_max_rsn\n"));
+		return -1;
+	}
+
+	ctdb_lock_all_databases_unmark(ctdb);
+
+	outdata->dptr = (uint8_t *)talloc(outdata, uint64_t);
+	if (!outdata->dptr) {
+		return -1;
+	}
+	(*(uint64_t *)outdata->dptr) = max_rsn;
+	outdata->dsize = sizeof(uint64_t);
+
+	return 0;
+}
+
+
+/*
+  callback for ctdb_control_set_rsn_nonempty
+ */
+static int traverse_set_rsn_nonempty(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	struct ctdb_ltdb_header *h = (struct ctdb_ltdb_header *)data.dptr;
+	uint64_t *rsn = (uint64_t *)p;
+
+	if (data.dsize > sizeof(*h)) {
+		h->rsn = *rsn;
+		if (tdb_store(tdb, key, data, TDB_REPLACE) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/*
+  set rsn for all non-empty records in a database to a given rsn
+ */
+int32_t ctdb_control_set_rsn_nonempty(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
+{
+	struct ctdb_control_set_rsn_nonempty *p = (struct ctdb_control_set_rsn_nonempty *)indata.dptr;
+	struct ctdb_db_context *ctdb_db;
+	int ret;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("rejecting ctdb_control_set_rsn_nonempty when not frozen\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, p->db_id);
+	if (!ctdb_db) {
+		DEBUG(0,(__location__ " Unknown db\n"));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+		DEBUG(0,(__location__ " Failed to get lock on entired db - failing\n"));
+		return -1;
+	}
+
+	ret = tdb_traverse(ctdb_db->ltdb->tdb, traverse_set_rsn_nonempty, &p->rsn);
+	if (ret < 0) {
+		DEBUG(0,(__location__ " traverse failed in ctdb_control_set_rsn_nonempty\n"));
+		return -1;
+	}
+
+	ctdb_lock_all_databases_unmark(ctdb);
+
+	return 0;
+}
+
+
+/*
+  callback for ctdb_control_delete_low_rsn
+ */
+static int traverse_delete_low_rsn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	struct ctdb_ltdb_header *h = (struct ctdb_ltdb_header *)data.dptr;
+	uint64_t *rsn = (uint64_t *)p;
+
+	if (data.dsize < sizeof(*h) || h->rsn < *rsn) {
+		if (tdb_delete(tdb, key) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/*
+  delete any records with a rsn < the given rsn
+ */
+int32_t ctdb_control_delete_low_rsn(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata)
+{
+	struct ctdb_control_delete_low_rsn *p = (struct ctdb_control_delete_low_rsn *)indata.dptr;
+	struct ctdb_db_context *ctdb_db;
+	int ret;
+
+	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+		DEBUG(0,("rejecting ctdb_control_delete_low_rsn when not frozen\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, p->db_id);
+	if (!ctdb_db) {
+		DEBUG(0,(__location__ " Unknown db\n"));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+		DEBUG(0,(__location__ " Failed to get lock on entired db - failing\n"));
+		return -1;
+	}
+
+	ret = tdb_traverse(ctdb_db->ltdb->tdb, traverse_delete_low_rsn, &p->rsn);
+	if (ret < 0) {
+		DEBUG(0,(__location__ " traverse failed in ctdb_control_delete_low_rsn\n"));
+		return -1;
+	}
+
+	ctdb_lock_all_databases_unmark(ctdb);
+
+	return 0;
+}
+
+
+/*
+  try and get the recovery lock in shared storage - should only work
+  on the recovery master recovery daemon. Anywhere else is a bug
+ */
+bool ctdb_recovery_lock(struct ctdb_context *ctdb, bool keep)
+{
+	struct flock lock;
+
+	if (ctdb->recovery_lock_fd != -1) {
+		close(ctdb->recovery_lock_fd);
+	}
+	ctdb->recovery_lock_fd = open(ctdb->recovery_lock_file, O_RDWR|O_CREAT, 0600);
+	if (ctdb->recovery_lock_fd == -1) {
+		DEBUG(0,("Unable to open %s - (%s)\n", 
+			 ctdb->recovery_lock_file, strerror(errno)));
+		return false;
+	}
+
+	lock.l_type = F_WRLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start = 0;
+	lock.l_len = 1;
+	lock.l_pid = 0;
+
+	if (fcntl(ctdb->recovery_lock_fd, F_SETLK, &lock) != 0) {
+		return false;
+	}
+
+	if (!keep) {
+		close(ctdb->recovery_lock_fd);
+		ctdb->recovery_lock_fd = -1;
+	}
+
+	return true;
+}
diff --git a/source4/cluster/ctdb/server/ctdb_recoverd.c b/source4/cluster/ctdb/server/ctdb_recoverd.c
new file mode 100644
index 0000000000..5cb985521d
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_recoverd.c
@@ -0,0 +1,1511 @@
+/* 
+   ctdb recovery daemon
+
+   Copyright (C) Ronnie Sahlberg  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "system/time.h"
+#include "popt.h"
+#include "cmdline.h"
+#include "../include/ctdb.h"
+#include "../include/ctdb_private.h"
+
+
+struct ban_state {
+	struct ctdb_recoverd *rec;
+	uint32_t banned_node;
+};
+
+/*
+  private state of recovery daemon
+ */
+struct ctdb_recoverd {
+	struct ctdb_context *ctdb;
+	uint32_t last_culprit;
+	uint32_t culprit_counter;
+	struct timeval first_recover_time;
+	struct ban_state **banned_nodes;
+	struct timeval priority_time;
+};
+
+#define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
+#define MONITOR_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_interval, 0)
+
+/*
+  unban a node
+ */
+static void ctdb_unban_node(struct ctdb_recoverd *rec, uint32_t vnn)
+{
+	struct ctdb_context *ctdb = rec->ctdb;
+
+	if (!ctdb_validate_vnn(ctdb, vnn)) {
+		DEBUG(0,("Bad vnn %u in ctdb_ban_node\n", vnn));
+		return;
+	}
+
+	if (rec->banned_nodes[vnn] == NULL) {
+		return;
+	}
+
+	ctdb_ctrl_modflags(ctdb, CONTROL_TIMEOUT(), vnn, 0, NODE_FLAGS_BANNED);
+
+	talloc_free(rec->banned_nodes[vnn]);
+	rec->banned_nodes[vnn] = NULL;
+}
+
+
+/*
+  called when a ban has timed out
+ */
+static void ctdb_ban_timeout(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
+{
+	struct ban_state *state = talloc_get_type(p, struct ban_state);
+	struct ctdb_recoverd *rec = state->rec;
+	uint32_t vnn = state->banned_node;
+
+	DEBUG(0,("Node %u is now unbanned\n", vnn));
+	ctdb_unban_node(rec, vnn);
+}
+
+/*
+  ban a node for a period of time
+ */
+static void ctdb_ban_node(struct ctdb_recoverd *rec, uint32_t vnn, uint32_t ban_time)
+{
+	struct ctdb_context *ctdb = rec->ctdb;
+
+	if (!ctdb_validate_vnn(ctdb, vnn)) {
+		DEBUG(0,("Bad vnn %u in ctdb_ban_node\n", vnn));
+		return;
+	}
+
+	if (vnn == ctdb->vnn) {
+		DEBUG(0,("self ban - lowering our election priority\n"));
+		/* banning ourselves - lower our election priority */
+		rec->priority_time = timeval_current();
+	}
+
+	ctdb_ctrl_modflags(ctdb, CONTROL_TIMEOUT(), vnn, NODE_FLAGS_BANNED, 0);
+
+	rec->banned_nodes[vnn] = talloc(rec, struct ban_state);
+	CTDB_NO_MEMORY_FATAL(ctdb, rec->banned_nodes[vnn]);
+
+	rec->banned_nodes[vnn]->rec = rec;
+	rec->banned_nodes[vnn]->banned_node = vnn;
+
+	if (ban_time != 0) {
+		event_add_timed(ctdb->ev, rec->banned_nodes[vnn], 
+				timeval_current_ofs(ban_time, 0),
+				ctdb_ban_timeout, rec->banned_nodes[vnn]);
+	}
+}
+
+
+/*
+  change recovery mode on all nodes
+ */
+static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, uint32_t rec_mode)
+{
+	int j, ret;
+
+	/* start the freeze process immediately on all nodes */
+	ctdb_control(ctdb, CTDB_BROADCAST_CONNECTED, 0, 
+		     CTDB_CONTROL_FREEZE, CTDB_CTRL_FLAG_NOREPLY, tdb_null, 
+		     NULL, NULL, NULL, NULL, NULL);
+
+	/* set recovery mode to active on all nodes */
+	for (j=0; j<nodemap->num; j++) {
+		/* dont change it for nodes that are unavailable */
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+
+		if (rec_mode == CTDB_RECOVERY_ACTIVE) {
+			ret = ctdb_ctrl_freeze(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to freeze node %u\n", nodemap->nodes[j].vnn));
+				return -1;
+			}
+		}
+
+		ret = ctdb_ctrl_setrecmode(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, rec_mode);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to set recmode on node %u\n", nodemap->nodes[j].vnn));
+			return -1;
+		}
+
+		if (rec_mode == CTDB_RECOVERY_NORMAL) {
+			ret = ctdb_ctrl_thaw(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to thaw node %u\n", nodemap->nodes[j].vnn));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/*
+  change recovery master on all node
+ */
+static int set_recovery_master(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, uint32_t vnn)
+{
+	int j, ret;
+
+	/* set recovery master to vnn on all nodes */
+	for (j=0; j<nodemap->num; j++) {
+		/* dont change it for nodes that are unavailable */
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_setrecmaster(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, vnn);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to set recmaster on node %u\n", nodemap->nodes[j].vnn));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  ensure all other nodes have attached to any databases that we have
+ */
+static int create_missing_remote_databases(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+					   uint32_t vnn, struct ctdb_dbid_map *dbmap, TALLOC_CTX *mem_ctx)
+{
+	int i, j, db, ret;
+	struct ctdb_dbid_map *remote_dbmap;
+
+	/* verify that all other nodes have all our databases */
+	for (j=0; j<nodemap->num; j++) {
+		/* we dont need to ourself ourselves */
+		if (nodemap->nodes[j].vnn == vnn) {
+			continue;
+		}
+		/* dont check nodes that are unavailable */
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_getdbmap(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, 
+					 mem_ctx, &remote_dbmap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to get dbids from node %u\n", vnn));
+			return -1;
+		}
+
+		/* step through all local databases */
+		for (db=0; db<dbmap->num;db++) {
+			const char *name;
+
+
+			for (i=0;i<remote_dbmap->num;i++) {
+				if (dbmap->dbids[db] == remote_dbmap->dbids[i]) {
+					break;
+				}
+			}
+			/* the remote node already have this database */
+			if (i!=remote_dbmap->num) {
+				continue;
+			}
+			/* ok so we need to create this database */
+			ctdb_ctrl_getdbname(ctdb, CONTROL_TIMEOUT(), vnn, dbmap->dbids[db], mem_ctx, &name);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to get dbname from node %u\n", vnn));
+				return -1;
+			}
+			ctdb_ctrl_createdb(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, mem_ctx, name);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to create remote db:%s\n", name));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  ensure we are attached to any databases that anyone else is attached to
+ */
+static int create_missing_local_databases(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+					  uint32_t vnn, struct ctdb_dbid_map **dbmap, TALLOC_CTX *mem_ctx)
+{
+	int i, j, db, ret;
+	struct ctdb_dbid_map *remote_dbmap;
+
+	/* verify that we have all database any other node has */
+	for (j=0; j<nodemap->num; j++) {
+		/* we dont need to ourself ourselves */
+		if (nodemap->nodes[j].vnn == vnn) {
+			continue;
+		}
+		/* dont check nodes that are unavailable */
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_getdbmap(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, 
+					 mem_ctx, &remote_dbmap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to get dbids from node %u\n", vnn));
+			return -1;
+		}
+
+		/* step through all databases on the remote node */
+		for (db=0; db<remote_dbmap->num;db++) {
+			const char *name;
+
+			for (i=0;i<(*dbmap)->num;i++) {
+				if (remote_dbmap->dbids[db] == (*dbmap)->dbids[i]) {
+					break;
+				}
+			}
+			/* we already have this db locally */
+			if (i!=(*dbmap)->num) {
+				continue;
+			}
+			/* ok so we need to create this database and
+			   rebuild dbmap
+			 */
+			ctdb_ctrl_getdbname(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, 
+					    remote_dbmap->dbids[db], mem_ctx, &name);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to get dbname from node %u\n", 
+					  nodemap->nodes[j].vnn));
+				return -1;
+			}
+			ctdb_ctrl_createdb(ctdb, CONTROL_TIMEOUT(), vnn, mem_ctx, name);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to create local db:%s\n", name));
+				return -1;
+			}
+			ret = ctdb_ctrl_getdbmap(ctdb, CONTROL_TIMEOUT(), vnn, mem_ctx, dbmap);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to reread dbmap on node %u\n", vnn));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  pull all the remote database contents into ours
+ */
+static int pull_all_remote_databases(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+				     uint32_t vnn, struct ctdb_dbid_map *dbmap, TALLOC_CTX *mem_ctx)
+{
+	int i, j, ret;
+
+	/* pull all records from all other nodes across onto this node
+	   (this merges based on rsn)
+	*/
+	for (i=0;i<dbmap->num;i++) {
+		for (j=0; j<nodemap->num; j++) {
+			/* we dont need to merge with ourselves */
+			if (nodemap->nodes[j].vnn == vnn) {
+				continue;
+			}
+			/* dont merge from nodes that are unavailable */
+			if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+				continue;
+			}
+			ret = ctdb_ctrl_copydb(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, 
+					       vnn, dbmap->dbids[i], CTDB_LMASTER_ANY, mem_ctx);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to copy db from node %u to node %u\n", 
+					  nodemap->nodes[j].vnn, vnn));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  change the dmaster on all databases to point to us
+ */
+static int update_dmaster_on_all_databases(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+					   uint32_t vnn, struct ctdb_dbid_map *dbmap, TALLOC_CTX *mem_ctx)
+{
+	int i, j, ret;
+
+	/* update dmaster to point to this node for all databases/nodes */
+	for (i=0;i<dbmap->num;i++) {
+		for (j=0; j<nodemap->num; j++) {
+			/* dont repoint nodes that are unavailable */
+			if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+				continue;
+			}
+			ret = ctdb_ctrl_setdmaster(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, ctdb, dbmap->dbids[i], vnn);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to set dmaster for node %u db:0x%08x\n", nodemap->nodes[j].vnn, dbmap->dbids[i]));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  update flags on all active nodes
+ */
+static int update_flags_on_all_nodes(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
+{
+	int i;
+	for (i=0;i<nodemap->num;i++) {
+		struct ctdb_node_flag_change c;
+		TDB_DATA data;
+
+		c.vnn = nodemap->nodes[i].vnn;
+		c.flags = nodemap->nodes[i].flags;
+
+		data.dptr = (uint8_t *)&c;
+		data.dsize = sizeof(c);
+
+		ctdb_send_message(ctdb, CTDB_BROADCAST_CONNECTED,
+				  CTDB_SRVID_NODE_FLAGS_CHANGED, data);
+
+	}
+	return 0;
+}
+
+/*
+  vacuum one database
+ */
+static int vacuum_db(struct ctdb_context *ctdb, uint32_t db_id, struct ctdb_node_map *nodemap)
+{
+	uint64_t max_rsn;
+	int ret, i;
+
+	/* find max rsn on our local node for this db */
+	ret = ctdb_ctrl_get_max_rsn(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, db_id, &max_rsn);
+	if (ret != 0) {
+		return -1;
+	}
+
+	/* set rsn on non-empty records to max_rsn+1 */
+	for (i=0;i<nodemap->num;i++) {
+		if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		ret = ctdb_ctrl_set_rsn_nonempty(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[i].vnn,
+						 db_id, max_rsn+1);
+		if (ret != 0) {
+			DEBUG(0,(__location__ " Failed to set rsn on node %u to %llu\n",
+				 nodemap->nodes[i].vnn, (unsigned long long)max_rsn+1));
+			return -1;
+		}
+	}
+
+	/* delete records with rsn < max_rsn+1 on all nodes */
+	for (i=0;i<nodemap->num;i++) {
+		if (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		ret = ctdb_ctrl_delete_low_rsn(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[i].vnn,
+						 db_id, max_rsn+1);
+		if (ret != 0) {
+			DEBUG(0,(__location__ " Failed to delete records on node %u with rsn below %llu\n",
+				 nodemap->nodes[i].vnn, (unsigned long long)max_rsn+1));
+			return -1;
+		}
+	}
+
+
+	return 0;
+}
+
+
+/*
+  vacuum all attached databases
+ */
+static int vacuum_all_databases(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+				struct ctdb_dbid_map *dbmap)
+{
+	int i;
+
+	/* update dmaster to point to this node for all databases/nodes */
+	for (i=0;i<dbmap->num;i++) {
+		if (vacuum_db(ctdb, dbmap->dbids[i], nodemap) != 0) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+
+/*
+  push out all our database contents to all other nodes
+ */
+static int push_all_local_databases(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+				    uint32_t vnn, struct ctdb_dbid_map *dbmap, TALLOC_CTX *mem_ctx)
+{
+	int i, j, ret;
+
+	/* push all records out to the nodes again */
+	for (i=0;i<dbmap->num;i++) {
+		for (j=0; j<nodemap->num; j++) {
+			/* we dont need to push to ourselves */
+			if (nodemap->nodes[j].vnn == vnn) {
+				continue;
+			}
+			/* dont push to nodes that are unavailable */
+			if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+				continue;
+			}
+			ret = ctdb_ctrl_copydb(ctdb, CONTROL_TIMEOUT(), vnn, nodemap->nodes[j].vnn, 
+					       dbmap->dbids[i], CTDB_LMASTER_ANY, mem_ctx);
+			if (ret != 0) {
+				DEBUG(0, (__location__ " Unable to copy db from node %u to node %u\n", 
+					  vnn, nodemap->nodes[j].vnn));
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  ensure all nodes have the same vnnmap we do
+ */
+static int update_vnnmap_on_all_nodes(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, 
+				      uint32_t vnn, struct ctdb_vnn_map *vnnmap, TALLOC_CTX *mem_ctx)
+{
+	int j, ret;
+
+	/* push the new vnn map out to all the nodes */
+	for (j=0; j<nodemap->num; j++) {
+		/* dont push to nodes that are unavailable */
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_setvnnmap(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, mem_ctx, vnnmap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to set vnnmap for node %u\n", vnn));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  handler for when the admin bans a node
+*/
+static void ban_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			TDB_DATA data, void *private_data)
+{
+	struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+	struct ctdb_ban_info *b = (struct ctdb_ban_info *)data.dptr;
+	uint32_t recmaster;
+	int ret;
+
+	if (data.dsize != sizeof(*b)) {
+		DEBUG(0,("Bad data in ban_handler\n"));
+		return;
+	}
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to find the recmaster\n"));
+		return;
+	}
+
+	if (recmaster != ctdb->vnn) {
+		DEBUG(0,("We are not the recmaster - ignoring ban request\n"));
+		return;
+	}
+
+	DEBUG(0,("Node %u has been banned for %u seconds by the administrator\n", 
+		 b->vnn, b->ban_time));
+	ctdb_ban_node(rec, b->vnn, b->ban_time);
+}
+
+/*
+  handler for when the admin unbans a node
+*/
+static void unban_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			  TDB_DATA data, void *private_data)
+{
+	struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+	uint32_t vnn;
+	int ret;
+	uint32_t recmaster;
+
+	if (data.dsize != sizeof(uint32_t)) {
+		DEBUG(0,("Bad data in unban_handler\n"));
+		return;
+	}
+	vnn = *(uint32_t *)data.dptr;
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to find the recmaster\n"));
+		return;
+	}
+
+	if (recmaster != ctdb->vnn) {
+		DEBUG(0,("We are not the recmaster - ignoring unban request\n"));
+		return;
+	}
+
+	DEBUG(0,("Node %u has been unbanned by the administrator\n", vnn));
+	ctdb_unban_node(rec, vnn);
+}
+
+
+
+/*
+  called when ctdb_wait_timeout should finish
+ */
+static void ctdb_wait_handler(struct event_context *ev, struct timed_event *te, 
+			      struct timeval yt, void *p)
+{
+	uint32_t *timed_out = (uint32_t *)p;
+	(*timed_out) = 1;
+}
+
+/*
+  wait for a given number of seconds
+ */
+static void ctdb_wait_timeout(struct ctdb_context *ctdb, uint32_t secs)
+{
+	uint32_t timed_out = 0;
+	event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(secs, 0), ctdb_wait_handler, &timed_out);
+	while (!timed_out) {
+		event_loop_once(ctdb->ev);
+	}
+}
+
+/*
+  we are the recmaster, and recovery is needed - start a recovery run
+ */
+static int do_recovery(struct ctdb_recoverd *rec, 
+		       TALLOC_CTX *mem_ctx, uint32_t vnn, uint32_t num_active,
+		       struct ctdb_node_map *nodemap, struct ctdb_vnn_map *vnnmap,
+		       uint32_t culprit)
+{
+	struct ctdb_context *ctdb = rec->ctdb;
+	int i, j, ret;
+	uint32_t generation;
+	struct ctdb_dbid_map *dbmap;
+
+	if (rec->last_culprit != culprit ||
+	    timeval_elapsed(&rec->first_recover_time) > ctdb->tunable.recovery_grace_period) {
+		/* either a new node is the culprit, or we've decide to forgive them */
+		rec->last_culprit = culprit;
+		rec->first_recover_time = timeval_current();
+		rec->culprit_counter = 0;
+	}
+	rec->culprit_counter++;
+
+	if (rec->culprit_counter > 2*nodemap->num) {
+		DEBUG(0,("Node %u has caused %u recoveries in %.0f seconds - banning it for %u seconds\n",
+			 culprit, rec->culprit_counter, timeval_elapsed(&rec->first_recover_time),
+			 ctdb->tunable.recovery_ban_period));
+		ctdb_ban_node(rec, culprit, ctdb->tunable.recovery_ban_period);
+	}
+
+	if (!ctdb_recovery_lock(ctdb, true)) {
+		DEBUG(0,("Unable to get recovery lock - aborting recovery\n"));
+		return -1;
+	}
+
+	/* set recovery mode to active on all nodes */
+	ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_ACTIVE);
+	if (ret!=0) {
+		DEBUG(0, (__location__ " Unable to set recovery mode to active on cluster\n"));
+		return -1;
+	}
+
+	DEBUG(0, (__location__ " Recovery initiated due to problem with node %u\n", culprit));
+
+	/* pick a new generation number */
+	generation = random();
+
+	/* change the vnnmap on this node to use the new generation 
+	   number but not on any other nodes.
+	   this guarantees that if we abort the recovery prematurely
+	   for some reason (a node stops responding?)
+	   that we can just return immediately and we will reenter
+	   recovery shortly again.
+	   I.e. we deliberately leave the cluster with an inconsistent
+	   generation id to allow us to abort recovery at any stage and
+	   just restart it from scratch.
+	 */
+	vnnmap->generation = generation;
+	ret = ctdb_ctrl_setvnnmap(ctdb, CONTROL_TIMEOUT(), vnn, mem_ctx, vnnmap);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to set vnnmap for node %u\n", vnn));
+		return -1;
+	}
+
+	/* get a list of all databases */
+	ret = ctdb_ctrl_getdbmap(ctdb, CONTROL_TIMEOUT(), vnn, mem_ctx, &dbmap);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to get dbids from node :%u\n", vnn));
+		return -1;
+	}
+
+
+
+	/* verify that all other nodes have all our databases */
+	ret = create_missing_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to create missing remote databases\n"));
+		return -1;
+	}
+
+	/* verify that we have all the databases any other node has */
+	ret = create_missing_local_databases(ctdb, nodemap, vnn, &dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to create missing local databases\n"));
+		return -1;
+	}
+
+
+
+	/* verify that all other nodes have all our databases */
+	ret = create_missing_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to create missing remote databases\n"));
+		return -1;
+	}
+
+
+	DEBUG(1, (__location__ " Recovery - created remote databases\n"));
+
+	/* pull all remote databases onto the local node */
+	ret = pull_all_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to pull remote databases\n"));
+		return -1;
+	}
+
+	DEBUG(1, (__location__ " Recovery - pulled remote databases\n"));
+
+	/* push all local databases to the remote nodes */
+	ret = push_all_local_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to push local databases\n"));
+		return -1;
+	}
+
+	DEBUG(1, (__location__ " Recovery - pushed remote databases\n"));
+
+	/* build a new vnn map with all the currently active and
+	   unbanned nodes */
+	generation = random();
+	vnnmap = talloc(mem_ctx, struct ctdb_vnn_map);
+	CTDB_NO_MEMORY(ctdb, vnnmap);
+	vnnmap->generation = generation;
+	vnnmap->size = num_active;
+	vnnmap->map = talloc_zero_array(vnnmap, uint32_t, vnnmap->size);
+	for (i=j=0;i<nodemap->num;i++) {
+		if (!(nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE)) {
+			vnnmap->map[j++] = nodemap->nodes[i].vnn;
+		}
+	}
+
+
+
+	/* update to the new vnnmap on all nodes */
+	ret = update_vnnmap_on_all_nodes(ctdb, nodemap, vnn, vnnmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to update vnnmap on all nodes\n"));
+		return -1;
+	}
+
+	DEBUG(1, (__location__ " Recovery - updated vnnmap\n"));
+
+	/* update recmaster to point to us for all nodes */
+	ret = set_recovery_master(ctdb, nodemap, vnn);
+	if (ret!=0) {
+		DEBUG(0, (__location__ " Unable to set recovery master\n"));
+		return -1;
+	}
+
+	DEBUG(1, (__location__ " Recovery - updated recmaster\n"));
+
+	/* repoint all local and remote database records to the local
+	   node as being dmaster
+	 */
+	ret = update_dmaster_on_all_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to update dmaster on all databases\n"));
+		return -1;
+	}
+
+	DEBUG(1, (__location__ " Recovery - updated dmaster on all databases\n"));
+
+	/*
+	  update all nodes to have the same flags that we have
+	 */
+	ret = update_flags_on_all_nodes(ctdb, nodemap);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to update flags on all nodes\n"));
+		return -1;
+	}
+	
+	DEBUG(1, (__location__ " Recovery - updated flags\n"));
+
+	/*
+	  run a vacuum operation on empty records
+	 */
+	ret = vacuum_all_databases(ctdb, nodemap, dbmap);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to vacuum all databases\n"));
+		return -1;
+	}
+
+	DEBUG(1, (__location__ " Recovery - vacuumed all databases\n"));
+
+	/*
+	  if enabled, tell nodes to takeover their public IPs
+	 */
+	if (ctdb->takeover.enabled) {
+		ret = ctdb_takeover_run(ctdb, nodemap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to setup public takeover addresses\n"));
+			return -1;
+		}
+		DEBUG(1, (__location__ " Recovery - done takeover\n"));
+	}
+
+
+	/* disable recovery mode */
+	ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_NORMAL);
+	if (ret!=0) {
+		DEBUG(0, (__location__ " Unable to set recovery mode to normal on cluster\n"));
+		return -1;
+	}
+
+	/* send a message to all clients telling them that the cluster 
+	   has been reconfigured */
+	ctdb_send_message(ctdb, CTDB_BROADCAST_ALL, CTDB_SRVID_RECONFIGURE, tdb_null);
+
+	DEBUG(0, (__location__ " Recovery complete\n"));
+
+	/* We just finished a recovery successfully. 
+	   We now wait for rerecovery_timeout before we allow 
+	   another recovery to take place.
+	*/
+	DEBUG(0, (__location__ " New recoveries supressed for the rerecovery timeout\n"));
+	ctdb_wait_timeout(ctdb, ctdb->tunable.rerecovery_timeout);
+	DEBUG(0, (__location__ " Rerecovery timeout elapsed. Recovery reactivated.\n"));
+
+	return 0;
+}
+
+
+/*
+  elections are won by first checking the number of connected nodes, then
+  the priority time, then the vnn
+ */
+struct election_message {
+	uint32_t num_connected;
+	struct timeval priority_time;
+	uint32_t vnn;
+};
+
+/*
+  form this nodes election data
+ */
+static void ctdb_election_data(struct ctdb_recoverd *rec, struct election_message *em)
+{
+	int ret, i;
+	struct ctdb_node_map *nodemap;
+	struct ctdb_context *ctdb = rec->ctdb;
+
+	ZERO_STRUCTP(em);
+
+	em->vnn = rec->ctdb->vnn;
+	em->priority_time = rec->priority_time;
+
+	ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, rec, &nodemap);
+	if (ret != 0) {
+		return;
+	}
+
+	for (i=0;i<nodemap->num;i++) {
+		if (!(nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED)) {
+			em->num_connected++;
+		}
+	}
+	talloc_free(nodemap);
+}
+
+/*
+  see if the given election data wins
+ */
+static bool ctdb_election_win(struct ctdb_recoverd *rec, struct election_message *em)
+{
+	struct election_message myem;
+	int cmp;
+
+	ctdb_election_data(rec, &myem);
+
+	/* try to use the most connected node */
+	cmp = (int)myem.num_connected - (int)em->num_connected;
+
+	/* then the longest running node */
+	if (cmp == 0) {
+		cmp = timeval_compare(&em->priority_time, &myem.priority_time);
+	}
+
+	if (cmp == 0) {
+		cmp = (int)myem.vnn - (int)em->vnn;
+	}
+
+	return cmp > 0;
+}
+
+/*
+  send out an election request
+ */
+static int send_election_request(struct ctdb_recoverd *rec, TALLOC_CTX *mem_ctx, uint32_t vnn)
+{
+	int ret;
+	TDB_DATA election_data;
+	struct election_message emsg;
+	uint64_t srvid;
+	struct ctdb_context *ctdb = rec->ctdb;
+	
+	srvid = CTDB_SRVID_RECOVERY;
+
+	ctdb_election_data(rec, &emsg);
+
+	election_data.dsize = sizeof(struct election_message);
+	election_data.dptr  = (unsigned char *)&emsg;
+
+
+	/* first we assume we will win the election and set 
+	   recoverymaster to be ourself on the current node
+	 */
+	ret = ctdb_ctrl_setrecmaster(ctdb, CONTROL_TIMEOUT(), vnn, vnn);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " failed to send recmaster election request\n"));
+		return -1;
+	}
+
+
+	/* send an election message to all active nodes */
+	ctdb_send_message(ctdb, CTDB_BROADCAST_ALL, srvid, election_data);
+
+	return 0;
+}
+
+/*
+  this function will unban all nodes in the cluster
+*/
+static void unban_all_nodes(struct ctdb_context *ctdb)
+{
+	int ret, i;
+	struct ctdb_node_map *nodemap;
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+	
+	ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " failed to get nodemap to unban all nodes\n"));
+		return;
+	}
+
+	for (i=0;i<nodemap->num;i++) {
+		if ( (!(nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED))
+		  && (nodemap->nodes[i].flags & NODE_FLAGS_BANNED) ) {
+			ctdb_ctrl_modflags(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[i].vnn, 0, NODE_FLAGS_BANNED);
+		}
+	}
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+  handler for recovery master elections
+*/
+static void election_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			     TDB_DATA data, void *private_data)
+{
+	struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+	int ret;
+	struct election_message *em = (struct election_message *)data.dptr;
+	TALLOC_CTX *mem_ctx;
+
+	mem_ctx = talloc_new(ctdb);
+
+	/* someone called an election. check their election data
+	   and if we disagree and we would rather be the elected node, 
+	   send a new election message to all other nodes
+	 */
+	if (ctdb_election_win(rec, em)) {
+		ret = send_election_request(rec, mem_ctx, ctdb_get_vnn(ctdb));
+		if (ret!=0) {
+			DEBUG(0, (__location__ " failed to initiate recmaster election"));
+		}
+		talloc_free(mem_ctx);
+		/*unban_all_nodes(ctdb);*/
+		return;
+	}
+
+	/* release the recmaster lock */
+	if (em->vnn != ctdb->vnn &&
+	    ctdb->recovery_lock_fd != -1) {
+		close(ctdb->recovery_lock_fd);
+		ctdb->recovery_lock_fd = -1;
+		unban_all_nodes(ctdb);
+	}
+
+	/* ok, let that guy become recmaster then */
+	ret = ctdb_ctrl_setrecmaster(ctdb, CONTROL_TIMEOUT(), ctdb_get_vnn(ctdb), em->vnn);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " failed to send recmaster election request"));
+		talloc_free(mem_ctx);
+		return;
+	}
+
+	/* release any bans */
+	rec->last_culprit = (uint32_t)-1;
+	talloc_free(rec->banned_nodes);
+	rec->banned_nodes = talloc_zero_array(rec, struct ban_state *, ctdb->num_nodes);
+	CTDB_NO_MEMORY_FATAL(ctdb, rec->banned_nodes);
+
+	talloc_free(mem_ctx);
+	return;
+}
+
+
+/*
+  force the start of the election process
+ */
+static void force_election(struct ctdb_recoverd *rec, TALLOC_CTX *mem_ctx, uint32_t vnn, 
+			   struct ctdb_node_map *nodemap)
+{
+	int ret;
+	struct ctdb_context *ctdb = rec->ctdb;
+
+	/* set all nodes to recovery mode to stop all internode traffic */
+	ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_ACTIVE);
+	if (ret!=0) {
+		DEBUG(0, (__location__ " Unable to set recovery mode to active on cluster\n"));
+		return;
+	}
+	
+	ret = send_election_request(rec, mem_ctx, vnn);
+	if (ret!=0) {
+		DEBUG(0, (__location__ " failed to initiate recmaster election"));
+		return;
+	}
+
+	/* wait for a few seconds to collect all responses */
+	ctdb_wait_timeout(ctdb, ctdb->tunable.election_timeout);
+}
+
+
+
+/*
+  handler for when a node changes its flags
+*/
+static void monitor_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			    TDB_DATA data, void *private_data)
+{
+	int ret;
+	struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)data.dptr;
+	struct ctdb_node_map *nodemap=NULL;
+	TALLOC_CTX *tmp_ctx;
+	int i;
+
+	if (data.dsize != sizeof(*c)) {
+		DEBUG(0,(__location__ "Invalid data in ctdb_node_flag_change\n"));
+		return;
+	}
+
+	tmp_ctx = talloc_new(ctdb);
+	CTDB_NO_MEMORY_VOID(ctdb, tmp_ctx);
+
+	ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, tmp_ctx, &nodemap);
+
+	for (i=0;i<nodemap->num;i++) {
+		if (nodemap->nodes[i].vnn == c->vnn) break;
+	}
+
+	if (i == nodemap->num) {
+		DEBUG(0,(__location__ "Flag change for non-existant node %u\n", c->vnn));
+		talloc_free(tmp_ctx);
+		return;
+	}
+
+	/* Dont let messages from remote nodes change the DISCONNECTED flag. 
+	   This flag is handled locally based on whether the local node
+	   can communicate with the node or not.
+	*/
+	c->flags &= ~NODE_FLAGS_DISCONNECTED;
+	if (nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED) {
+		c->flags |= NODE_FLAGS_DISCONNECTED;
+	}
+
+	if (nodemap->nodes[i].flags != c->flags) {
+		DEBUG(0,("Node %u has changed flags - now 0x%x\n", c->vnn, c->flags));
+	}
+
+	nodemap->nodes[i].flags = c->flags;
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), 
+				     CTDB_CURRENT_NODE, &ctdb->recovery_master);
+
+	if (ret == 0) {
+		ret = ctdb_ctrl_getrecmode(ctdb, CONTROL_TIMEOUT(), 
+					   CTDB_CURRENT_NODE, &ctdb->recovery_mode);
+	}
+	
+	if (ret == 0 &&
+	    ctdb->recovery_master == ctdb->vnn &&
+	    ctdb->recovery_mode == CTDB_RECOVERY_NORMAL &&
+	    ctdb->takeover.enabled) {
+		ret = ctdb_takeover_run(ctdb, nodemap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to setup public takeover addresses\n"));
+		}
+	}
+
+	talloc_free(tmp_ctx);
+}
+
+
+
+/*
+  the main monitoring loop
+ */
+static void monitor_cluster(struct ctdb_context *ctdb)
+{
+	uint32_t vnn, num_active, recmode, recmaster;
+	TALLOC_CTX *mem_ctx=NULL;
+	struct ctdb_node_map *nodemap=NULL;
+	struct ctdb_node_map *remote_nodemap=NULL;
+	struct ctdb_vnn_map *vnnmap=NULL;
+	struct ctdb_vnn_map *remote_vnnmap=NULL;
+	int i, j, ret;
+	bool need_takeover_run;
+	struct ctdb_recoverd *rec;
+
+	rec = talloc_zero(ctdb, struct ctdb_recoverd);
+	CTDB_NO_MEMORY_FATAL(ctdb, rec);
+
+	rec->ctdb = ctdb;
+	rec->banned_nodes = talloc_zero_array(rec, struct ban_state *, ctdb->num_nodes);
+	CTDB_NO_MEMORY_FATAL(ctdb, rec->banned_nodes);
+
+	rec->priority_time = timeval_current();
+
+	/* register a message port for recovery elections */
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_RECOVERY, election_handler, rec);
+
+	/* and one for when nodes are disabled/enabled */
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_NODE_FLAGS_CHANGED, monitor_handler, rec);
+
+	/* and one for when nodes are banned */
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_BAN_NODE, ban_handler, rec);
+
+	/* and one for when nodes are unbanned */
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_UNBAN_NODE, unban_handler, rec);
+	
+again:
+	need_takeover_run = false;
+
+	if (mem_ctx) {
+		talloc_free(mem_ctx);
+		mem_ctx = NULL;
+	}
+	mem_ctx = talloc_new(ctdb);
+	if (!mem_ctx) {
+		DEBUG(0,("Failed to create temporary context\n"));
+		exit(-1);
+	}
+
+	/* we only check for recovery once every second */
+	ctdb_wait_timeout(ctdb, ctdb->tunable.recover_interval);
+
+	/* get relevant tunables */
+	ret = ctdb_ctrl_get_all_tunables(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &ctdb->tunable);
+	if (ret != 0) {
+		DEBUG(0,("Failed to get tunables - retrying\n"));
+		goto again;
+	}
+
+	vnn = ctdb_ctrl_getvnn(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE);
+	if (vnn == (uint32_t)-1) {
+		DEBUG(0,("Failed to get local vnn - retrying\n"));
+		goto again;
+	}
+
+	/* get the vnnmap */
+	ret = ctdb_ctrl_getvnnmap(ctdb, CONTROL_TIMEOUT(), vnn, mem_ctx, &vnnmap);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to get vnnmap from node %u\n", vnn));
+		goto again;
+	}
+
+
+	/* get number of nodes */
+	ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), vnn, mem_ctx, &nodemap);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to get nodemap from node %u\n", vnn));
+		goto again;
+	}
+
+
+	/* count how many active nodes there are */
+	num_active = 0;
+	for (i=0; i<nodemap->num; i++) {
+		if (rec->banned_nodes[nodemap->nodes[i].vnn] != NULL) {
+			nodemap->nodes[i].flags |= NODE_FLAGS_BANNED;
+		} else {
+			nodemap->nodes[i].flags &= ~NODE_FLAGS_BANNED;
+		}
+		if (!(nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE)) {
+			num_active++;
+		}
+	}
+
+
+	/* check which node is the recovery master */
+	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), vnn, &recmaster);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Unable to get recmaster from node %u\n", vnn));
+		goto again;
+	}
+
+	if (recmaster == (uint32_t)-1) {
+		DEBUG(0,(__location__ " Initial recovery master set - forcing election\n"));
+		force_election(rec, mem_ctx, vnn, nodemap);
+		goto again;
+	}
+	
+	/* verify that the recmaster node is still active */
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].vnn==recmaster) {
+			break;
+		}
+	}
+
+	if (j == nodemap->num) {
+		DEBUG(0, ("Recmaster node %u not in list. Force reelection\n", recmaster));
+		force_election(rec, mem_ctx, vnn, nodemap);
+		goto again;
+	}
+
+	if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+		DEBUG(0, ("Recmaster node %u no longer available. Force reelection\n", nodemap->nodes[j].vnn));
+		force_election(rec, mem_ctx, vnn, nodemap);
+		goto again;
+	}
+	
+
+	/* if we are not the recmaster then we do not need to check
+	   if recovery is needed
+	 */
+	if (vnn!=recmaster) {
+		goto again;
+	}
+
+
+	/* verify that all active nodes agree that we are the recmaster */
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		if (nodemap->nodes[j].vnn == vnn) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, &recmaster);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to get recmaster from node %u\n", vnn));
+			goto again;
+		}
+
+		if (recmaster!=vnn) {
+			DEBUG(0, ("Node %u does not agree we are the recmaster. Force reelection\n", 
+				  nodemap->nodes[j].vnn));
+			force_election(rec, mem_ctx, vnn, nodemap);
+			goto again;
+		}
+	}
+
+
+	/* verify that all active nodes are in normal mode 
+	   and not in recovery mode 
+	 */
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_getrecmode(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, &recmode);
+		if (ret != 0) {
+			DEBUG(0, ("Unable to get recmode from node %u\n", vnn));
+			goto again;
+		}
+		if (recmode != CTDB_RECOVERY_NORMAL) {
+			DEBUG(0, (__location__ " Node:%u was in recovery mode. Restart recovery process\n", 
+				  nodemap->nodes[j].vnn));
+			do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, nodemap->nodes[j].vnn);
+			goto again;
+		}
+	}
+
+
+	/* get the nodemap for all active remote nodes and verify
+	   they are the same as for this node
+	 */
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		if (nodemap->nodes[j].vnn == vnn) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_getnodemap(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, 
+					   mem_ctx, &remote_nodemap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to get nodemap from remote node %u\n", 
+				  nodemap->nodes[j].vnn));
+			goto again;
+		}
+
+		/* if the nodes disagree on how many nodes there are
+		   then this is a good reason to try recovery
+		 */
+		if (remote_nodemap->num != nodemap->num) {
+			DEBUG(0, (__location__ " Remote node:%u has different node count. %u vs %u of the local node\n",
+				  nodemap->nodes[j].vnn, remote_nodemap->num, nodemap->num));
+			do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, nodemap->nodes[j].vnn);
+			goto again;
+		}
+
+		/* if the nodes disagree on which nodes exist and are
+		   active, then that is also a good reason to do recovery
+		 */
+		for (i=0;i<nodemap->num;i++) {
+			if (remote_nodemap->nodes[i].vnn != nodemap->nodes[i].vnn) {
+				DEBUG(0, (__location__ " Remote node:%u has different nodemap vnn for %d (%u vs %u).\n", 
+					  nodemap->nodes[j].vnn, i, 
+					  remote_nodemap->nodes[i].vnn, nodemap->nodes[i].vnn));
+				do_recovery(rec, mem_ctx, vnn, num_active, nodemap, 
+					    vnnmap, nodemap->nodes[j].vnn);
+				goto again;
+			}
+			if ((remote_nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE) != 
+			    (nodemap->nodes[i].flags & NODE_FLAGS_INACTIVE)) {
+				DEBUG(0, (__location__ " Remote node:%u has different nodemap flag for %d (0x%x vs 0x%x)\n", 
+					  nodemap->nodes[j].vnn, i,
+					  remote_nodemap->nodes[i].flags, nodemap->nodes[i].flags));
+				do_recovery(rec, mem_ctx, vnn, num_active, nodemap, 
+					    vnnmap, nodemap->nodes[j].vnn);
+				goto again;
+			}
+		}
+
+		/* update our nodemap flags according to the other
+		   server - this gets the NODE_FLAGS_DISABLED
+		   flag. Note that the remote node is authoritative
+		   for its flags (except CONNECTED, which we know
+		   matches in this code) */
+		if (nodemap->nodes[j].flags != remote_nodemap->nodes[j].flags) {
+			nodemap->nodes[j].flags = remote_nodemap->nodes[j].flags;
+			need_takeover_run = true;
+		}
+	}
+
+
+	/* there better be the same number of lmasters in the vnn map
+	   as there are active nodes or we will have to do a recovery
+	 */
+	if (vnnmap->size != num_active) {
+		DEBUG(0, (__location__ " The vnnmap count is different from the number of active nodes. %u vs %u\n", 
+			  vnnmap->size, num_active));
+		do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, ctdb->vnn);
+		goto again;
+	}
+
+	/* verify that all active nodes in the nodemap also exist in 
+	   the vnnmap.
+	 */
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		if (nodemap->nodes[j].vnn == vnn) {
+			continue;
+		}
+
+		for (i=0; i<vnnmap->size; i++) {
+			if (vnnmap->map[i] == nodemap->nodes[j].vnn) {
+				break;
+			}
+		}
+		if (i == vnnmap->size) {
+			DEBUG(0, (__location__ " Node %u is active in the nodemap but did not exist in the vnnmap\n", 
+				  nodemap->nodes[j].vnn));
+			do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, nodemap->nodes[j].vnn);
+			goto again;
+		}
+	}
+
+	
+	/* verify that all other nodes have the same vnnmap
+	   and are from the same generation
+	 */
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		if (nodemap->nodes[j].vnn == vnn) {
+			continue;
+		}
+
+		ret = ctdb_ctrl_getvnnmap(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, 
+					  mem_ctx, &remote_vnnmap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to get vnnmap from remote node %u\n", 
+				  nodemap->nodes[j].vnn));
+			goto again;
+		}
+
+		/* verify the vnnmap generation is the same */
+		if (vnnmap->generation != remote_vnnmap->generation) {
+			DEBUG(0, (__location__ " Remote node %u has different generation of vnnmap. %u vs %u (ours)\n", 
+				  nodemap->nodes[j].vnn, remote_vnnmap->generation, vnnmap->generation));
+			do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, nodemap->nodes[j].vnn);
+			goto again;
+		}
+
+		/* verify the vnnmap size is the same */
+		if (vnnmap->size != remote_vnnmap->size) {
+			DEBUG(0, (__location__ " Remote node %u has different size of vnnmap. %u vs %u (ours)\n", 
+				  nodemap->nodes[j].vnn, remote_vnnmap->size, vnnmap->size));
+			do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, nodemap->nodes[j].vnn);
+			goto again;
+		}
+
+		/* verify the vnnmap is the same */
+		for (i=0;i<vnnmap->size;i++) {
+			if (remote_vnnmap->map[i] != vnnmap->map[i]) {
+				DEBUG(0, (__location__ " Remote node %u has different vnnmap.\n", 
+					  nodemap->nodes[j].vnn));
+				do_recovery(rec, mem_ctx, vnn, num_active, nodemap, 
+					    vnnmap, nodemap->nodes[j].vnn);
+				goto again;
+			}
+		}
+	}
+
+	/* we might need to change who has what IP assigned */
+	if (need_takeover_run && ctdb->takeover.enabled) {
+		ret = ctdb_takeover_run(ctdb, nodemap);
+		if (ret != 0) {
+			DEBUG(0, (__location__ " Unable to setup public takeover addresses\n"));
+		}
+	}
+
+	goto again;
+
+}
+
+/*
+  event handler for when the main ctdbd dies
+ */
+static void ctdb_recoverd_parent(struct event_context *ev, struct fd_event *fde, 
+				 uint16_t flags, void *private_data)
+{
+	DEBUG(0,("recovery daemon parent died - exiting\n"));
+	_exit(1);
+}
+
+
+
+/*
+  startup the recovery daemon as a child of the main ctdb daemon
+ */
+int ctdb_start_recoverd(struct ctdb_context *ctdb)
+{
+	int ret;
+	int fd[2];
+	pid_t child;
+
+	if (pipe(fd) != 0) {
+		return -1;
+	}
+
+	child = fork();
+	if (child == -1) {
+		return -1;
+	}
+	
+	if (child != 0) {
+		close(fd[0]);
+		return 0;
+	}
+
+	close(fd[1]);
+
+	/* shutdown the transport */
+	ctdb->methods->shutdown(ctdb);
+
+	/* get a new event context */
+	talloc_free(ctdb->ev);
+	ctdb->ev = event_context_init(ctdb);
+
+	event_add_fd(ctdb->ev, ctdb, fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+		     ctdb_recoverd_parent, &fd[0]);	
+
+	close(ctdb->daemon.sd);
+	ctdb->daemon.sd = -1;
+
+	srandom(getpid() ^ time(NULL));
+
+	/* initialise ctdb */
+	ret = ctdb_socket_connect(ctdb);
+	if (ret != 0) {
+		DEBUG(0, (__location__ " Failed to init ctdb\n"));
+		exit(1);
+	}
+
+	monitor_cluster(ctdb);
+
+	DEBUG(0,("ERROR: ctdb_recoverd finished!?\n"));
+	return -1;
+}
diff --git a/source4/cluster/ctdb/server/ctdb_server.c b/source4/cluster/ctdb/server/ctdb_server.c
new file mode 100644
index 0000000000..1480127327
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_server.c
@@ -0,0 +1,469 @@
+/* 
+   ctdb main protocol code
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/tdb/include/tdb.h"
+#include "lib/events/events.h"
+#include "lib/util/dlinklist.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "../include/ctdb_private.h"
+
+/*
+  choose the transport we will use
+*/
+int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
+{
+	ctdb->transport = talloc_strdup(ctdb, transport);
+	return 0;
+}
+
+/*
+  choose the recovery lock file
+*/
+int ctdb_set_recovery_lock_file(struct ctdb_context *ctdb, const char *file)
+{
+	ctdb->recovery_lock_file = talloc_strdup(ctdb, file);
+	return 0;
+}
+
+/*
+  choose the logfile location
+*/
+int ctdb_set_logfile(struct ctdb_context *ctdb, const char *logfile)
+{
+	ctdb->logfile = talloc_strdup(ctdb, logfile);
+	if (ctdb->logfile != NULL && strcmp(logfile, "-") != 0) {
+		int fd;
+		fd = open(ctdb->logfile, O_WRONLY|O_APPEND|O_CREAT, 0666);
+		if (fd == -1) {
+			printf("Failed to open logfile %s\n", ctdb->logfile);
+			abort();
+		}
+		close(1);
+		close(2);
+		if (fd != 1) {
+			dup2(fd, 1);
+			close(fd);
+		}
+		/* also catch stderr of subcommands to the log file */
+		dup2(1, 2);
+	}
+	return 0;
+}
+
+
+/*
+  set the directory for the local databases
+*/
+int ctdb_set_tdb_dir(struct ctdb_context *ctdb, const char *dir)
+{
+	ctdb->db_directory = talloc_strdup(ctdb, dir);
+	if (ctdb->db_directory == NULL) {
+		return -1;
+	}
+	return 0;
+}
+
+/*
+  add a node to the list of active nodes
+*/
+static int ctdb_add_node(struct ctdb_context *ctdb, char *nstr)
+{
+	struct ctdb_node *node, **nodep;
+
+	nodep = talloc_realloc(ctdb, ctdb->nodes, struct ctdb_node *, ctdb->num_nodes+1);
+	CTDB_NO_MEMORY(ctdb, nodep);
+
+	ctdb->nodes = nodep;
+	nodep = &ctdb->nodes[ctdb->num_nodes];
+	(*nodep) = talloc_zero(ctdb->nodes, struct ctdb_node);
+	CTDB_NO_MEMORY(ctdb, *nodep);
+	node = *nodep;
+
+	if (ctdb_parse_address(ctdb, node, nstr, &node->address) != 0) {
+		return -1;
+	}
+	node->ctdb = ctdb;
+	node->name = talloc_asprintf(node, "%s:%u", 
+				     node->address.address, 
+				     node->address.port);
+	/* this assumes that the nodes are kept in sorted order, and no gaps */
+	node->vnn = ctdb->num_nodes;
+
+	/* nodes start out disconnected */
+	node->flags |= NODE_FLAGS_DISCONNECTED;
+
+	if (ctdb->address.address &&
+	    ctdb_same_address(&ctdb->address, &node->address)) {
+		ctdb->vnn = node->vnn;
+		node->flags &= ~NODE_FLAGS_DISCONNECTED;
+	}
+
+	ctdb->num_nodes++;
+	node->dead_count = 0;
+
+	return 0;
+}
+
+/*
+  setup the node list from a file
+*/
+int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist)
+{
+	char **lines;
+	int nlines;
+	int i;
+
+	talloc_free(ctdb->node_list_file);
+	ctdb->node_list_file = talloc_strdup(ctdb, nlist);
+
+	lines = file_lines_load(nlist, &nlines, ctdb);
+	if (lines == NULL) {
+		ctdb_set_error(ctdb, "Failed to load nlist '%s'\n", nlist);
+		return -1;
+	}
+	while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
+		nlines--;
+	}
+
+	for (i=0;i<nlines;i++) {
+		if (ctdb_add_node(ctdb, lines[i]) != 0) {
+			talloc_free(lines);
+			return -1;
+		}
+	}
+
+	/* initialize the vnn mapping table now that we have num_nodes setup */
+/*
+XXX we currently initialize it to the maximum number of nodes to 
+XXX make it behave the same way as previously.  
+XXX Once we have recovery working we should initialize this always to 
+XXX generation==0 (==invalid) and let the recovery tool populate this 
+XXX table for the daemons. 
+*/
+	ctdb->vnn_map = talloc(ctdb, struct ctdb_vnn_map);
+	CTDB_NO_MEMORY(ctdb, ctdb->vnn_map);
+
+	ctdb->vnn_map->generation = 1;
+	ctdb->vnn_map->size = ctdb->num_nodes;
+	ctdb->vnn_map->map = talloc_array(ctdb->vnn_map, uint32_t, ctdb->vnn_map->size);
+	CTDB_NO_MEMORY(ctdb, ctdb->vnn_map->map);
+
+	for(i=0;i<ctdb->vnn_map->size;i++) {
+		ctdb->vnn_map->map[i] = i;
+	}
+	
+	talloc_free(lines);
+	return 0;
+}
+
+
+/*
+  setup the local node address
+*/
+int ctdb_set_address(struct ctdb_context *ctdb, const char *address)
+{
+	if (ctdb_parse_address(ctdb, ctdb, address, &ctdb->address) != 0) {
+		return -1;
+	}
+	
+	ctdb->name = talloc_asprintf(ctdb, "%s:%u", 
+				     ctdb->address.address, 
+				     ctdb->address.port);
+	return 0;
+}
+
+
+/*
+  return the number of active nodes
+*/
+uint32_t ctdb_get_num_active_nodes(struct ctdb_context *ctdb)
+{
+	int i;
+	uint32_t count=0;
+	for (i=0;i<ctdb->vnn_map->size;i++) {
+		struct ctdb_node *node = ctdb->nodes[ctdb->vnn_map->map[i]];
+		if (!(node->flags & NODE_FLAGS_INACTIVE)) {
+			count++;
+		}
+	}
+	return count;
+}
+
+
+/*
+  called when we need to process a packet. This can be a requeued packet
+  after a lockwait, or a real packet from another node
+*/
+void ctdb_input_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	TALLOC_CTX *tmp_ctx;
+
+	/* place the packet as a child of the tmp_ctx. We then use
+	   talloc_free() below to free it. If any of the calls want
+	   to keep it, then they will steal it somewhere else, and the
+	   talloc_free() will only free the tmp_ctx */
+	tmp_ctx = talloc_new(ctdb);
+	talloc_steal(tmp_ctx, hdr);
+
+	DEBUG(3,(__location__ " ctdb request %u of type %u length %u from "
+		 "node %u to %u\n", hdr->reqid, hdr->operation, hdr->length,
+		 hdr->srcnode, hdr->destnode));
+
+	switch (hdr->operation) {
+	case CTDB_REQ_CALL:
+	case CTDB_REPLY_CALL:
+	case CTDB_REQ_DMASTER:
+	case CTDB_REPLY_DMASTER:
+		/* for ctdb_call inter-node operations verify that the
+		   remote node that sent us the call is running in the
+		   same generation instance as this node
+		*/
+		if (ctdb->vnn_map->generation != hdr->generation) {
+			DEBUG(0,(__location__ " ctdb request %u"
+				" length %u from node %u to %u had an"
+				" invalid generation id:%u while our"
+				" generation id is:%u\n", 
+				 hdr->reqid, hdr->length, 
+				 hdr->srcnode, hdr->destnode, 
+				 hdr->generation, ctdb->vnn_map->generation));
+			goto done;
+		}
+	}
+
+	switch (hdr->operation) {
+	case CTDB_REQ_CALL:
+		ctdb->statistics.node.req_call++;
+		ctdb_request_call(ctdb, hdr);
+		break;
+
+	case CTDB_REPLY_CALL:
+		ctdb->statistics.node.reply_call++;
+		ctdb_reply_call(ctdb, hdr);
+		break;
+
+	case CTDB_REPLY_ERROR:
+		ctdb->statistics.node.reply_error++;
+		ctdb_reply_error(ctdb, hdr);
+		break;
+
+	case CTDB_REQ_DMASTER:
+		ctdb->statistics.node.req_dmaster++;
+		ctdb_request_dmaster(ctdb, hdr);
+		break;
+
+	case CTDB_REPLY_DMASTER:
+		ctdb->statistics.node.reply_dmaster++;
+		ctdb_reply_dmaster(ctdb, hdr);
+		break;
+
+	case CTDB_REQ_MESSAGE:
+		ctdb->statistics.node.req_message++;
+		ctdb_request_message(ctdb, hdr);
+		break;
+
+	case CTDB_REQ_CONTROL:
+		ctdb->statistics.node.req_control++;
+		ctdb_request_control(ctdb, hdr);
+		break;
+
+	case CTDB_REPLY_CONTROL:
+		ctdb->statistics.node.reply_control++;
+		ctdb_reply_control(ctdb, hdr);
+		break;
+
+	case CTDB_REQ_KEEPALIVE:
+		ctdb->statistics.keepalive_packets_recv++;
+		break;
+
+	default:
+		DEBUG(0,("%s: Packet with unknown operation %u\n", 
+			 __location__, hdr->operation));
+		break;
+	}
+
+done:
+	talloc_free(tmp_ctx);
+}
+
+
+/*
+  called by the transport layer when a node is dead
+*/
+void ctdb_node_dead(struct ctdb_node *node)
+{
+	if (node->flags & NODE_FLAGS_DISCONNECTED) {
+		DEBUG(1,("%s: node %s is already marked disconnected: %u connected\n", 
+			 node->ctdb->name, node->name, 
+			 node->ctdb->num_connected));
+		return;
+	}
+	node->ctdb->num_connected--;
+	node->flags |= NODE_FLAGS_DISCONNECTED;
+	node->rx_cnt = 0;
+	node->dead_count = 0;
+	DEBUG(1,("%s: node %s is dead: %u connected\n", 
+		 node->ctdb->name, node->name, node->ctdb->num_connected));
+	ctdb_daemon_cancel_controls(node->ctdb, node);
+}
+
+/*
+  called by the transport layer when a node is connected
+*/
+void ctdb_node_connected(struct ctdb_node *node)
+{
+	if (!(node->flags & NODE_FLAGS_DISCONNECTED)) {
+		DEBUG(1,("%s: node %s is already marked connected: %u connected\n", 
+			 node->ctdb->name, node->name, 
+			 node->ctdb->num_connected));
+		return;
+	}
+	node->ctdb->num_connected++;
+	node->dead_count = 0;
+	node->flags &= ~NODE_FLAGS_DISCONNECTED;
+	DEBUG(1,("%s: connected to %s - %u connected\n", 
+		 node->ctdb->name, node->name, node->ctdb->num_connected));
+}
+
+struct queue_next {
+	struct ctdb_context *ctdb;
+	struct ctdb_req_header *hdr;
+};
+
+
+/*
+  trigered when a deferred packet is due
+ */
+static void queue_next_trigger(struct event_context *ev, struct timed_event *te, 
+			       struct timeval t, void *private_data)
+{
+	struct queue_next *q = talloc_get_type(private_data, struct queue_next);
+	ctdb_input_pkt(q->ctdb, q->hdr);
+	talloc_free(q);
+}	
+
+/*
+  defer a packet, so it is processed on the next event loop
+  this is used for sending packets to ourselves
+ */
+static void ctdb_defer_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	struct queue_next *q;
+	q = talloc(ctdb, struct queue_next);
+	if (q == NULL) {
+		DEBUG(0,(__location__ " Failed to allocate deferred packet\n"));
+		return;
+	}
+	q->ctdb = ctdb;
+	q->hdr = talloc_memdup(ctdb, hdr, hdr->length);
+	if (q->hdr == NULL) {
+		DEBUG(0,("Error copying deferred packet to self\n"));
+		return;
+	}
+#if 0
+	/* use this to put packets directly into our recv function */
+	ctdb_input_pkt(q->ctdb, q->hdr);
+#else
+	event_add_timed(ctdb->ev, q, timeval_zero(), queue_next_trigger, q);
+#endif
+}
+
+
+/*
+  broadcast a packet to all nodes
+*/
+static void ctdb_broadcast_packet_all(struct ctdb_context *ctdb, 
+				      struct ctdb_req_header *hdr)
+{
+	int i;
+	for (i=0;i<ctdb->num_nodes;i++) {
+		hdr->destnode = ctdb->nodes[i]->vnn;
+		ctdb_queue_packet(ctdb, hdr);
+	}
+}
+
+/*
+  broadcast a packet to all nodes in the current vnnmap
+*/
+static void ctdb_broadcast_packet_vnnmap(struct ctdb_context *ctdb, 
+					 struct ctdb_req_header *hdr)
+{
+	int i;
+	for (i=0;i<ctdb->vnn_map->size;i++) {
+		hdr->destnode = ctdb->vnn_map->map[i];
+		ctdb_queue_packet(ctdb, hdr);
+	}
+}
+
+/*
+  broadcast a packet to all connected nodes
+*/
+static void ctdb_broadcast_packet_connected(struct ctdb_context *ctdb, 
+					    struct ctdb_req_header *hdr)
+{
+	int i;
+	for (i=0;i<ctdb->num_nodes;i++) {
+		if (!(ctdb->nodes[i]->flags & NODE_FLAGS_DISCONNECTED)) {
+			hdr->destnode = ctdb->nodes[i]->vnn;
+			ctdb_queue_packet(ctdb, hdr);
+		}
+	}
+}
+
+/*
+  queue a packet or die
+*/
+void ctdb_queue_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+	struct ctdb_node *node;
+
+	switch (hdr->destnode) {
+	case CTDB_BROADCAST_ALL:
+		ctdb_broadcast_packet_all(ctdb, hdr);
+		return;
+	case CTDB_BROADCAST_VNNMAP:
+		ctdb_broadcast_packet_vnnmap(ctdb, hdr);
+		return;
+	case CTDB_BROADCAST_CONNECTED:
+		ctdb_broadcast_packet_connected(ctdb, hdr);
+		return;
+	}
+
+	ctdb->statistics.node_packets_sent++;
+
+	if (!ctdb_validate_vnn(ctdb, hdr->destnode)) {
+	  	DEBUG(0,(__location__ " cant send to node %u that does not exist\n", 
+			 hdr->destnode));
+		return;
+	}
+
+	node = ctdb->nodes[hdr->destnode];
+
+	if (hdr->destnode == ctdb->vnn) {
+		ctdb_defer_packet(ctdb, hdr);
+	} else {
+		node->tx_cnt++;
+		if (ctdb->methods->queue_pkt(node, (uint8_t *)hdr, hdr->length) != 0) {
+			ctdb_fatal(ctdb, "Unable to queue packet\n");
+		}
+	}
+}
+
+
diff --git a/source4/cluster/ctdb/server/ctdb_takeover.c b/source4/cluster/ctdb/server/ctdb_takeover.c
new file mode 100644
index 0000000000..42a23808dd
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_takeover.c
@@ -0,0 +1,822 @@
+/* 
+   ctdb recovery code
+
+   Copyright (C) Ronnie Sahlberg  2007
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "lib/util/dlinklist.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+
+
+#define TAKEOVER_TIMEOUT() timeval_current_ofs(ctdb->tunable.takeover_timeout,0)
+
+#define CTDB_ARP_INTERVAL 1
+#define CTDB_ARP_REPEAT   3
+
+struct ctdb_takeover_arp {
+	struct ctdb_context *ctdb;
+	uint32_t count;
+	struct sockaddr_in sin;
+	struct ctdb_tcp_list *tcp_list;
+};
+
+/*
+  lists of tcp endpoints
+ */
+struct ctdb_tcp_list {
+	struct ctdb_tcp_list *prev, *next;
+	uint32_t vnn;
+	struct sockaddr_in saddr;
+	struct sockaddr_in daddr;
+};
+
+
+/*
+  list of clients to kill on IP release
+ */
+struct ctdb_client_ip {
+	struct ctdb_client_ip *prev, *next;
+	struct ctdb_context *ctdb;
+	struct sockaddr_in ip;
+	uint32_t client_id;
+};
+
+
+/*
+  send a gratuitous arp
+ */
+static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *te, 
+				  struct timeval t, void *private_data)
+{
+	struct ctdb_takeover_arp *arp = talloc_get_type(private_data, 
+							struct ctdb_takeover_arp);
+	int ret;
+	struct ctdb_tcp_list *tcp;
+
+	ret = ctdb_sys_send_arp(&arp->sin, arp->ctdb->takeover.interface);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " sending of arp failed (%s)\n", strerror(errno)));
+	}
+
+	for (tcp=arp->tcp_list;tcp;tcp=tcp->next) {
+		DEBUG(2,("sending tcp tickle ack for %u->%s:%u\n",
+			 (unsigned)ntohs(tcp->daddr.sin_port), 
+			 inet_ntoa(tcp->saddr.sin_addr),
+			 (unsigned)ntohs(tcp->saddr.sin_port)));
+		ret = ctdb_sys_send_tcp(&tcp->saddr, &tcp->daddr, 0, 0, 0);
+		if (ret != 0) {
+			DEBUG(0,(__location__ " Failed to send tcp tickle ack for %s\n",
+				 inet_ntoa(tcp->saddr.sin_addr)));
+		}
+	}
+
+	arp->count++;
+
+	if (arp->count == CTDB_ARP_REPEAT) {
+		talloc_free(arp);
+		return;
+	}
+	
+	event_add_timed(arp->ctdb->ev, arp->ctdb->takeover.last_ctx, 
+			timeval_current_ofs(CTDB_ARP_INTERVAL, 0), 
+			ctdb_control_send_arp, arp);
+}
+
+struct takeover_callback_state {
+	struct ctdb_req_control *c;
+	struct sockaddr_in *sin;
+};
+
+/*
+  called when takeip event finishes
+ */
+static void takeover_ip_callback(struct ctdb_context *ctdb, int status, 
+				 void *private_data)
+{
+	struct takeover_callback_state *state = 
+		talloc_get_type(private_data, struct takeover_callback_state);
+	struct ctdb_takeover_arp *arp;
+	char *ip = inet_ntoa(state->sin->sin_addr);
+	struct ctdb_tcp_list *tcp;
+
+	ctdb_start_monitoring(ctdb);
+
+	if (status != 0) {
+		DEBUG(0,(__location__ " Failed to takeover IP %s on interface %s\n",
+			 ip, ctdb->takeover.interface));
+		ctdb_request_control_reply(ctdb, state->c, NULL, status, NULL);
+		talloc_free(state);
+		return;
+	}
+
+	if (!ctdb->takeover.last_ctx) {
+		ctdb->takeover.last_ctx = talloc_new(ctdb);
+		if (!ctdb->takeover.last_ctx) goto failed;
+	}
+
+	arp = talloc_zero(ctdb->takeover.last_ctx, struct ctdb_takeover_arp);
+	if (!arp) goto failed;
+	
+	arp->ctdb = ctdb;
+	arp->sin = *state->sin;
+
+	/* add all of the known tcp connections for this IP to the
+	   list of tcp connections to send tickle acks for */
+	for (tcp=ctdb->tcp_list;tcp;tcp=tcp->next) {
+		if (state->sin->sin_addr.s_addr == tcp->daddr.sin_addr.s_addr) {
+			struct ctdb_tcp_list *t2 = talloc(arp, struct ctdb_tcp_list);
+			if (t2 == NULL) goto failed;
+			*t2 = *tcp;
+			DLIST_ADD(arp->tcp_list, t2);
+		}
+	}
+
+	event_add_timed(arp->ctdb->ev, arp->ctdb->takeover.last_ctx, 
+			timeval_zero(), ctdb_control_send_arp, arp);
+
+	/* the control succeeded */
+	ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
+	talloc_free(state);
+	return;
+
+failed:
+	ctdb_request_control_reply(ctdb, state->c, NULL, -1, NULL);
+	talloc_free(state);
+	return;
+}
+
+/*
+  take over an ip address
+ */
+int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply)
+{
+	int ret;
+	struct takeover_callback_state *state;
+	struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
+	char *ip = inet_ntoa(pip->sin.sin_addr);
+
+
+	/* update out node table */
+	ctdb->nodes[pip->vnn]->takeover_vnn = pip->takeover_vnn;
+
+	/* if our kernel already has this IP, do nothing */
+	if (ctdb_sys_have_ip(ip)) {
+		return 0;
+	}
+
+	state = talloc(ctdb, struct takeover_callback_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	state->c = talloc_steal(ctdb, c);
+	state->sin = talloc(ctdb, struct sockaddr_in);       
+	CTDB_NO_MEMORY(ctdb, state->sin);
+	*state->sin = pip->sin;
+
+	DEBUG(0,("Takover of IP %s/%u on interface %s\n", 
+		 ip, ctdb->nodes[ctdb->vnn]->public_netmask_bits, 
+		 ctdb->takeover.interface));
+
+	ctdb_stop_monitoring(ctdb);
+
+	ret = ctdb_event_script_callback(ctdb, 
+					 timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+					 state, takeover_ip_callback, state,
+					 "takeip %s %s %u",
+					 ctdb->takeover.interface, 
+					 ip,
+					 ctdb->nodes[ctdb->vnn]->public_netmask_bits);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to takeover IP %s on interface %s\n",
+			 ip, ctdb->takeover.interface));
+		talloc_free(state);
+		return -1;
+	}
+
+	/* tell ctdb_control.c that we will be replying asynchronously */
+	*async_reply = true;
+
+	return 0;
+}
+
+/*
+  kill any clients that are registered with a IP that is being released
+ */
+static void release_kill_clients(struct ctdb_context *ctdb, struct in_addr in)
+{
+	struct ctdb_client_ip *ip;
+
+	for (ip=ctdb->client_ip_list; ip; ip=ip->next) {
+		if (ip->ip.sin_addr.s_addr == in.s_addr) {
+			struct ctdb_client *client = ctdb_reqid_find(ctdb, 
+								     ip->client_id, 
+								     struct ctdb_client);
+			if (client->pid != 0) {
+				DEBUG(0,(__location__ " Killing client pid %u for IP %s on client_id %u\n",
+					 (unsigned)client->pid, inet_ntoa(in),
+					 ip->client_id));
+				kill(client->pid, SIGKILL);
+			}
+		}
+	}
+}
+
+/*
+  called when releaseip event finishes
+ */
+static void release_ip_callback(struct ctdb_context *ctdb, int status, 
+				void *private_data)
+{
+	struct takeover_callback_state *state = 
+		talloc_get_type(private_data, struct takeover_callback_state);
+	char *ip = inet_ntoa(state->sin->sin_addr);
+	TDB_DATA data;
+	struct ctdb_tcp_list *tcp;
+
+	ctdb_start_monitoring(ctdb);
+
+	/* send a message to all clients of this node telling them
+	   that the cluster has been reconfigured and they should
+	   release any sockets on this IP */
+	data.dptr = (uint8_t *)ip;
+	data.dsize = strlen(ip)+1;
+
+	ctdb_daemon_send_message(ctdb, ctdb->vnn, CTDB_SRVID_RELEASE_IP, data);
+
+	/* kill clients that have registered with this IP */
+	release_kill_clients(ctdb, state->sin->sin_addr);
+	
+
+	/* tell other nodes about any tcp connections we were holding with this IP */
+	for (tcp=ctdb->tcp_list;tcp;tcp=tcp->next) {
+		if (tcp->vnn == ctdb->vnn && 
+		    state->sin->sin_addr.s_addr == tcp->daddr.sin_addr.s_addr) {
+			struct ctdb_control_tcp_vnn t;
+
+			t.vnn  = ctdb->vnn;
+			t.src  = tcp->saddr;
+			t.dest = tcp->daddr;
+
+			data.dptr = (uint8_t *)&t;
+			data.dsize = sizeof(t);
+
+			ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0, 
+						 CTDB_CONTROL_TCP_ADD,
+						 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		}
+	}
+
+	/* the control succeeded */
+	ctdb_request_control_reply(ctdb, state->c, NULL, 0, NULL);
+	talloc_free(state);
+}
+
+
+/*
+  release an ip address
+ */
+int32_t ctdb_control_release_ip(struct ctdb_context *ctdb, 
+				struct ctdb_req_control *c,
+				TDB_DATA indata, 
+				bool *async_reply)
+{
+	int ret;
+	struct takeover_callback_state *state;
+	struct ctdb_public_ip *pip = (struct ctdb_public_ip *)indata.dptr;
+	char *ip = inet_ntoa(pip->sin.sin_addr);
+
+	/* update out node table */
+	ctdb->nodes[pip->vnn]->takeover_vnn = pip->takeover_vnn;
+
+	if (!ctdb_sys_have_ip(ip)) {
+		return 0;
+	}
+
+	DEBUG(0,("Release of IP %s/%u on interface %s\n", 
+		 ip, ctdb->nodes[ctdb->vnn]->public_netmask_bits, 
+		 ctdb->takeover.interface));
+
+	/* stop any previous arps */
+	talloc_free(ctdb->takeover.last_ctx);
+	ctdb->takeover.last_ctx = NULL;
+
+	state = talloc(ctdb, struct takeover_callback_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	state->c = talloc_steal(state, c);
+	state->sin = talloc(state, struct sockaddr_in);       
+	CTDB_NO_MEMORY(ctdb, state->sin);
+	*state->sin = pip->sin;
+
+	ctdb_stop_monitoring(ctdb);
+
+	ret = ctdb_event_script_callback(ctdb, 
+					 timeval_current_ofs(ctdb->tunable.script_timeout, 0),
+					 state, release_ip_callback, state,
+					 "releaseip %s %s %u",
+					 ctdb->takeover.interface, 
+					 ip,
+					 ctdb->nodes[ctdb->vnn]->public_netmask_bits);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to release IP %s on interface %s\n",
+			 ip, ctdb->takeover.interface));
+		talloc_free(state);
+		return -1;
+	}
+
+	/* tell the control that we will be reply asynchronously */
+	*async_reply = true;
+
+	return 0;
+}
+
+
+/*
+  setup the event script
+*/
+int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script)
+{
+	ctdb->takeover.event_script = talloc_strdup(ctdb, script);
+	CTDB_NO_MEMORY(ctdb, ctdb->takeover.event_script);
+	return 0;
+}
+
+/*
+  setup the public address list from a file
+*/
+int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist)
+{
+	char **lines;
+	int nlines;
+	int i;
+
+	lines = file_lines_load(alist, &nlines, ctdb);
+	if (lines == NULL) {
+		ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", alist);
+		return -1;
+	}
+	while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
+		nlines--;
+	}
+
+	if (nlines != ctdb->num_nodes) {
+		DEBUG(0,("Number of lines in %s does not match number of nodes!\n", alist));
+		talloc_free(lines);
+		return -1;
+	}
+
+	for (i=0;i<nlines;i++) {
+		char *p;
+		struct in_addr in;
+
+		ctdb->nodes[i]->public_address = talloc_strdup(ctdb->nodes[i], lines[i]);
+		CTDB_NO_MEMORY(ctdb, ctdb->nodes[i]->public_address);
+		ctdb->nodes[i]->takeover_vnn = -1;
+
+		/* see if they supplied a netmask length */
+		p = strchr(ctdb->nodes[i]->public_address, '/');
+		if (!p) {
+			DEBUG(0,("You must supply a netmask for public address %s\n",
+				 ctdb->nodes[i]->public_address));
+			return -1;
+		}
+		*p = 0;
+		ctdb->nodes[i]->public_netmask_bits = atoi(p+1);
+
+		if (ctdb->nodes[i]->public_netmask_bits > 32) {
+			DEBUG(0, ("Illegal netmask for IP %s\n", ctdb->nodes[i]->public_address));
+			return -1;
+		}
+
+		if (inet_aton(ctdb->nodes[i]->public_address, &in) == 0) {
+			DEBUG(0,("Badly formed IP '%s' in public address list\n", ctdb->nodes[i]->public_address));
+			return -1;
+		}
+	}
+
+	talloc_free(lines);
+	return 0;
+}
+
+/*
+  see if two IPs are on the same subnet
+ */
+static bool ctdb_same_subnet(const char *ip1, const char *ip2, uint8_t netmask_bits)
+{
+	struct in_addr in1, in2;
+	uint32_t mask;
+
+	inet_aton(ip1, &in1);
+	inet_aton(ip2, &in2);
+
+	mask = ~((1LL<<(32-netmask_bits))-1);
+
+	if ((ntohl(in1.s_addr) & mask) != (ntohl(in2.s_addr) & mask)) {
+		return false;
+	}
+
+	return true;
+}
+
+
+/*
+  try to find an available node to take a given nodes IP that meets the
+  criterion given by the flags
+ */
+static void ctdb_takeover_find_node(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap,
+				    int start_node, uint32_t mask_flags)
+{
+	int j;
+	for (j=(start_node+1)%nodemap->num;
+	     j != start_node;
+	     j=(j+1)%nodemap->num) {
+		if (!(nodemap->nodes[j].flags & mask_flags) &&
+		    ctdb_same_subnet(ctdb->nodes[j]->public_address, 
+				     ctdb->nodes[start_node]->public_address, 
+				     ctdb->nodes[j]->public_netmask_bits)) {
+			ctdb->nodes[start_node]->takeover_vnn = nodemap->nodes[j].vnn;
+			break;
+		}
+	}
+}
+
+
+/*
+  make any IP alias changes for public addresses that are necessary 
+ */
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
+{
+	int i, j;
+	int ret;
+	struct ctdb_public_ip ip;
+
+	ZERO_STRUCT(ip);
+
+	/* Work out which node will look after each public IP.
+	 * takeover_node cycles over the nodes and is incremented each time a 
+	 * node has been assigned to take over for another node.
+	 * This spreads the failed nodes out across the remaining
+	 * nodes more evenly
+	 */
+	for (i=0;i<nodemap->num;i++) {
+		if (!(nodemap->nodes[i].flags & (NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED))) {
+			ctdb->nodes[i]->takeover_vnn = nodemap->nodes[i].vnn;
+		} else {
+			ctdb->nodes[i]->takeover_vnn = (uint32_t)-1;	
+
+			ctdb_takeover_find_node(ctdb, nodemap, i, NODE_FLAGS_INACTIVE|NODE_FLAGS_DISABLED);
+			
+			/* if no enabled node can take it, then we
+			   might as well use any active node. It
+			   probably means that some subsystem (such as
+			   NFS) is sick on all nodes. Best we can do
+			   is to keep the other services up. */
+			if (ctdb->nodes[i]->takeover_vnn == (uint32_t)-1) {
+				ctdb_takeover_find_node(ctdb, nodemap, i, NODE_FLAGS_INACTIVE);
+			}
+
+			if (ctdb->nodes[i]->takeover_vnn == (uint32_t)-1) {
+				DEBUG(0,(__location__ " No node available on same network to take %s\n",
+					 ctdb->nodes[i]->public_address));
+			}
+		}
+	}	
+
+	/* at this point ctdb->nodes[i]->takeover_vnn is the vnn which will own each IP */
+
+	/* now tell all nodes to delete any alias that they should not
+	   have.  This will be a NOOP on nodes that don't currently
+	   hold the given alias */
+	for (i=0;i<nodemap->num;i++) {
+		/* don't talk to unconnected nodes, but do talk to banned nodes */
+		if (nodemap->nodes[i].flags & NODE_FLAGS_DISCONNECTED) {
+			continue;
+		}
+
+		/* tell this node to delete all of the aliases that it should not have */
+		for (j=0;j<nodemap->num;j++) {
+			if (ctdb->nodes[j]->takeover_vnn != nodemap->nodes[i].vnn) {
+				ip.vnn = j;
+				ip.takeover_vnn = ctdb->nodes[j]->takeover_vnn;
+				ip.sin.sin_family = AF_INET;
+				inet_aton(ctdb->nodes[j]->public_address, &ip.sin.sin_addr);
+
+				ret = ctdb_ctrl_release_ip(ctdb, TAKEOVER_TIMEOUT(),
+							   nodemap->nodes[i].vnn, 
+							   &ip);
+				if (ret != 0) {
+					DEBUG(0,("Failed to tell vnn %u to release IP %s\n",
+						 nodemap->nodes[i].vnn,
+						 ctdb->nodes[j]->public_address));
+					return -1;
+				}
+			}
+		}
+	}
+
+	/* tell all nodes to get their own IPs */
+	for (i=0;i<nodemap->num;i++) {
+		if (ctdb->nodes[i]->takeover_vnn == -1) {
+			/* this IP won't be taken over */
+			continue;
+		}
+		ip.vnn = i;
+		ip.takeover_vnn = ctdb->nodes[i]->takeover_vnn;
+		ip.sin.sin_family = AF_INET;
+		inet_aton(ctdb->nodes[i]->public_address, &ip.sin.sin_addr);
+
+		ret = ctdb_ctrl_takeover_ip(ctdb, TAKEOVER_TIMEOUT(), 
+					    ctdb->nodes[i]->takeover_vnn, 
+					    &ip);
+		if (ret != 0) {
+			DEBUG(0,("Failed asking vnn %u to take over IP %s\n",
+				 ctdb->nodes[i]->takeover_vnn, 
+				 ctdb->nodes[i]->public_address));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  destroy a ctdb_client_ip structure
+ */
+static int ctdb_client_ip_destructor(struct ctdb_client_ip *ip)
+{
+	DLIST_REMOVE(ip->ctdb->client_ip_list, ip);
+	return 0;
+}
+
+/*
+  called by a client to inform us of a TCP connection that it is managing
+  that should tickled with an ACK when IP takeover is done
+ */
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, uint32_t vnn,
+				TDB_DATA indata)
+{
+	struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+	struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+	struct ctdb_tcp_list *tcp;
+	struct ctdb_control_tcp_vnn t;
+	int ret;
+	TDB_DATA data;
+	struct ctdb_client_ip *ip;
+
+	ip = talloc(client, struct ctdb_client_ip);
+	CTDB_NO_MEMORY(ctdb, ip);
+
+	ip->ctdb = ctdb;
+	ip->ip = p->dest;
+	ip->client_id = client_id;
+	talloc_set_destructor(ip, ctdb_client_ip_destructor);
+	DLIST_ADD(ctdb->client_ip_list, ip);
+
+	tcp = talloc(client, struct ctdb_tcp_list);
+	CTDB_NO_MEMORY(ctdb, tcp);
+
+	tcp->vnn   = vnn;
+	tcp->saddr = p->src;
+	tcp->daddr = p->dest;
+
+	DLIST_ADD(client->tcp_list, tcp);
+
+	t.vnn  = vnn;
+	t.src  = p->src;
+	t.dest = p->dest;
+
+	data.dptr = (uint8_t *)&t;
+	data.dsize = sizeof(t);
+
+	/* tell all nodes about this tcp connection */
+	ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_CONNECTED, 0, 
+				       CTDB_CONTROL_TCP_ADD,
+				       0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  see if two sockaddr_in are the same
+ */
+static bool same_sockaddr_in(struct sockaddr_in *in1, struct sockaddr_in *in2)
+{
+	return in1->sin_family == in2->sin_family &&
+		in1->sin_port == in2->sin_port &&
+		in1->sin_addr.s_addr == in2->sin_addr.s_addr;
+}
+
+/*
+  find a tcp address on a list
+ */
+static struct ctdb_tcp_list *ctdb_tcp_find(struct ctdb_tcp_list *list, 
+					   struct ctdb_tcp_list *tcp)
+{
+	while (list) {
+		if (same_sockaddr_in(&list->saddr, &tcp->saddr) &&
+		    same_sockaddr_in(&list->daddr, &tcp->daddr)) {
+			return list;
+		}
+		list = list->next;
+	}
+	return NULL;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_tcp_vnn *p = (struct ctdb_control_tcp_vnn *)indata.dptr;
+	struct ctdb_tcp_list *tcp;
+
+	tcp = talloc(ctdb, struct ctdb_tcp_list);
+	CTDB_NO_MEMORY(ctdb, tcp);
+
+	tcp->vnn   = p->vnn;
+	tcp->saddr = p->src;
+	tcp->daddr = p->dest;
+
+	if (NULL == ctdb_tcp_find(ctdb->tcp_list, tcp)) {
+		DLIST_ADD(ctdb->tcp_list, tcp);
+		DEBUG(2,("Added tickle info for %s:%u from vnn %u\n",
+			 inet_ntoa(tcp->daddr.sin_addr), ntohs(tcp->daddr.sin_port),
+			 tcp->vnn));
+	} else {
+		DEBUG(4,("Already had tickle info for %s:%u from vnn %u\n",
+			 inet_ntoa(tcp->daddr.sin_addr), ntohs(tcp->daddr.sin_port),
+			 tcp->vnn));
+	}
+
+	return 0;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_tcp_vnn *p = (struct ctdb_control_tcp_vnn *)indata.dptr;
+	struct ctdb_tcp_list t, *tcp;
+
+	t.vnn   = p->vnn;
+	t.saddr = p->src;
+	t.daddr = p->dest;
+
+	tcp = ctdb_tcp_find(ctdb->tcp_list, &t);
+	if (tcp) {
+		DEBUG(2,("Removed tickle info for %s:%u from vnn %u\n",
+			 inet_ntoa(tcp->daddr.sin_addr), ntohs(tcp->daddr.sin_port),
+			 tcp->vnn));
+		DLIST_REMOVE(ctdb->tcp_list, tcp);
+		talloc_free(tcp);
+	}
+
+	return 0;
+}
+
+
+/*
+  called when a daemon restarts - wipes all tcp entries from that vnn
+ */
+int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn)
+{
+	struct ctdb_tcp_list *tcp, *next;	
+	for (tcp=ctdb->tcp_list;tcp;tcp=next) {
+		next = tcp->next;
+		if (tcp->vnn == vnn) {
+			DLIST_REMOVE(ctdb->tcp_list, tcp);
+			talloc_free(tcp);
+		}
+
+		/* and tell the new guy about any that he should have
+		   from us */
+		if (tcp->vnn == ctdb->vnn) {
+			struct ctdb_control_tcp_vnn t;
+			TDB_DATA data;
+
+			t.vnn  = tcp->vnn;
+			t.src  = tcp->saddr;
+			t.dest = tcp->daddr;
+
+			data.dptr = (uint8_t *)&t;
+			data.dsize = sizeof(t);
+
+			ctdb_daemon_send_control(ctdb, vnn, 0, 
+						 CTDB_CONTROL_TCP_ADD,
+						 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		}
+	}
+	return 0;
+}
+
+
+/*
+  called when a client structure goes away - hook to remove
+  elements from the tcp_list in all daemons
+ */
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
+{
+	while (client->tcp_list) {
+		TDB_DATA data;
+		struct ctdb_control_tcp_vnn p;
+		struct ctdb_tcp_list *tcp = client->tcp_list;
+		DLIST_REMOVE(client->tcp_list, tcp);
+		p.vnn = tcp->vnn;
+		p.src = tcp->saddr;
+		p.dest = tcp->daddr;
+		data.dptr = (uint8_t *)&p;
+		data.dsize = sizeof(p);
+		ctdb_daemon_send_control(client->ctdb, CTDB_BROADCAST_CONNECTED, 0, 
+					 CTDB_CONTROL_TCP_REMOVE,
+					 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		talloc_free(tcp);
+	}
+}
+
+
+/*
+  release all IPs on shutdown
+ */
+void ctdb_release_all_ips(struct ctdb_context *ctdb)
+{
+	int i;
+
+	if (!ctdb->takeover.enabled) {
+		return;
+	}
+
+	for (i=0;i<ctdb->num_nodes;i++) {
+		struct ctdb_node *node = ctdb->nodes[i];
+		if (ctdb_sys_have_ip(node->public_address)) {
+			struct in_addr in;
+			ctdb_event_script(ctdb, "releaseip %s %s %u",
+					  ctdb->takeover.interface, 
+					  node->public_address,
+					  node->public_netmask_bits);
+			if (inet_aton(node->public_address, &in) != 0) {
+				release_kill_clients(ctdb, in);
+			}
+		}
+	}
+}
+
+
+/*
+  get list of public IPs
+ */
+int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA *outdata)
+{
+	int i, len;
+	struct ctdb_all_public_ips *ips;
+
+	len = offsetof(struct ctdb_all_public_ips, ips) + ctdb->num_nodes*sizeof(struct ctdb_public_ip);
+
+	ips = talloc_zero_size(outdata, len);
+	CTDB_NO_MEMORY(ctdb, ips);
+
+	outdata->dsize = len;
+	outdata->dptr  = (uint8_t *)ips;
+
+	ips->num = ctdb->num_nodes;
+	for(i=0;i<ctdb->num_nodes;i++){
+		ips->ips[i].vnn = i;
+		ips->ips[i].takeover_vnn = ctdb->nodes[i]->takeover_vnn;
+		ips->ips[i].sin.sin_family = AF_INET;
+		if (ctdb->nodes[i]->public_address) {
+			inet_aton(ctdb->nodes[i]->public_address, &ips->ips[i].sin.sin_addr);
+		}
+	}
+
+	return 0;
+}
diff --git a/source4/cluster/ctdb/server/ctdb_traverse.c b/source4/cluster/ctdb/server/ctdb_traverse.c
new file mode 100644
index 0000000000..d44c27401a
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_traverse.c
@@ -0,0 +1,463 @@
+/* 
+   efficient async ctdb traverse
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "db_wrap.h"
+#include "lib/tdb/include/tdb.h"
+#include "../include/ctdb_private.h"
+
+typedef void (*ctdb_traverse_fn_t)(void *private_data, TDB_DATA key, TDB_DATA data);
+
+/*
+  handle returned to caller - freeing this handler will kill the child and 
+  terminate the traverse
+ */
+struct ctdb_traverse_local_handle {
+	struct ctdb_db_context *ctdb_db;
+	int fd[2];
+	pid_t child;
+	void *private_data;
+	ctdb_traverse_fn_t callback;
+	struct timeval start_time;
+	struct ctdb_queue *queue;
+};
+
+/*
+  called when data is available from the child
+ */
+static void ctdb_traverse_local_handler(uint8_t *rawdata, size_t length, void *private_data)
+{
+	struct ctdb_traverse_local_handle *h = talloc_get_type(private_data, 
+							       struct ctdb_traverse_local_handle);
+	TDB_DATA key, data;
+	ctdb_traverse_fn_t callback = h->callback;
+	void *p = h->private_data;
+	struct ctdb_rec_data *tdata = (struct ctdb_rec_data *)rawdata;
+
+	if (rawdata == NULL || length < 4 || length != tdata->length) {
+		/* end of traverse */
+		talloc_free(h);
+		callback(p, tdb_null, tdb_null);
+		return;
+	}
+
+	key.dsize = tdata->keylen;
+	key.dptr  = &tdata->data[0];
+	data.dsize = tdata->datalen;
+	data.dptr = &tdata->data[tdata->keylen];
+
+	callback(p, key, data);	
+}
+
+/*
+  destroy a in-flight traverse operation
+ */
+static int traverse_local_destructor(struct ctdb_traverse_local_handle *h)
+{
+	kill(h->child, SIGKILL);
+	waitpid(h->child, NULL, 0);
+	return 0;
+}
+
+/*
+  callback from tdb_traverse_read()
+ */
+static int ctdb_traverse_local_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *p)
+{
+	struct ctdb_traverse_local_handle *h = talloc_get_type(p, 
+							       struct ctdb_traverse_local_handle);
+	struct ctdb_rec_data *d;
+	struct ctdb_ltdb_header *hdr;
+
+	/* filter out non-authoritative and zero-length records */
+	hdr = (struct ctdb_ltdb_header *)data.dptr;
+	if (data.dsize <= sizeof(struct ctdb_ltdb_header) ||
+	    hdr->dmaster != h->ctdb_db->ctdb->vnn) {
+		return 0;
+	}
+
+	d = ctdb_marshall_record(h, 0, key, data);
+	if (d == NULL) {
+		/* error handling is tricky in this child code .... */
+		return -1;
+	}
+
+	if (write(h->fd[1], (uint8_t *)d, d->length) != d->length) {
+		return -1;
+	}
+	return 0;
+}
+
+
+/*
+  setup a non-blocking traverse of a local ltdb. The callback function
+  will be called on every record in the local ltdb. To stop the
+  travserse, talloc_free() the travserse_handle.
+
+  The traverse is finished when the callback is called with tdb_null for key and data
+ */
+static struct ctdb_traverse_local_handle *ctdb_traverse_local(struct ctdb_db_context *ctdb_db,
+							      ctdb_traverse_fn_t callback,
+							      void *private_data)
+{
+	struct ctdb_traverse_local_handle *h;
+	int ret;
+
+	h = talloc_zero(ctdb_db, struct ctdb_traverse_local_handle);
+	if (h == NULL) {
+		return NULL;
+	}
+
+	ret = pipe(h->fd);
+
+	if (ret != 0) {
+		talloc_free(h);
+		return NULL;
+	}
+
+	h->child = fork();
+
+	if (h->child == (pid_t)-1) {
+		close(h->fd[0]);
+		close(h->fd[1]);
+		talloc_free(h);
+		return NULL;
+	}
+
+	h->callback = callback;
+	h->private_data = private_data;
+	h->ctdb_db = ctdb_db;
+
+	if (h->child == 0) {
+		/* start the traverse in the child */
+		close(h->fd[0]);
+		tdb_traverse_read(ctdb_db->ltdb->tdb, ctdb_traverse_local_fn, h);
+		_exit(0);
+	}
+
+	close(h->fd[1]);
+	talloc_set_destructor(h, traverse_local_destructor);
+
+	/*
+	  setup a packet queue between the child and the parent. This
+	  copes with all the async and packet boundary issues
+	 */
+	h->queue = ctdb_queue_setup(ctdb_db->ctdb, h, h->fd[0], 0, ctdb_traverse_local_handler, h);
+	if (h->queue == NULL) {
+		talloc_free(h);
+		return NULL;
+	}
+
+	h->start_time = timeval_current();
+
+	return h;
+}
+
+
+struct ctdb_traverse_all_handle {
+	struct ctdb_context *ctdb;
+	uint32_t reqid;
+	ctdb_traverse_fn_t callback;
+	void *private_data;
+	uint32_t null_count;
+};
+
+/*
+  destroy a traverse_all op
+ */
+static int ctdb_traverse_all_destructor(struct ctdb_traverse_all_handle *state)
+{
+	ctdb_reqid_remove(state->ctdb, state->reqid);
+	return 0;
+}
+
+struct ctdb_traverse_all {
+	uint32_t db_id;
+	uint32_t reqid;
+	uint32_t vnn;
+};
+
+/* called when a traverse times out */
+static void ctdb_traverse_all_timeout(struct event_context *ev, struct timed_event *te, 
+				      struct timeval t, void *private_data)
+{
+	struct ctdb_traverse_all_handle *state = talloc_get_type(private_data, struct ctdb_traverse_all_handle);
+
+	state->ctdb->statistics.timeouts.traverse++;
+
+	state->callback(state->private_data, tdb_null, tdb_null);
+	talloc_free(state);
+}
+
+/*
+  setup a cluster-wide non-blocking traverse of a ctdb. The
+  callback function will be called on every record in the local
+  ltdb. To stop the travserse, talloc_free() the traverse_handle.
+
+  The traverse is finished when the callback is called with tdb_null
+  for key and data
+ */
+static struct ctdb_traverse_all_handle *ctdb_daemon_traverse_all(struct ctdb_db_context *ctdb_db,
+								 ctdb_traverse_fn_t callback,
+								 void *private_data)
+{
+	struct ctdb_traverse_all_handle *state;
+	struct ctdb_context *ctdb = ctdb_db->ctdb;
+	int ret;
+	TDB_DATA data;
+	struct ctdb_traverse_all r;
+
+	state = talloc(ctdb_db, struct ctdb_traverse_all_handle);
+	if (state == NULL) {
+		return NULL;
+	}
+
+	state->ctdb = ctdb;
+	state->reqid = ctdb_reqid_new(ctdb_db->ctdb, state);
+	state->callback = callback;
+	state->private_data = private_data;
+	state->null_count = 0;
+	
+	talloc_set_destructor(state, ctdb_traverse_all_destructor);
+
+	r.db_id = ctdb_db->db_id;
+	r.reqid = state->reqid;
+	r.vnn   = ctdb->vnn;
+
+	data.dptr = (uint8_t *)&r;
+	data.dsize = sizeof(r);
+
+	/* tell all the nodes in the cluster to start sending records to this node */
+	ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+				       CTDB_CONTROL_TRAVERSE_ALL,
+				       0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+	if (ret != 0) {
+		talloc_free(state);
+		return NULL;
+	}
+
+	/* timeout the traverse */
+	event_add_timed(ctdb->ev, state, 
+			timeval_current_ofs(ctdb->tunable.traverse_timeout, 0), 
+			ctdb_traverse_all_timeout, state);
+
+	return state;
+}
+
+struct traverse_all_state {
+	struct ctdb_context *ctdb;
+	struct ctdb_traverse_local_handle *h;
+	uint32_t reqid;
+	uint32_t srcnode;
+};
+
+/*
+  called for each record during a traverse all 
+ */
+static void traverse_all_callback(void *p, TDB_DATA key, TDB_DATA data)
+{
+	struct traverse_all_state *state = talloc_get_type(p, struct traverse_all_state);
+	int ret;
+	struct ctdb_rec_data *d;
+	TDB_DATA cdata;
+
+	d = ctdb_marshall_record(state, state->reqid, key, data);
+	if (d == NULL) {
+		/* darn .... */
+		DEBUG(0,("Out of memory in traverse_all_callback\n"));
+		return;
+	}
+
+	cdata.dptr = (uint8_t *)d;
+	cdata.dsize = d->length;
+
+	ret = ctdb_daemon_send_control(state->ctdb, state->srcnode, 0, CTDB_CONTROL_TRAVERSE_DATA,
+				       0, CTDB_CTRL_FLAG_NOREPLY, cdata, NULL, NULL);
+	if (ret != 0) {
+		DEBUG(0,("Failed to send traverse data\n"));
+	}
+
+	if (key.dsize == 0 && data.dsize == 0) {
+		/* we're done */
+		talloc_free(state);
+	}
+}
+
+/*
+  called when a CTDB_CONTROL_TRAVERSE_ALL control comes in. We then
+  setup a traverse of our local ltdb, sending the records as
+  CTDB_CONTROL_TRAVERSE_DATA records back to the originator
+ */
+int32_t ctdb_control_traverse_all(struct ctdb_context *ctdb, TDB_DATA data, TDB_DATA *outdata)
+{
+	struct ctdb_traverse_all *c = (struct ctdb_traverse_all *)data.dptr;
+	struct traverse_all_state *state;
+	struct ctdb_db_context *ctdb_db;
+
+	if (data.dsize != sizeof(struct ctdb_traverse_all)) {
+		DEBUG(0,("Invalid size in ctdb_control_traverse_all\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, c->db_id);
+	if (ctdb_db == NULL) {
+		return -1;
+	}
+
+	state = talloc(ctdb_db, struct traverse_all_state);
+	if (state == NULL) {
+		return -1;
+	}
+
+	state->reqid = c->reqid;
+	state->srcnode = c->vnn;
+	state->ctdb = ctdb;
+
+	state->h = ctdb_traverse_local(ctdb_db, traverse_all_callback, state);
+	if (state->h == NULL) {
+		talloc_free(state);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/*
+  called when a CTDB_CONTROL_TRAVERSE_DATA control comes in. We then
+  call the traverse_all callback with the record
+ */
+int32_t ctdb_control_traverse_data(struct ctdb_context *ctdb, TDB_DATA data, TDB_DATA *outdata)
+{
+	struct ctdb_rec_data *d = (struct ctdb_rec_data *)data.dptr;
+	struct ctdb_traverse_all_handle *state;
+	TDB_DATA key;
+	ctdb_traverse_fn_t callback;
+	void *private_data;
+
+	if (data.dsize < sizeof(uint32_t) || data.dsize != d->length) {
+		DEBUG(0,("Bad record size in ctdb_control_traverse_data\n"));
+		return -1;
+	}
+
+	state = ctdb_reqid_find(ctdb, d->reqid, struct ctdb_traverse_all_handle);
+	if (state == NULL || d->reqid != state->reqid) {
+		/* traverse might have been terminated already */
+		return -1;
+	}
+
+	key.dsize = d->keylen;
+	key.dptr  = &d->data[0];
+	data.dsize = d->datalen;
+	data.dptr = &d->data[d->keylen];
+
+	if (key.dsize == 0 && data.dsize == 0) {
+		state->null_count++;
+		if (state->null_count != ctdb_get_num_active_nodes(ctdb)) {
+			return 0;
+		}
+	}
+
+	callback = state->callback;
+	private_data = state->private_data;
+
+	callback(private_data, key, data);
+	if (key.dsize == 0 && data.dsize == 0) {
+		/* we've received all of the null replies, so all
+		   nodes are finished */
+		talloc_free(state);
+	}
+	return 0;
+}	
+
+struct traverse_start_state {
+	struct ctdb_context *ctdb;
+	struct ctdb_traverse_all_handle *h;
+	uint32_t srcnode;
+	uint32_t reqid;
+	uint64_t srvid;
+};
+
+/*
+  callback which sends records as messages to the client
+ */
+static void traverse_start_callback(void *p, TDB_DATA key, TDB_DATA data)
+{
+	struct traverse_start_state *state;
+	struct ctdb_rec_data *d;
+	TDB_DATA cdata;
+
+	state = talloc_get_type(p, struct traverse_start_state);
+
+	d = ctdb_marshall_record(state, state->reqid, key, data);
+	if (d == NULL) {
+		return;
+	}
+
+	cdata.dptr = (uint8_t *)d;
+	cdata.dsize = d->length;
+
+	ctdb_dispatch_message(state->ctdb, state->srvid, cdata);
+	if (key.dsize == 0 && data.dsize == 0) {
+		/* end of traverse */
+		talloc_free(state);
+	}
+}
+
+/*
+  start a traverse_all - called as a control from a client
+ */
+int32_t ctdb_control_traverse_start(struct ctdb_context *ctdb, TDB_DATA data, 
+				    TDB_DATA *outdata, uint32_t srcnode)
+{
+	struct ctdb_traverse_start *d = (struct ctdb_traverse_start *)data.dptr;
+	struct traverse_start_state *state;
+	struct ctdb_db_context *ctdb_db;
+
+	if (data.dsize != sizeof(*d)) {
+		DEBUG(0,("Bad record size in ctdb_control_traverse_start\n"));
+		return -1;
+	}
+
+	ctdb_db = find_ctdb_db(ctdb, d->db_id);
+	if (ctdb_db == NULL) {
+		return -1;
+	}
+
+	state = talloc(ctdb_db, struct traverse_start_state);
+	if (state == NULL) {
+		return -1;
+	}
+	
+	state->srcnode = srcnode;
+	state->reqid = d->reqid;
+	state->srvid = d->srvid;
+	state->ctdb = ctdb;
+
+	state->h = ctdb_daemon_traverse_all(ctdb_db, traverse_start_callback, state);
+	if (state->h == NULL) {
+		talloc_free(state);
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/source4/cluster/ctdb/server/ctdb_tunables.c b/source4/cluster/ctdb/server/ctdb_tunables.c
new file mode 100644
index 0000000000..491c965656
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdb_tunables.c
@@ -0,0 +1,163 @@
+/* 
+   ctdb tunables code
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "includes.h"
+#include "../include/ctdb_private.h"
+
+static const struct {
+	const char *name;
+	uint32_t default_v;
+	size_t offset;	
+} tunable_map[] = {
+	{ "MaxRedirectCount",     3,  offsetof(struct ctdb_tunable, max_redirect_count) },
+	{ "SeqnumFrequency",      1,  offsetof(struct ctdb_tunable, seqnum_frequency) },
+	{ "ControlTimeout",      60, offsetof(struct ctdb_tunable, control_timeout) },
+	{ "TraverseTimeout",     20, offsetof(struct ctdb_tunable, traverse_timeout) },
+	{ "KeepaliveInterval",    2,  offsetof(struct ctdb_tunable, keepalive_interval) },
+	{ "KeepaliveLimit",       5,  offsetof(struct ctdb_tunable, keepalive_limit) },
+	{ "MaxLACount",           7,  offsetof(struct ctdb_tunable, max_lacount) },
+	{ "RecoverTimeout",       5,  offsetof(struct ctdb_tunable, recover_timeout) },
+	{ "RecoverInterval",      1,  offsetof(struct ctdb_tunable, recover_interval) },
+	{ "ElectionTimeout",      3,  offsetof(struct ctdb_tunable, election_timeout) },
+	{ "TakeoverTimeout",      5,  offsetof(struct ctdb_tunable, takeover_timeout) },
+	{ "MonitorInterval",     15,  offsetof(struct ctdb_tunable, monitor_interval) },
+	{ "EventScriptTimeout",  20,  offsetof(struct ctdb_tunable, script_timeout) },
+	{ "RecoveryGracePeriod", 60,  offsetof(struct ctdb_tunable, recovery_grace_period) },
+	{ "RecoveryBanPeriod",  300,  offsetof(struct ctdb_tunable, recovery_ban_period) },
+	{ "DatabaseHashSize", 10000,  offsetof(struct ctdb_tunable, database_hash_size) },
+	{ "RerecoveryTimeout",   10,  offsetof(struct ctdb_tunable, rerecovery_timeout) },
+};
+
+/*
+  set all tunables to defaults
+ */
+void ctdb_tunables_set_defaults(struct ctdb_context *ctdb)
+{
+	int i;
+	for (i=0;i<ARRAY_SIZE(tunable_map);i++) {
+		*(uint32_t *)(tunable_map[i].offset + (uint8_t*)&ctdb->tunable) = tunable_map[i].default_v;
+	}
+}
+
+
+/*
+  get a tunable
+ */
+int32_t ctdb_control_get_tunable(struct ctdb_context *ctdb, TDB_DATA indata, 
+				 TDB_DATA *outdata)
+{
+	struct ctdb_control_get_tunable *t = 
+		(struct ctdb_control_get_tunable *)indata.dptr;
+	char *name;
+	uint32_t val;
+	int i;
+
+	if (indata.dsize < sizeof(*t) ||
+	    t->length > indata.dsize - offsetof(struct ctdb_control_get_tunable, name)) {
+		DEBUG(0,("Bad indata in ctdb_control_get_tunable\n"));
+		return -1;
+	}
+
+	name = talloc_strndup(ctdb, (char*)t->name, t->length);
+	CTDB_NO_MEMORY(ctdb, name);
+
+	for (i=0;i<ARRAY_SIZE(tunable_map);i++) {
+		if (strcasecmp(name, tunable_map[i].name) == 0) break;
+	}
+	talloc_free(name);
+	
+	if (i == ARRAY_SIZE(tunable_map)) {
+		return -1;
+	}
+
+	val = *(uint32_t *)(tunable_map[i].offset + (uint8_t*)&ctdb->tunable);
+
+	outdata->dptr = (uint8_t *)talloc(outdata, uint32_t);
+	CTDB_NO_MEMORY(ctdb, outdata->dptr);
+
+	*(uint32_t *)outdata->dptr = val;
+	outdata->dsize = sizeof(uint32_t);
+
+	return 0;
+}
+
+
+/*
+  set a tunable
+ */
+int32_t ctdb_control_set_tunable(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_set_tunable *t = 
+		(struct ctdb_control_set_tunable *)indata.dptr;
+	char *name;
+	int i;
+
+	if (indata.dsize < sizeof(*t) ||
+	    t->length > indata.dsize - offsetof(struct ctdb_control_set_tunable, name)) {
+		DEBUG(0,("Bad indata in ctdb_control_set_tunable\n"));
+		return -1;
+	}
+
+	name = talloc_strndup(ctdb, (char *)t->name, t->length);
+	CTDB_NO_MEMORY(ctdb, name);
+
+	for (i=0;i<ARRAY_SIZE(tunable_map);i++) {
+		if (strcasecmp(name, tunable_map[i].name) == 0) break;
+	}
+
+	talloc_free(name);
+	
+	if (i == ARRAY_SIZE(tunable_map)) {
+		return -1;
+	}
+	
+	*(uint32_t *)(tunable_map[i].offset + (uint8_t*)&ctdb->tunable) = t->value;
+
+	return 0;
+}
+
+/*
+  list tunables
+ */
+int32_t ctdb_control_list_tunables(struct ctdb_context *ctdb, TDB_DATA *outdata)
+{
+	char *list = NULL;
+	int i;
+	struct ctdb_control_list_tunable *t;
+
+	list = talloc_strdup(outdata, tunable_map[0].name);
+	CTDB_NO_MEMORY(ctdb, list);
+
+	for (i=1;i<ARRAY_SIZE(tunable_map);i++) {
+		list = talloc_asprintf_append(list, ":%s", tunable_map[i].name);
+		CTDB_NO_MEMORY(ctdb, list);		
+	}
+
+	outdata->dsize = offsetof(struct ctdb_control_list_tunable, data) + 
+		strlen(list) + 1;
+	outdata->dptr = talloc_size(outdata, outdata->dsize);
+	CTDB_NO_MEMORY(ctdb, outdata->dptr);
+
+	t = (struct ctdb_control_list_tunable *)outdata->dptr;
+	t->length = strlen(list)+1;
+
+	memcpy(t->data, list, t->length);
+	talloc_free(list);
+
+	return 0;	
+}
diff --git a/source4/cluster/ctdb/server/ctdbd.c b/source4/cluster/ctdb/server/ctdbd.c
new file mode 100644
index 0000000000..2c4a23b673
--- /dev/null
+++ b/source4/cluster/ctdb/server/ctdbd.c
@@ -0,0 +1,229 @@
+/* 
+   standalone ctdb daemon
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "popt.h"
+#include "system/wait.h"
+#include "cmdline.h"
+#include "../include/ctdb_private.h"
+
+static void block_signal(int signum)
+{
+	struct sigaction act;
+
+	memset(&act, 0, sizeof(act));
+
+	act.sa_handler = SIG_IGN;
+	sigemptyset(&act.sa_mask);
+	sigaddset(&act.sa_mask, signum);
+	sigaction(signum, &act, NULL);
+}
+
+static struct {
+	const char *nlist;
+	const char *transport;
+	const char *myaddress;
+	const char *public_address_list;
+	const char *public_interface;
+	const char *event_script;
+	const char *logfile;
+	const char *recovery_lock_file;
+	const char *db_dir;
+} options = {
+	.nlist = ETCDIR "/ctdb/nodes",
+	.transport = "tcp",
+	.event_script = ETCDIR "/ctdb/events",
+	.logfile = VARDIR "/log/log.ctdb",
+	.db_dir = VARDIR "/ctdb",
+};
+
+
+/*
+  called by the transport layer when a packet comes in
+*/
+static void ctdb_recv_pkt(struct ctdb_context *ctdb, uint8_t *data, uint32_t length)
+{
+	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
+
+	ctdb->statistics.node_packets_recv++;
+
+	/* up the counter for this source node, so we know its alive */
+	if (ctdb_validate_vnn(ctdb, hdr->srcnode)) {
+		/* as a special case, redirected calls don't increment the rx_cnt */
+		if (hdr->operation != CTDB_REQ_CALL ||
+		    ((struct ctdb_req_call *)hdr)->hopcount == 0) {
+			ctdb->nodes[hdr->srcnode]->rx_cnt++;
+		}
+	}
+
+	ctdb_input_pkt(ctdb, hdr);
+}
+
+
+
+static const struct ctdb_upcalls ctdb_upcalls = {
+	.recv_pkt       = ctdb_recv_pkt,
+	.node_dead      = ctdb_node_dead,
+	.node_connected = ctdb_node_connected
+};
+
+
+
+/*
+  main program
+*/
+int main(int argc, const char *argv[])
+{
+	struct ctdb_context *ctdb;
+	int interactive = 0;
+
+	struct poptOption popt_options[] = {
+		POPT_AUTOHELP
+		POPT_CTDB_CMDLINE
+		{ "interactive", 'i', POPT_ARG_NONE, &interactive, 0, "don't fork", NULL },
+		{ "public-addresses", 0, POPT_ARG_STRING, &options.public_address_list, 0, "public address list file", "filename" },
+		{ "public-interface", 0, POPT_ARG_STRING, &options.public_interface, 0, "public interface", "interface"},
+		{ "event-script", 0, POPT_ARG_STRING, &options.event_script, 0, "event script", "filename" },
+		{ "logfile", 0, POPT_ARG_STRING, &options.logfile, 0, "log file location", "filename" },
+		{ "nlist", 0, POPT_ARG_STRING, &options.nlist, 0, "node list file", "filename" },
+		{ "listen", 0, POPT_ARG_STRING, &options.myaddress, 0, "address to listen on", "address" },
+		{ "transport", 0, POPT_ARG_STRING, &options.transport, 0, "protocol transport", NULL },
+		{ "dbdir", 0, POPT_ARG_STRING, &options.db_dir, 0, "directory for the tdb files", NULL },
+		{ "reclock", 0, POPT_ARG_STRING, &options.recovery_lock_file, 0, "location of recovery lock file", "filename" },
+		POPT_TABLEEND
+	};
+	int opt, ret;
+	const char **extra_argv;
+	int extra_argc = 0;
+	poptContext pc;
+	struct event_context *ev;
+
+	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
+
+	while ((opt = poptGetNextOpt(pc)) != -1) {
+		switch (opt) {
+		default:
+			fprintf(stderr, "Invalid option %s: %s\n", 
+				poptBadOption(pc, 0), poptStrerror(opt));
+			exit(1);
+		}
+	}
+
+	/* setup the remaining options for the main program to use */
+	extra_argv = poptGetArgs(pc);
+	if (extra_argv) {
+		extra_argv++;
+		while (extra_argv[extra_argc]) extra_argc++;
+	}
+
+	if (!options.recovery_lock_file) {
+		DEBUG(0,("You must specifiy the location of a recovery lock file with --reclock\n"));
+		exit(1);
+	}
+
+	block_signal(SIGPIPE);
+
+	ev = event_context_init(NULL);
+
+	ctdb = ctdb_cmdline_init(ev);
+
+	ctdb->recovery_mode    = CTDB_RECOVERY_NORMAL;
+	ctdb->recovery_master  = (uint32_t)-1;
+	ctdb->upcalls          = &ctdb_upcalls;
+	ctdb->idr              = idr_init(ctdb);
+	ctdb->recovery_lock_fd = -1;
+	ctdb->monitoring_mode  = CTDB_MONITORING_ACTIVE;
+
+	ctdb_tunables_set_defaults(ctdb);
+
+	ret = ctdb_set_recovery_lock_file(ctdb, options.recovery_lock_file);
+	if (ret == -1) {
+		printf("ctdb_set_recovery_lock_file failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	ret = ctdb_set_transport(ctdb, options.transport);
+	if (ret == -1) {
+		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	/* tell ctdb what address to listen on */
+	if (options.myaddress) {
+		ret = ctdb_set_address(ctdb, options.myaddress);
+		if (ret == -1) {
+			printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb));
+			exit(1);
+		}
+	}
+
+	/* tell ctdb what nodes are available */
+	ret = ctdb_set_nlist(ctdb, options.nlist);
+	if (ret == -1) {
+		printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	if (options.db_dir) {
+		ret = ctdb_set_tdb_dir(ctdb, options.db_dir);
+		if (ret == -1) {
+			printf("ctdb_set_tdb_dir failed - %s\n", ctdb_errstr(ctdb));
+			exit(1);
+		}
+	}
+
+	ret = ctdb_set_logfile(ctdb, options.logfile);
+	if (ret == -1) {
+		printf("ctdb_set_logfile to %s failed - %s\n", options.logfile, ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	if (options.public_interface) {
+		ctdb->takeover.interface = talloc_strdup(ctdb, options.public_interface);
+		CTDB_NO_MEMORY(ctdb, ctdb->takeover.interface);
+	}
+
+	if (options.public_address_list) {
+		ret = ctdb_set_public_addresses(ctdb, options.public_address_list);
+		if (ret == -1) {
+			printf("Unable to setup public address list\n");
+			exit(1);
+		}
+		ctdb->takeover.enabled = true;
+	}
+
+	ret = ctdb_set_event_script(ctdb, options.event_script);
+	if (ret == -1) {
+		printf("Unable to setup event script\n");
+		exit(1);
+	}
+
+	/* useful default logfile */
+	if (ctdb->logfile == NULL) {
+		char *name = talloc_asprintf(ctdb, "%s/log.ctdb.vnn%u", 
+					     VARDIR, ctdb->vnn);
+		ctdb_set_logfile(ctdb, name);
+		talloc_free(name);
+	}
+
+	/* start the protocol running (as a child) */
+	return ctdb_start_daemon(ctdb, interactive?False:True);
+}
diff --git a/source4/cluster/ctdb/server/eventscript.c b/source4/cluster/ctdb/server/eventscript.c
new file mode 100644
index 0000000000..e23157056c
--- /dev/null
+++ b/source4/cluster/ctdb/server/eventscript.c
@@ -0,0 +1,191 @@
+/* 
+   event script handling
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+#include "lib/events/events.h"
+
+/*
+  run the event script - varargs version
+ */
+static int ctdb_event_script_v(struct ctdb_context *ctdb, const char *fmt, va_list ap)
+{
+	char *options, *cmdstr;
+	int ret;
+	va_list ap2;
+	struct stat st;
+
+	if (stat(ctdb->takeover.event_script, &st) != 0 && 
+	    errno == ENOENT) {
+		DEBUG(0,("No event script found at '%s'\n", ctdb->takeover.event_script));
+		return 0;
+	}
+
+	va_copy(ap2, ap);
+	options  = talloc_vasprintf(ctdb, fmt, ap2);
+	va_end(ap2);
+	CTDB_NO_MEMORY(ctdb, options);
+
+	cmdstr = talloc_asprintf(ctdb, "%s %s", ctdb->takeover.event_script, options);
+	CTDB_NO_MEMORY(ctdb, cmdstr);
+
+	ret = system(cmdstr);
+	if (ret != -1) {
+		ret = WEXITSTATUS(ret);
+	}
+
+	talloc_free(cmdstr);
+	talloc_free(options);
+
+	return ret;
+}
+
+/*
+  run the event script
+ */
+int ctdb_event_script(struct ctdb_context *ctdb, const char *fmt, ...)
+{
+	va_list ap;
+	int ret;
+
+	va_start(ap, fmt);
+	ret = ctdb_event_script_v(ctdb, fmt, ap);
+	va_end(ap);
+
+	return ret;
+}
+
+
+struct ctdb_event_script_state {
+	struct ctdb_context *ctdb;
+	pid_t child;
+	void (*callback)(struct ctdb_context *, int, void *);
+	int fd[2];
+	void *private_data;
+};
+
+/* called when child is finished */
+static void ctdb_event_script_handler(struct event_context *ev, struct fd_event *fde, 
+				      uint16_t flags, void *p)
+{
+	struct ctdb_event_script_state *state = 
+		talloc_get_type(p, struct ctdb_event_script_state);
+	int status = -1;
+	void (*callback)(struct ctdb_context *, int, void *) = state->callback;
+	void *private_data = state->private_data;
+	struct ctdb_context *ctdb = state->ctdb;
+
+	waitpid(state->child, &status, 0);
+	if (status != -1) {
+		status = WEXITSTATUS(status);
+	}
+	talloc_set_destructor(state, NULL);
+	talloc_free(state);
+	callback(ctdb, status, private_data);
+}
+
+
+/* called when child times out */
+static void ctdb_event_script_timeout(struct event_context *ev, struct timed_event *te, 
+				      struct timeval t, void *p)
+{
+	struct ctdb_event_script_state *state = talloc_get_type(p, struct ctdb_event_script_state);
+	void (*callback)(struct ctdb_context *, int, void *) = state->callback;
+	void *private_data = state->private_data;
+	struct ctdb_context *ctdb = state->ctdb;
+
+	DEBUG(0,("event script timed out\n"));
+	talloc_free(state);
+	callback(ctdb, -1, private_data);
+}
+
+/*
+  destroy a running event script
+ */
+static int event_script_destructor(struct ctdb_event_script_state *state)
+{
+	kill(state->child, SIGKILL);
+	waitpid(state->child, NULL, 0);
+	return 0;
+}
+
+/*
+  run the event script in the background, calling the callback when 
+  finished
+ */
+int ctdb_event_script_callback(struct ctdb_context *ctdb, 
+			       struct timeval timeout,
+			       TALLOC_CTX *mem_ctx,
+			       void (*callback)(struct ctdb_context *, int, void *),
+			       void *private_data,
+			       const char *fmt, ...)
+{
+	struct ctdb_event_script_state *state;
+	va_list ap;
+	int ret;
+
+	state = talloc(mem_ctx, struct ctdb_event_script_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	state->ctdb = ctdb;
+	state->callback = callback;
+	state->private_data = private_data;
+	
+	ret = pipe(state->fd);
+	if (ret != 0) {
+		talloc_free(state);
+		return -1;
+	}
+
+	state->child = fork();
+
+	if (state->child == (pid_t)-1) {
+		close(state->fd[0]);
+		close(state->fd[1]);
+		talloc_free(state);
+		return -1;
+	}
+
+	if (state->child == 0) {
+		close(state->fd[0]);
+		ctdb_set_realtime(false);
+		set_close_on_exec(state->fd[1]);
+		va_start(ap, fmt);
+		ret = ctdb_event_script_v(ctdb, fmt, ap);
+		va_end(ap);
+		_exit(ret);
+	}
+
+	talloc_set_destructor(state, event_script_destructor);
+
+	close(state->fd[1]);
+
+	event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+		     ctdb_event_script_handler, state);
+
+	if (!timeval_is_zero(&timeout)) {
+		event_add_timed(ctdb->ev, state, timeout, ctdb_event_script_timeout, state);
+	}
+
+	return 0;
+}
+
+
diff --git a/source4/cluster/ctdb/takeover/ctdb_takeover.c b/source4/cluster/ctdb/takeover/ctdb_takeover.c
new file mode 100644
index 0000000000..862382d90a
--- /dev/null
+++ b/source4/cluster/ctdb/takeover/ctdb_takeover.c
@@ -0,0 +1,583 @@
+/* 
+   ctdb recovery code
+
+   Copyright (C) Ronnie Sahlberg  2007
+   Copyright (C) Andrew Tridgell  2007
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+
+
+#define TAKEOVER_TIMEOUT() timeval_current_ofs(5,0)
+
+#define CTDB_ARP_INTERVAL 1
+#define CTDB_ARP_REPEAT   3
+
+struct ctdb_takeover_arp {
+	struct ctdb_context *ctdb;
+	uint32_t count;
+	struct sockaddr_in sin;
+	struct ctdb_tcp_list *tcp_list;
+};
+
+/*
+  lists of tcp endpoints
+ */
+struct ctdb_tcp_list {
+	struct ctdb_tcp_list *prev, *next;
+	uint32_t vnn;
+	struct sockaddr_in saddr;
+	struct sockaddr_in daddr;
+};
+
+
+
+/*
+  send a gratuitous arp
+ */
+static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *te, 
+				  struct timeval t, void *private_data)
+{
+	struct ctdb_takeover_arp *arp = talloc_get_type(private_data, 
+							struct ctdb_takeover_arp);
+	int ret;
+	struct ctdb_tcp_list *tcp;
+
+	ret = ctdb_sys_send_arp(&arp->sin, arp->ctdb->takeover.interface);
+	if (ret != 0) {
+		DEBUG(0,(__location__ "sending of arp failed (%s)\n", strerror(errno)));
+	}
+
+	for (tcp=arp->tcp_list;tcp;tcp=tcp->next) {
+		DEBUG(2,("sending tcp tickle ack for %u->%s:%u\n",
+			 (unsigned)ntohs(tcp->daddr.sin_port), 
+			 inet_ntoa(tcp->saddr.sin_addr),
+			 (unsigned)ntohs(tcp->saddr.sin_port)));
+		ret = ctdb_sys_send_ack(&tcp->saddr, &tcp->daddr);
+		if (ret != 0) {
+			DEBUG(0,(__location__ " Failed to send tcp tickle ack for %s\n",
+				 inet_ntoa(tcp->saddr.sin_addr)));
+		}
+	}
+
+	arp->count++;
+
+	if (arp->count == CTDB_ARP_REPEAT) {
+		talloc_free(arp);
+		return;
+	}
+	
+	event_add_timed(arp->ctdb->ev, arp->ctdb->takeover.last_ctx, 
+			timeval_current_ofs(CTDB_ARP_INTERVAL, 0), 
+			ctdb_control_send_arp, arp);
+}
+
+
+/*
+  take over an ip address
+ */
+int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	int ret;
+	struct sockaddr_in *sin = (struct sockaddr_in *)indata.dptr;
+	struct ctdb_takeover_arp *arp;
+	char *ip = inet_ntoa(sin->sin_addr);
+	struct ctdb_tcp_list *tcp;
+
+	if (ctdb_sys_have_ip(ip)) {
+		return 0;
+	}
+
+	DEBUG(0,("Takover of IP %s/%u on interface %s\n", 
+		 ip, ctdb->nodes[ctdb->vnn]->public_netmask_bits, 
+		 ctdb->takeover.interface));
+	ret = ctdb_event_script(ctdb, "takeip %s %s %u",
+				ctdb->takeover.interface, 
+				ip,
+				ctdb->nodes[ctdb->vnn]->public_netmask_bits);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to takeover IP %s on interface %s\n",
+			 ip, ctdb->takeover.interface));
+		return -1;
+	}
+
+	if (!ctdb->takeover.last_ctx) {
+		ctdb->takeover.last_ctx = talloc_new(ctdb);
+		CTDB_NO_MEMORY(ctdb, ctdb->takeover.last_ctx);
+	}
+
+	arp = talloc_zero(ctdb->takeover.last_ctx, struct ctdb_takeover_arp);
+	CTDB_NO_MEMORY(ctdb, arp);
+	
+	arp->ctdb = ctdb;
+	arp->sin = *sin;
+
+	/* add all of the known tcp connections for this IP to the
+	   list of tcp connections to send tickle acks for */
+	for (tcp=ctdb->tcp_list;tcp;tcp=tcp->next) {
+		if (sin->sin_addr.s_addr == tcp->daddr.sin_addr.s_addr) {
+			struct ctdb_tcp_list *t2 = talloc(arp, struct ctdb_tcp_list);
+			CTDB_NO_MEMORY(ctdb, t2);
+			*t2 = *tcp;
+			DLIST_ADD(arp->tcp_list, t2);
+		}
+	}
+
+	event_add_timed(arp->ctdb->ev, arp->ctdb->takeover.last_ctx, 
+			timeval_zero(), ctdb_control_send_arp, arp);
+
+	return ret;
+}
+
+/*
+  release an ip address
+ */
+int32_t ctdb_control_release_ip(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct sockaddr_in *sin = (struct sockaddr_in *)indata.dptr;
+	TDB_DATA data;
+	char *ip = inet_ntoa(sin->sin_addr);
+	int ret;
+	struct ctdb_tcp_list *tcp;
+
+	if (!ctdb_sys_have_ip(ip)) {
+		return 0;
+	}
+
+	DEBUG(0,("Release of IP %s/%u on interface %s\n", 
+		 ip, ctdb->nodes[ctdb->vnn]->public_netmask_bits, 
+		 ctdb->takeover.interface));
+
+	/* stop any previous arps */
+	talloc_free(ctdb->takeover.last_ctx);
+	ctdb->takeover.last_ctx = NULL;
+
+	ret = ctdb_event_script(ctdb, "releaseip %s %s %u",
+				ctdb->takeover.interface, 
+				ip,
+				ctdb->nodes[ctdb->vnn]->public_netmask_bits);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to release IP %s on interface %s\n",
+			 ip, ctdb->takeover.interface));
+		return -1;
+	}
+
+	/* send a message to all clients of this node telling them
+	   that the cluster has been reconfigured and they should
+	   release any sockets on this IP */
+	data.dptr = (uint8_t *)ip;
+	data.dsize = strlen(ip)+1;
+
+	ctdb_daemon_send_message(ctdb, ctdb->vnn, CTDB_SRVID_RELEASE_IP, data);
+
+	/* tell other nodes about any tcp connections we were holding with this IP */
+	for (tcp=ctdb->tcp_list;tcp;tcp=tcp->next) {
+		if (tcp->vnn == ctdb->vnn && 
+		    sin->sin_addr.s_addr == tcp->daddr.sin_addr.s_addr) {
+			struct ctdb_control_tcp_vnn t;
+
+			t.vnn  = ctdb->vnn;
+			t.src  = tcp->saddr;
+			t.dest = tcp->daddr;
+
+			data.dptr = (uint8_t *)&t;
+			data.dsize = sizeof(t);
+
+			ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+						 CTDB_CONTROL_TCP_ADD,
+						 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		}
+	}
+
+
+	return 0;
+}
+
+
+/*
+  setup the event script
+*/
+int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script)
+{
+	ctdb->takeover.event_script = talloc_strdup(ctdb, script);
+	CTDB_NO_MEMORY(ctdb, ctdb->takeover.event_script);
+	return 0;
+}
+
+/*
+  setup the public address list from a file
+*/
+int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist)
+{
+	char **lines;
+	int nlines;
+	int i;
+
+	lines = file_lines_load(alist, &nlines, ctdb);
+	if (lines == NULL) {
+		ctdb_set_error(ctdb, "Failed to load public address list '%s'\n", alist);
+		return -1;
+	}
+	while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
+		nlines--;
+	}
+
+	if (nlines != ctdb->num_nodes) {
+		DEBUG(0,("Number of lines in %s does not match number of nodes!\n", alist));
+		talloc_free(lines);
+		return -1;
+	}
+
+	for (i=0;i<nlines;i++) {
+		char *p;
+		struct in_addr in;
+
+		ctdb->nodes[i]->public_address = talloc_strdup(ctdb->nodes[i], lines[i]);
+		CTDB_NO_MEMORY(ctdb, ctdb->nodes[i]->public_address);
+		ctdb->nodes[i]->takeover_vnn = -1;
+
+		/* see if they supplied a netmask length */
+		p = strchr(ctdb->nodes[i]->public_address, '/');
+		if (!p) {
+			DEBUG(0,("You must supply a netmask for public address %s\n",
+				 ctdb->nodes[i]->public_address));
+			return -1;
+		}
+		*p = 0;
+		ctdb->nodes[i]->public_netmask_bits = atoi(p+1);
+
+		if (ctdb->nodes[i]->public_netmask_bits > 32) {
+			DEBUG(0, ("Illegal netmask for IP %s\n", ctdb->nodes[i]->public_address));
+			return -1;
+		}
+
+		if (inet_aton(ctdb->nodes[i]->public_address, &in) == 0) {
+			DEBUG(0,("Badly formed IP '%s' in public address list\n", ctdb->nodes[i]->public_address));
+			return -1;
+		}
+	}
+
+	talloc_free(lines);
+	return 0;
+}
+
+/*
+  see if two IPs are on the same subnet
+ */
+static bool ctdb_same_subnet(const char *ip1, const char *ip2, uint8_t netmask_bits)
+{
+	struct in_addr in1, in2;
+	uint32_t mask;
+
+	inet_aton(ip1, &in1);
+	inet_aton(ip2, &in2);
+
+	mask = ~((1LL<<(32-netmask_bits))-1);
+
+	if ((ntohl(in1.s_addr) & mask) != (ntohl(in2.s_addr) & mask)) {
+		return false;
+	}
+
+	return true;
+}
+
+/*
+  make any IP alias changes for public addresses that are necessary 
+ */
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
+{
+	int i, j;
+	int ret;
+
+	/* work out which node will look after each public IP */
+	for (i=0;i<nodemap->num;i++) {
+		if (nodemap->nodes[i].flags & NODE_FLAGS_CONNECTED) {
+			ctdb->nodes[i]->takeover_vnn = nodemap->nodes[i].vnn;
+		} else {
+			/* assign this dead nodes IP to the next higher node */
+			for (j=(i+1)%nodemap->num;
+			     j != i;
+			     j=(j+1)%nodemap->num) {
+				if ((nodemap->nodes[j].flags & NODE_FLAGS_CONNECTED) &&
+				    ctdb_same_subnet(ctdb->nodes[j]->public_address, 
+						     ctdb->nodes[i]->public_address, 
+						     ctdb->nodes[j]->public_netmask_bits)) {
+					ctdb->nodes[i]->takeover_vnn = nodemap->nodes[j].vnn;
+					break;
+				}
+			}
+			if (j == i) {
+				DEBUG(0,(__location__ " No node available on same network to take %s\n",
+					 ctdb->nodes[i]->public_address));
+				ctdb->nodes[i]->takeover_vnn = -1;	
+			}
+		}
+	}	
+
+	/* at this point ctdb->nodes[i]->takeover_vnn is the vnn which will own each IP */
+
+
+	/* now tell all nodes to delete any alias that they should not
+	   have.  This will be a NOOP on nodes that don't currently
+	   hold the given alias */
+	for (i=0;i<nodemap->num;i++) {
+		/* don't talk to unconnected nodes */
+		if (!(nodemap->nodes[i].flags & NODE_FLAGS_CONNECTED)) continue;
+
+		/* tell this node to delete all of the aliases that it should not have */
+		for (j=0;j<nodemap->num;j++) {
+			if (ctdb->nodes[j]->takeover_vnn != nodemap->nodes[i].vnn) {
+				ret = ctdb_ctrl_release_ip(ctdb, TAKEOVER_TIMEOUT(),
+							   nodemap->nodes[i].vnn, 
+							   ctdb->nodes[j]->public_address);
+				if (ret != 0) {
+					DEBUG(0,("Failed to tell vnn %u to release IP %s\n",
+						 nodemap->nodes[i].vnn,
+						 ctdb->nodes[j]->public_address));
+					return -1;
+				}
+			}
+		}
+	}
+
+	/* tell all nodes to get their own IPs */
+	for (i=0;i<nodemap->num;i++) {
+		ret = ctdb_ctrl_takeover_ip(ctdb, TAKEOVER_TIMEOUT(), 
+					    ctdb->nodes[i]->takeover_vnn, 
+					    ctdb->nodes[i]->public_address);
+		if (ret != 0) {
+			DEBUG(0,("Failed asking vnn %u to take over IP %s\n",
+				 ctdb->nodes[i]->takeover_vnn, 
+				 ctdb->nodes[i]->public_address));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+
+/*
+  called by a client to inform us of a TCP connection that it is managing
+  that should tickled with an ACK when IP takeover is done
+ */
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, uint32_t vnn,
+				TDB_DATA indata)
+{
+	struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+	struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+	struct ctdb_tcp_list *tcp;
+	struct ctdb_control_tcp_vnn t;
+	int ret;
+	TDB_DATA data;
+
+	tcp = talloc(client, struct ctdb_tcp_list);
+	CTDB_NO_MEMORY(ctdb, tcp);
+
+	tcp->vnn   = vnn;
+	tcp->saddr = p->src;
+	tcp->daddr = p->dest;
+
+	DLIST_ADD(client->tcp_list, tcp);
+
+	t.vnn  = vnn;
+	t.src  = p->src;
+	t.dest = p->dest;
+
+	data.dptr = (uint8_t *)&t;
+	data.dsize = sizeof(t);
+
+	/* tell all nodes about this tcp connection */
+	ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+				       CTDB_CONTROL_TCP_ADD,
+				       0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  see if two sockaddr_in are the same
+ */
+static bool same_sockaddr_in(struct sockaddr_in *in1, struct sockaddr_in *in2)
+{
+	return in1->sin_family == in2->sin_family &&
+		in1->sin_port == in2->sin_port &&
+		in1->sin_addr.s_addr == in2->sin_addr.s_addr;
+}
+
+/*
+  find a tcp address on a list
+ */
+static struct ctdb_tcp_list *ctdb_tcp_find(struct ctdb_tcp_list *list, 
+					   struct ctdb_tcp_list *tcp)
+{
+	while (list) {
+		if (same_sockaddr_in(&list->saddr, &tcp->saddr) &&
+		    same_sockaddr_in(&list->daddr, &tcp->daddr)) {
+			return list;
+		}
+		list = list->next;
+	}
+	return NULL;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_tcp_vnn *p = (struct ctdb_control_tcp_vnn *)indata.dptr;
+	struct ctdb_tcp_list *tcp;
+
+	tcp = talloc(ctdb, struct ctdb_tcp_list);
+	CTDB_NO_MEMORY(ctdb, tcp);
+
+	tcp->vnn   = p->vnn;
+	tcp->saddr = p->src;
+	tcp->daddr = p->dest;
+
+	if (NULL == ctdb_tcp_find(ctdb->tcp_list, tcp)) {
+		DLIST_ADD(ctdb->tcp_list, tcp);
+		DEBUG(2,("Added tickle info for %s:%u from vnn %u\n",
+			 inet_ntoa(tcp->daddr.sin_addr), ntohs(tcp->daddr.sin_port),
+			 tcp->vnn));
+	} else {
+		DEBUG(4,("Already had tickle info for %s:%u from vnn %u\n",
+			 inet_ntoa(tcp->daddr.sin_addr), ntohs(tcp->daddr.sin_port),
+			 tcp->vnn));
+	}
+
+	return 0;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_tcp_vnn *p = (struct ctdb_control_tcp_vnn *)indata.dptr;
+	struct ctdb_tcp_list t, *tcp;
+
+	t.vnn   = p->vnn;
+	t.saddr = p->src;
+	t.daddr = p->dest;
+
+	tcp = ctdb_tcp_find(ctdb->tcp_list, &t);
+	if (tcp) {
+		DEBUG(2,("Removed tickle info for %s:%u from vnn %u\n",
+			 inet_ntoa(tcp->daddr.sin_addr), ntohs(tcp->daddr.sin_port),
+			 tcp->vnn));
+		DLIST_REMOVE(ctdb->tcp_list, tcp);
+		talloc_free(tcp);
+	}
+
+	return 0;
+}
+
+
+/*
+  called when a daemon restarts - wipes all tcp entries from that vnn
+ */
+int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn)
+{
+	struct ctdb_tcp_list *tcp, *next;	
+	for (tcp=ctdb->tcp_list;tcp;tcp=next) {
+		next = tcp->next;
+		if (tcp->vnn == vnn) {
+			DLIST_REMOVE(ctdb->tcp_list, tcp);
+			talloc_free(tcp);
+		}
+
+		/* and tell the new guy about any that he should have
+		   from us */
+		if (tcp->vnn == ctdb->vnn) {
+			struct ctdb_control_tcp_vnn t;
+			TDB_DATA data;
+
+			t.vnn  = tcp->vnn;
+			t.src  = tcp->saddr;
+			t.dest = tcp->daddr;
+
+			data.dptr = (uint8_t *)&t;
+			data.dsize = sizeof(t);
+
+			ctdb_daemon_send_control(ctdb, vnn, 0, 
+						 CTDB_CONTROL_TCP_ADD,
+						 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		}
+	}
+	return 0;
+}
+
+
+/*
+  called when a client structure goes away - hook to remove
+  elements from the tcp_list in all daemons
+ */
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
+{
+	while (client->tcp_list) {
+		TDB_DATA data;
+		struct ctdb_control_tcp_vnn p;
+		struct ctdb_tcp_list *tcp = client->tcp_list;
+		DLIST_REMOVE(client->tcp_list, tcp);
+		p.vnn = tcp->vnn;
+		p.src = tcp->saddr;
+		p.dest = tcp->daddr;
+		data.dptr = (uint8_t *)&p;
+		data.dsize = sizeof(p);
+		ctdb_daemon_send_control(client->ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+					 CTDB_CONTROL_TCP_REMOVE,
+					 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		talloc_free(tcp);
+	}
+}
+
+
+/*
+  release all IPs on shutdown
+ */
+void ctdb_release_all_ips(struct ctdb_context *ctdb)
+{
+	int i;
+
+	if (!ctdb->takeover.enabled) {
+		return;
+	}
+
+	for (i=0;i<ctdb->num_nodes;i++) {
+		struct ctdb_node *node = ctdb->nodes[i];
+		if (ctdb_sys_have_ip(node->public_address)) {
+			ctdb_event_script(ctdb, "releaseip %s %s %u",
+					  ctdb->takeover.interface, 
+					  node->public_address,
+					  node->public_netmask_bits);
+		}
+	}
+}
diff --git a/source4/cluster/ctdb/takeover/system.c b/source4/cluster/ctdb/takeover/system.c
new file mode 100644
index 0000000000..13f3d1501b
--- /dev/null
+++ b/source4/cluster/ctdb/takeover/system.c
@@ -0,0 +1,381 @@
+/* 
+   ctdb recovery code
+
+   Copyright (C) Ronnie Sahlberg  2007
+   Copyright (C) Andrew Tridgell  2007
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "system/network.h"
+#include "system/filesys.h"
+#include "system/wait.h"
+#include "../include/ctdb_private.h"
+#include "lib/events/events.h"
+#include <net/ethernet.h>
+#include <net/if_arp.h>
+
+
+
+/*
+  send gratuitous arp reply after we have taken over an ip address
+
+  saddr is the address we are trying to claim
+  iface is the interface name we will be using to claim the address
+ */
+int ctdb_sys_send_arp(const struct sockaddr_in *saddr, const char *iface)
+{
+	int s, ret;
+	struct sockaddr sa;
+	struct ether_header *eh;
+	struct arphdr *ah;
+	struct ifreq if_hwaddr;
+	unsigned char buffer[64]; /*minimum eth frame size */
+	char *ptr;
+
+	/* for now, we only handle AF_INET addresses */
+	if (saddr->sin_family != AF_INET) {
+		DEBUG(0,(__location__ " not an ipv4 address\n"));
+		return -1;
+	}
+
+	s = socket(AF_INET, SOCK_PACKET, htons(ETHERTYPE_ARP));
+	if (s == -1){
+		DEBUG(0,(__location__ " failed to open raw socket\n"));
+		return -1;
+	}
+
+	/* get the mac address */
+	strcpy(if_hwaddr.ifr_name, iface);
+	ret = ioctl(s, SIOCGIFHWADDR, &if_hwaddr);
+	if ( ret < 0 ) {
+		close(s);
+		DEBUG(0,(__location__ " ioctl failed\n"));
+		return -1;
+	}
+	if (if_hwaddr.ifr_hwaddr.sa_family != AF_LOCAL) {
+		close(s);
+		DEBUG(0,(__location__ " not an ethernet address\n"));
+		return -1;
+	}
+
+
+	memset(buffer, 0 , 64);
+	eh = (struct ether_header *)buffer;
+	memset(eh->ether_dhost, 0xff, ETH_ALEN);
+	memcpy(eh->ether_shost, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	eh->ether_type = htons(ETHERTYPE_ARP);
+
+	ah = (struct arphdr *)&buffer[sizeof(struct ether_header)];
+	ah->ar_hrd = htons(ARPHRD_ETHER);
+	ah->ar_pro = htons(ETH_P_IP);
+	ah->ar_hln = ETH_ALEN;
+	ah->ar_pln = 4;
+
+	/* send a gratious arp */
+	ah->ar_op  = htons(ARPOP_REQUEST);
+	ptr = (char *)&ah[1];
+	memcpy(ptr, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+	memset(ptr, 0, ETH_ALEN); 
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+
+	strncpy(sa.sa_data, iface, sizeof(sa.sa_data));
+	ret = sendto(s, buffer, 64, 0, &sa, sizeof(sa));
+	if (ret < 0 ){
+		close(s);
+		DEBUG(0,(__location__ " failed sendto\n"));
+		return -1;
+	}
+
+	/* send unsolicited arp reply broadcast */
+	ah->ar_op  = htons(ARPOP_REPLY);
+	ptr = (char *)&ah[1];
+	memcpy(ptr, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+	memcpy(ptr, if_hwaddr.ifr_hwaddr.sa_data, ETH_ALEN);
+	ptr+=ETH_ALEN;
+	memcpy(ptr, &saddr->sin_addr, 4);	  
+	ptr+=4;
+
+	strncpy(sa.sa_data, iface, sizeof(sa.sa_data));
+	ret = sendto(s, buffer, 64, 0, &sa, sizeof(sa));
+	if (ret < 0 ){
+		DEBUG(0,(__location__ " failed sendto\n"));
+		return -1;
+	}
+
+	close(s);
+	return 0;
+}
+
+
+/*
+  uint16 checksum for n bytes
+ */
+static uint32_t uint16_checksum(uint16_t *data, size_t n)
+{
+	uint32_t sum=0;
+	while (n>=2) {
+		sum += (uint32_t)ntohs(*data);
+		data++;
+		n -= 2;
+	}
+	if (n == 1) {
+		sum += (uint32_t)ntohs(*(uint8_t *)data);
+	}
+	return sum;
+}
+
+/*
+  simple TCP checksum - assumes data is multiple of 2 bytes long
+ */
+static uint16_t tcp_checksum(uint16_t *data, size_t n, struct iphdr *ip)
+{
+	uint32_t sum = uint16_checksum(data, n);
+	uint16_t sum2;
+	sum += uint16_checksum((uint16_t *)&ip->saddr, sizeof(ip->saddr));
+	sum += uint16_checksum((uint16_t *)&ip->daddr, sizeof(ip->daddr));
+	sum += ip->protocol + n;
+	sum = (sum & 0xFFFF) + (sum >> 16);
+	sum = (sum & 0xFFFF) + (sum >> 16);
+	sum2 = htons(sum);
+	sum2 = ~sum2;
+	if (sum2 == 0) {
+		return 0xFFFF;
+	}
+	return sum2;
+}
+
+/*
+  send tcp ack packet from the specified IP/port to the specified
+  destination IP/port. 
+
+  This is used to trigger the receiving host into sending its own ACK,
+  which should trigger early detection of TCP reset by the client
+  after IP takeover
+ */
+int ctdb_sys_send_ack(const struct sockaddr_in *dest, 
+		      const struct sockaddr_in *src)
+{
+	int s, ret;
+	uint32_t one = 1;
+	struct {
+		struct iphdr ip;
+		struct tcphdr tcp;
+	} pkt;
+
+	/* for now, we only handle AF_INET addresses */
+	if (src->sin_family != AF_INET || dest->sin_family != AF_INET) {
+		DEBUG(0,(__location__ " not an ipv4 address\n"));
+		return -1;
+	}
+
+	s = socket(AF_INET, SOCK_RAW, htons(IPPROTO_RAW));
+	if (s == -1) {
+		DEBUG(0,(__location__ " failed to open raw socket (%s)\n",
+			 strerror(errno)));
+		return -1;
+	}
+
+	ret = setsockopt(s, SOL_IP, IP_HDRINCL, &one, sizeof(one));
+	if (ret != 0) {
+		DEBUG(0,(__location__ " failed to setup IP headers (%s)\n",
+			 strerror(errno)));
+		close(s);
+		return -1;
+	}
+
+	ZERO_STRUCT(pkt);
+	pkt.ip.version  = 4;
+	pkt.ip.ihl      = sizeof(pkt.ip)/4;
+	pkt.ip.tot_len  = htons(sizeof(pkt));
+	pkt.ip.ttl      = 255;
+	pkt.ip.protocol = IPPROTO_TCP;
+	pkt.ip.saddr    = src->sin_addr.s_addr;
+	pkt.ip.daddr    = dest->sin_addr.s_addr;
+	pkt.ip.check    = 0;
+
+	pkt.tcp.source   = src->sin_port;
+	pkt.tcp.dest     = dest->sin_port;
+	pkt.tcp.ack      = 1;
+	pkt.tcp.doff     = sizeof(pkt.tcp)/4;
+	pkt.tcp.window   = htons(1234);
+	pkt.tcp.check    = tcp_checksum((uint16_t *)&pkt.tcp, sizeof(pkt.tcp), &pkt.ip);
+
+	ret = sendto(s, &pkt, sizeof(pkt), 0, dest, sizeof(*dest));
+	if (ret != sizeof(pkt)) {
+		DEBUG(0,(__location__ " failed sendto (%s)\n", strerror(errno)));
+		close(s);
+		return -1;
+	}
+
+	close(s);
+	return 0;
+}
+
+
+/*
+  see if we currently have an interface with the given IP
+
+  we try to bind to it, and if that fails then we don't have that IP
+  on an interface
+ */
+bool ctdb_sys_have_ip(const char *ip)
+{
+	struct sockaddr_in sin;
+	int s;
+	int ret;
+
+	sin.sin_port = 0;
+	inet_aton(ip, &sin.sin_addr);
+	sin.sin_family = AF_INET;
+	s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+	if (s == -1) {
+		return false;
+	}
+	ret = bind(s, (struct sockaddr *)&sin, sizeof(sin));
+	close(s);
+	return ret == 0;
+}
+
+/*
+  run the event script - varargs version
+ */
+static int ctdb_event_script_v(struct ctdb_context *ctdb, const char *fmt, va_list ap)
+{
+	char *options, *cmdstr;
+	int ret;
+	va_list ap2;
+	struct stat st;
+
+	if (stat(ctdb->takeover.event_script, &st) != 0 && 
+	    errno == ENOENT) {
+		DEBUG(0,("No event script found at '%s'\n", ctdb->takeover.event_script));
+		return 0;
+	}
+
+	va_copy(ap2, ap);
+	options  = talloc_vasprintf(ctdb, fmt, ap2);
+	va_end(ap2);
+	CTDB_NO_MEMORY(ctdb, options);
+
+	cmdstr = talloc_asprintf(ctdb, "%s %s", ctdb->takeover.event_script, options);
+	CTDB_NO_MEMORY(ctdb, cmdstr);
+
+	ret = system(cmdstr);
+	if (ret != -1) {
+		ret = WEXITSTATUS(ret);
+	}
+
+	talloc_free(cmdstr);
+	talloc_free(options);
+
+	return ret;
+}
+
+/*
+  run the event script
+ */
+int ctdb_event_script(struct ctdb_context *ctdb, const char *fmt, ...)
+{
+	va_list ap;
+	int ret;
+
+	va_start(ap, fmt);
+	ret = ctdb_event_script_v(ctdb, fmt, ap);
+	va_end(ap);
+
+	return ret;
+}
+
+
+struct ctdb_event_script_state {
+	struct ctdb_context *ctdb;
+	pid_t child;
+	void (*callback)(struct ctdb_context *, int);
+	int fd[2];
+};
+
+/* called when child is finished */
+static void ctdb_event_script_handler(struct event_context *ev, struct fd_event *fde, 
+				      uint16_t flags, void *p)
+{
+	struct ctdb_event_script_state *state = 
+		talloc_get_type(p, struct ctdb_event_script_state);
+	int status = -1;
+	waitpid(state->child, &status, 0);
+	if (status != -1) {
+		status = WEXITSTATUS(status);
+	}
+	state->callback(state->ctdb, status);
+	talloc_free(state);
+}
+
+
+/*
+  run the event script in the background, calling the callback when 
+  finished
+ */
+int ctdb_event_script_callback(struct ctdb_context *ctdb, 
+			       void (*callback)(struct ctdb_context *, int),
+			       const char *fmt, ...)
+{
+	struct ctdb_event_script_state *state;
+	va_list ap;
+	int ret;
+
+	state = talloc(ctdb, struct ctdb_event_script_state);
+	CTDB_NO_MEMORY(ctdb, state);
+
+	state->ctdb = ctdb;
+	state->callback = callback;
+	
+	ret = pipe(state->fd);
+	if (ret != 0) {
+		talloc_free(state);
+		return -1;
+	}
+
+	state->child = fork();
+
+	if (state->child == (pid_t)-1) {
+		close(state->fd[0]);
+		close(state->fd[1]);
+		talloc_free(state);
+		return -1;
+	}
+
+	if (state->child == 0) {
+		close(state->fd[0]);
+		va_start(ap, fmt);
+		ret = ctdb_event_script_v(ctdb, fmt, ap);
+		va_end(ap);
+		_exit(ret);
+	}
+
+	close(state->fd[1]);
+
+	event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
+		     ctdb_event_script_handler, state);
+
+	return 0;
+}
diff --git a/source4/cluster/ctdb/tcp/ctdb_tcp.h b/source4/cluster/ctdb/tcp/ctdb_tcp.h
index 6fbe145900..3a1285f47c 100644
--- a/source4/cluster/ctdb/tcp/ctdb_tcp.h
+++ b/source4/cluster/ctdb/tcp/ctdb_tcp.h
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 
@@ -37,7 +37,9 @@ struct ctdb_incoming {
 */
 struct ctdb_tcp_node {
 	int fd;
-	struct ctdb_queue *queue;
+	struct ctdb_queue *out_queue;
+	struct fd_event *connect_fde;
+	struct timed_event *connect_te;
 };
 
 
diff --git a/source4/cluster/ctdb/tcp/tcp_connect.c b/source4/cluster/ctdb/tcp/tcp_connect.c
index c0a0da5d47..2f828e5717 100644
--- a/source4/cluster/ctdb/tcp/tcp_connect.c
+++ b/source4/cluster/ctdb/tcp/tcp_connect.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -25,14 +25,6 @@
 #include "../include/ctdb_private.h"
 #include "ctdb_tcp.h"
 
-static void set_nonblocking(int fd)
-{
-	unsigned v;
-	v = fcntl(fd, F_GETFL, 0);
-        fcntl(fd, F_SETFL, v | O_NONBLOCK);
-}
-
-
 /*
   called when a complete packet has come in - should not happen on this socket
  */
@@ -48,10 +40,9 @@ void ctdb_tcp_tnode_cb(uint8_t *data, size_t cnt, void *private_data)
 
 	/* start a new connect cycle to try to re-establish the
 	   link */
-	close(tnode->fd);
-	ctdb_queue_set_fd(tnode->queue, -1);
+	ctdb_queue_set_fd(tnode->out_queue, -1);
 	tnode->fd = -1;
-	event_add_timed(node->ctdb->ev, node, timeval_zero(), 
+	event_add_timed(node->ctdb->ev, tnode, timeval_zero(), 
 			ctdb_tcp_node_connect, node);
 }
 
@@ -70,12 +61,15 @@ static void ctdb_node_connect_write(struct event_context *ev, struct fd_event *f
 	socklen_t len = sizeof(error);
 	int one = 1;
 
+	talloc_free(tnode->connect_te);
+	tnode->connect_te = NULL;
+
 	if (getsockopt(tnode->fd, SOL_SOCKET, SO_ERROR, &error, &len) != 0 ||
 	    error != 0) {
 		talloc_free(fde);
 		close(tnode->fd);
 		tnode->fd = -1;
-		event_add_timed(ctdb->ev, node, timeval_current_ofs(1, 0), 
+		event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0), 
 				ctdb_tcp_node_connect, node);
 		return;
 	}
@@ -83,8 +77,9 @@ static void ctdb_node_connect_write(struct event_context *ev, struct fd_event *f
 	talloc_free(fde);
 	
         setsockopt(tnode->fd,IPPROTO_TCP,TCP_NODELAY,(char *)&one,sizeof(one));
+        setsockopt(tnode->fd,SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(one));
 
-	ctdb_queue_set_fd(tnode->queue, tnode->fd);
+	ctdb_queue_set_fd(tnode->out_queue, tnode->fd);
 
 	/* tell the ctdb layer we are connected */
 	node->ctdb->upcalls->node_connected(node);
@@ -120,10 +115,22 @@ void ctdb_tcp_node_connect(struct event_context *ev, struct timed_event *te,
         struct sockaddr_in sock_in;
         struct sockaddr_in sock_out;
 
+	if (tnode->fd != -1) {
+		talloc_free(tnode->connect_fde);
+		tnode->connect_fde = NULL;
+		close(tnode->fd);
+		tnode->fd = -1;
+	}
+
 	tnode->fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 
 	set_nonblocking(tnode->fd);
+	set_close_on_exec(tnode->fd);
 
+	ZERO_STRUCT(sock_out);
+#ifdef HAVE_SOCK_SIN_LEN
+	sock_out.sin_len = sizeof(sock_out);
+#endif
 	if (ctdb_tcp_get_address(ctdb, node->address.address, &sock_out.sin_addr) != 0) {
 		return;
 	}
@@ -137,6 +144,10 @@ void ctdb_tcp_node_connect(struct event_context *ev, struct timed_event *te,
 	 * the remote side is actually routable in case CTDB traffic will run on
 	 * a dedicated non-routeable network.
 	 */
+	ZERO_STRUCT(sock_in);
+#ifdef HAVE_SOCK_SIN_LEN
+	sock_in.sin_len = sizeof(sock_in);
+#endif
 	if (ctdb_tcp_get_address(ctdb, ctdb->address.address, &sock_in.sin_addr) != 0) {
 		return;
 	}
@@ -148,24 +159,22 @@ void ctdb_tcp_node_connect(struct event_context *ev, struct timed_event *te,
 	    errno != EINPROGRESS) {
 		/* try again once a second */
 		close(tnode->fd);
-		event_add_timed(ctdb->ev, node, timeval_current_ofs(1, 0), 
+		tnode->fd = -1;
+		event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0), 
 				ctdb_tcp_node_connect, node);
 		return;
 	}
 
 	/* non-blocking connect - wait for write event */
-	event_add_fd(node->ctdb->ev, node, tnode->fd, EVENT_FD_WRITE|EVENT_FD_READ, 
-		     ctdb_node_connect_write, node);
-}
-
-/*
-  destroy a ctdb_incoming structure 
-*/
-static int ctdb_incoming_destructor(struct ctdb_incoming *in)
-{
-	close(in->fd);
-	in->fd = -1;
-	return 0;
+	tnode->connect_fde = event_add_fd(node->ctdb->ev, tnode, tnode->fd, 
+					  EVENT_FD_WRITE|EVENT_FD_READ, 
+					  ctdb_node_connect_write, node);
+
+	/* don't give it long to connect - retry in one second. This ensures
+	   that we find a node is up quickly (tcp normally backs off a syn reply
+	   delay by quite a lot) */
+	tnode->connect_te = event_add_timed(ctdb->ev, tnode, timeval_current_ofs(1, 0), 
+					    ctdb_tcp_node_connect, node);
 }
 
 /*
@@ -176,30 +185,129 @@ static int ctdb_incoming_destructor(struct ctdb_incoming *in)
 static void ctdb_listen_event(struct event_context *ev, struct fd_event *fde, 
 			      uint16_t flags, void *private_data)
 {
-	struct ctdb_context *ctdb;
-	struct ctdb_tcp *ctcp;
+	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
+	struct ctdb_tcp *ctcp = talloc_get_type(ctdb->private_data, struct ctdb_tcp);
 	struct sockaddr_in addr;
 	socklen_t len;
-	int fd;
+	int fd, nodeid;
 	struct ctdb_incoming *in;
+	int one = 1;
+	const char *incoming_node;
 
-	ctdb = talloc_get_type(private_data, struct ctdb_context);
-	ctcp = talloc_get_type(ctdb->private_data, struct ctdb_tcp);
 	memset(&addr, 0, sizeof(addr));
 	len = sizeof(addr);
 	fd = accept(ctcp->listen_fd, (struct sockaddr *)&addr, &len);
 	if (fd == -1) return;
 
-	in = talloc_zero(ctdb, struct ctdb_incoming);
+	incoming_node = inet_ntoa(addr.sin_addr);
+	for (nodeid=0;nodeid<ctdb->num_nodes;nodeid++) {
+		if (!strcmp(incoming_node, ctdb->nodes[nodeid]->address.address)) {
+			DEBUG(0, ("Incoming connection from node:%d %s\n",nodeid,incoming_node));
+			break;
+		}
+	}
+	if (nodeid>=ctdb->num_nodes) {
+		DEBUG(0, ("Refused connection from unknown node %s\n", incoming_node));
+		close(fd);
+		return;
+	}
+
+	in = talloc_zero(ctcp, struct ctdb_incoming);
 	in->fd = fd;
 	in->ctdb = ctdb;
 
 	set_nonblocking(in->fd);
+	set_close_on_exec(in->fd);
+
+        setsockopt(in->fd,SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(one));
 
 	in->queue = ctdb_queue_setup(ctdb, in, in->fd, CTDB_TCP_ALIGNMENT, 
 				     ctdb_tcp_read_cb, in);
+}
+
+
+/*
+  automatically find which address to listen on
+*/
+static int ctdb_tcp_listen_automatic(struct ctdb_context *ctdb)
+{
+	struct ctdb_tcp *ctcp = talloc_get_type(ctdb->private_data,
+						struct ctdb_tcp);
+        struct sockaddr_in sock;
+	int lock_fd, i;
+	const char *lock_path = "/tmp/.ctdb_socket_lock";
+	struct flock lock;
+
+	/* in order to ensure that we don't get two nodes with the
+	   same adddress, we must make the bind() and listen() calls
+	   atomic. The SO_REUSEADDR setsockopt only prevents double
+	   binds if the first socket is in LISTEN state  */
+	lock_fd = open(lock_path, O_RDWR|O_CREAT, 0666);
+	if (lock_fd == -1) {
+		DEBUG(0,("Unable to open %s\n", lock_path));
+		return -1;
+	}
+
+	lock.l_type = F_WRLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start = 0;
+	lock.l_len = 1;
+	lock.l_pid = 0;
+
+	if (fcntl(lock_fd, F_SETLKW, &lock) != 0) {
+		DEBUG(0,("Unable to lock %s\n", lock_path));
+		close(lock_fd);
+		return -1;
+	}
+
+	for (i=0;i<ctdb->num_nodes;i++) {
+		ZERO_STRUCT(sock);
+#ifdef HAVE_SOCK_SIN_LEN
+		sock.sin_len = sizeof(sock);
+#endif
+		sock.sin_port = htons(ctdb->nodes[i]->address.port);
+		sock.sin_family = PF_INET;
+		if (ctdb_tcp_get_address(ctdb, ctdb->nodes[i]->address.address, 
+					 &sock.sin_addr) != 0) {
+			continue;
+		}
+		
+		if (bind(ctcp->listen_fd, (struct sockaddr * )&sock, 
+			 sizeof(sock)) == 0) {
+			break;
+		}
+	}
+	
+	if (i == ctdb->num_nodes) {
+		DEBUG(0,("Unable to bind to any of the node addresses - giving up\n"));
+		goto failed;
+	}
+	ctdb->address = ctdb->nodes[i]->address;
+	ctdb->name = talloc_asprintf(ctdb, "%s:%u", 
+				     ctdb->address.address, 
+				     ctdb->address.port);
+	ctdb->vnn = ctdb->nodes[i]->vnn;
+	ctdb->nodes[i]->flags &= ~NODE_FLAGS_DISCONNECTED;
+	DEBUG(1,("ctdb chose network address %s:%u vnn %u\n", 
+		 ctdb->address.address, 
+		 ctdb->address.port, 
+		 ctdb->vnn));
+
+	if (listen(ctcp->listen_fd, 10) == -1) {
+		goto failed;
+	}
+
+	event_add_fd(ctdb->ev, ctcp, ctcp->listen_fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+		     ctdb_listen_event, ctdb);	
 
-	talloc_set_destructor(in, ctdb_incoming_destructor);
+	close(lock_fd);
+	return 0;
+	
+failed:
+	close(lock_fd);
+	close(ctcp->listen_fd);
+	ctcp->listen_fd = -1;
+	return -1;
 }
 
 
@@ -213,37 +321,50 @@ int ctdb_tcp_listen(struct ctdb_context *ctdb)
         struct sockaddr_in sock;
 	int one = 1;
 
-        sock.sin_port = htons(ctdb->address.port);
-        sock.sin_family = PF_INET;
-	if (ctdb_tcp_get_address(ctdb, ctdb->address.address, &sock.sin_addr) != 0) {
+	ctcp->listen_fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+	if (ctcp->listen_fd == -1) {
+		ctdb_set_error(ctdb, "socket failed\n");
 		return -1;
 	}
 
-        ctcp->listen_fd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
-        if (ctcp->listen_fd == -1) {
-		ctdb_set_error(ctdb, "socket failed\n");
-		return -1;
-        }
+	set_close_on_exec(ctcp->listen_fd);
 
         setsockopt(ctcp->listen_fd,SOL_SOCKET,SO_REUSEADDR,(char *)&one,sizeof(one));
 
-        if (bind(ctcp->listen_fd, (struct sockaddr * )&sock, sizeof(sock)) != 0) {
-		ctdb_set_error(ctdb, "bind failed\n");
-		close(ctcp->listen_fd);
-		ctcp->listen_fd = -1;
-                return -1;
-        }
+	/* we can either auto-bind to the first available address, or we can
+	   use a specified address */
+	if (!ctdb->address.address) {
+		return ctdb_tcp_listen_automatic(ctdb);
+	}
+
+	ZERO_STRUCT(sock);
+#ifdef HAVE_SOCK_SIN_LEN
+	sock.sin_len = sizeof(sock);
+#endif
+	sock.sin_port = htons(ctdb->address.port);
+	sock.sin_family = PF_INET;
+	
+	if (ctdb_tcp_get_address(ctdb, ctdb->address.address, 
+				 &sock.sin_addr) != 0) {
+		goto failed;
+	}
+	
+	if (bind(ctcp->listen_fd, (struct sockaddr * )&sock, sizeof(sock)) != 0) {
+		goto failed;
+	}
 
 	if (listen(ctcp->listen_fd, 10) == -1) {
-		ctdb_set_error(ctdb, "listen failed\n");
-		close(ctcp->listen_fd);
-		ctcp->listen_fd = -1;
-		return -1;
+		goto failed;
 	}
 
-	event_add_fd(ctdb->ev, ctdb, ctcp->listen_fd, EVENT_FD_READ, 
+	event_add_fd(ctdb->ev, ctcp, ctcp->listen_fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
 		     ctdb_listen_event, ctdb);	
 
 	return 0;
+
+failed:
+	close(ctcp->listen_fd);
+	ctcp->listen_fd = -1;
+	return -1;
 }
 
diff --git a/source4/cluster/ctdb/tcp/tcp_init.c b/source4/cluster/ctdb/tcp/tcp_init.c
index b4b7910446..f5d4e4c1d6 100644
--- a/source4/cluster/ctdb/tcp/tcp_init.c
+++ b/source4/cluster/ctdb/tcp/tcp_init.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -25,24 +25,64 @@
 #include "../include/ctdb_private.h"
 #include "ctdb_tcp.h"
 
+
 /*
-  start the protocol going
+  initialise tcp portion of a ctdb node 
 */
-static int ctdb_tcp_start(struct ctdb_context *ctdb)
+static int ctdb_tcp_add_node(struct ctdb_node *node)
+{
+	struct ctdb_tcp *ctcp = talloc_get_type(node->ctdb->private_data,
+						struct ctdb_tcp);
+	struct ctdb_tcp_node *tnode;
+	tnode = talloc_zero(ctcp, struct ctdb_tcp_node);
+	CTDB_NO_MEMORY(node->ctdb, tnode);
+
+	tnode->fd = -1;
+	node->private_data = tnode;
+
+	tnode->out_queue = ctdb_queue_setup(node->ctdb, ctcp, tnode->fd, CTDB_TCP_ALIGNMENT,
+					ctdb_tcp_tnode_cb, node);
+	
+	return 0;
+}
+
+/*
+  initialise transport structures
+*/
+static int ctdb_tcp_initialise(struct ctdb_context *ctdb)
 {
 	int i;
 
 	/* listen on our own address */
 	if (ctdb_tcp_listen(ctdb) != 0) return -1;
 
+	for (i=0; i<ctdb->num_nodes; i++) {
+		if (ctdb_tcp_add_node(ctdb->nodes[i]) != 0) {
+			DEBUG(0, ("methods->add_node failed at %d\n", i));
+			return -1;
+		}
+	}
+	
+	return 0;
+}
+
+/*
+  start the protocol going
+*/
+static int ctdb_tcp_start(struct ctdb_context *ctdb)
+{
+	int i;
+
 	/* startup connections to the other servers - will happen on
 	   next event loop */
 	for (i=0;i<ctdb->num_nodes;i++) {
 		struct ctdb_node *node = *(ctdb->nodes + i);
-		if (!(ctdb->flags & CTDB_FLAG_SELF_CONNECT) &&
-		    ctdb_same_address(&ctdb->address, &node->address)) continue;
-		event_add_timed(ctdb->ev, node, timeval_zero(), 
-				ctdb_tcp_node_connect, node);
+		struct ctdb_tcp_node *tnode = talloc_get_type(
+			node->private_data, struct ctdb_tcp_node);
+		if (!ctdb_same_address(&ctdb->address, &node->address)) {
+			event_add_timed(ctdb->ev, tnode, timeval_zero(), 
+					ctdb_tcp_node_connect, node);
+		}
 	}
 
 	return 0;
@@ -50,21 +90,14 @@ static int ctdb_tcp_start(struct ctdb_context *ctdb)
 
 
 /*
-  initialise tcp portion of a ctdb node 
+  shutdown the transport
 */
-static int ctdb_tcp_add_node(struct ctdb_node *node)
+static void ctdb_tcp_shutdown(struct ctdb_context *ctdb)
 {
-	struct ctdb_tcp_node *tnode;
-	tnode = talloc_zero(node, struct ctdb_tcp_node);
-	CTDB_NO_MEMORY(node->ctdb, tnode);
-
-	tnode->fd = -1;
-	node->private_data = tnode;
-
-	tnode->queue = ctdb_queue_setup(node->ctdb, node, tnode->fd, CTDB_TCP_ALIGNMENT,
-					ctdb_tcp_tnode_cb, node);
-	
-	return 0;
+	struct ctdb_tcp *ctcp = talloc_get_type(ctdb->private_data,
+						struct ctdb_tcp);
+	talloc_free(ctcp);
+	ctdb->private_data = NULL;
 }
 
 
@@ -82,10 +115,12 @@ static void *ctdb_tcp_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
 
 
 static const struct ctdb_methods ctdb_tcp_methods = {
-	.start     = ctdb_tcp_start,
-	.add_node  = ctdb_tcp_add_node,
-	.queue_pkt = ctdb_tcp_queue_pkt,
-	.allocate_pkt = ctdb_tcp_allocate_pkt
+	.initialise   = ctdb_tcp_initialise,
+	.start        = ctdb_tcp_start,
+	.queue_pkt    = ctdb_tcp_queue_pkt,
+	.add_node     = ctdb_tcp_add_node,
+	.allocate_pkt = ctdb_tcp_allocate_pkt,
+	.shutdown     = ctdb_tcp_shutdown,
 };
 
 /*
diff --git a/source4/cluster/ctdb/tcp/tcp_io.c b/source4/cluster/ctdb/tcp/tcp_io.c
index 49d5132a8a..c10afb3425 100644
--- a/source4/cluster/ctdb/tcp/tcp_io.c
+++ b/source4/cluster/ctdb/tcp/tcp_io.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -33,38 +33,49 @@
 void ctdb_tcp_read_cb(uint8_t *data, size_t cnt, void *args)
 {
 	struct ctdb_incoming *in = talloc_get_type(args, struct ctdb_incoming);
-	struct ctdb_req_header *hdr;
+	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
 
 	if (data == NULL) {
 		/* incoming socket has died */
-		talloc_free(in);
-		return;
+		goto failed;
 	}
 
 	if (cnt < sizeof(*hdr)) {
-		ctdb_set_error(in->ctdb, "Bad packet length %d\n", cnt);
-		return;
+		DEBUG(0,(__location__ " Bad packet length %u\n", (unsigned)cnt));
+		goto failed;
+	}
+
+	if (cnt & (CTDB_TCP_ALIGNMENT-1)) {
+		DEBUG(0,(__location__ " Length 0x%x not multiple of alignment\n", 
+			 (unsigned)cnt));
+		goto failed;
 	}
-	hdr = (struct ctdb_req_header *)data;
+
+
 	if (cnt != hdr->length) {
-		ctdb_set_error(in->ctdb, "Bad header length %d expected %d\n", 
-			       hdr->length, cnt);
-		return;
+		DEBUG(0,(__location__ " Bad header length %u expected %u\n", 
+			 (unsigned)hdr->length, (unsigned)cnt));
+		goto failed;
 	}
 
 	if (hdr->ctdb_magic != CTDB_MAGIC) {
-		ctdb_set_error(in->ctdb, "Non CTDB packet rejected\n");
-		return;
+		DEBUG(0,(__location__ " Non CTDB packet 0x%x rejected\n", 
+			 hdr->ctdb_magic));
+		goto failed;
 	}
 
 	if (hdr->ctdb_version != CTDB_VERSION) {
-		ctdb_set_error(in->ctdb, "Bad CTDB version 0x%x rejected\n", hdr->ctdb_version);
-		return;
+		DEBUG(0, (__location__ " Bad CTDB version 0x%x rejected\n", 
+			  hdr->ctdb_version));
+		goto failed;
 	}
 
-	/* most common case - we got a whole packet in one go
-	   tell the ctdb layer above that we have a packet */
+	/* tell the ctdb layer above that we have a packet */
 	in->ctdb->upcalls->recv_pkt(in->ctdb, data, cnt);
+	return;
+
+failed:
+	talloc_free(in);
 }
 
 /*
@@ -74,5 +85,5 @@ int ctdb_tcp_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length)
 {
 	struct ctdb_tcp_node *tnode = talloc_get_type(node->private_data,
 						      struct ctdb_tcp_node);
-	return ctdb_queue_send(tnode->queue, data, length);
+	return ctdb_queue_send(tnode->out_queue, data, length);
 }
diff --git a/source4/cluster/ctdb/tests/1node.txt b/source4/cluster/ctdb/tests/1node.txt
deleted file mode 100644
index db4350c0c0..0000000000
--- a/source4/cluster/ctdb/tests/1node.txt
+++ /dev/null
@@ -1 +0,0 @@
-127.0.0.1:9001
diff --git a/source4/cluster/ctdb/tests/4nodes.txt b/source4/cluster/ctdb/tests/4nodes.txt
deleted file mode 100644
index 880fe914ff..0000000000
--- a/source4/cluster/ctdb/tests/4nodes.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-127.0.0.1:9001
-127.0.0.2:9001
-127.0.0.3:9001
-127.0.0.4:9001
diff --git a/source4/cluster/ctdb/tests/bench-ssh.sh b/source4/cluster/ctdb/tests/bench-ssh.sh
deleted file mode 100755
index 0d11ee9cdd..0000000000
--- a/source4/cluster/ctdb/tests/bench-ssh.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/sh
-
-if [ $# -lt 1 ]; then
-    echo "Usage: bench-ssh.sh <NODES> <OPTIONS>"
-    exit 1
-fi
-
-while :; do
-    if [ "`echo $1 | cut -c1`" = "-" -o $# -eq 0 ]; then break; fi
-    nodes="$nodes $1";
-    shift;
-done
-
-options=$*
-dir=`pwd`
-
-echo "Creating nodes-ssh.txt"
-rm -f nodes-ssh.txt
-count=0
-for h in $nodes; do
-    echo "$h:9001" >> nodes-ssh.txt
-    count=`expr $count + 1`
-done
-
-
-echo "Killing old processes"
-for h in $nodes; do
-    scp -q nodes-ssh.txt $h:$dir
-    ssh $h killall -q ctdb_bench
-done
-
-echo "Starting nodes"
-i=0
-for h in $nodes; do
-    if [ $i -eq `expr $count - 1` ]; then
-	ssh $h $dir/bin/ctdb_bench --nlist $dir/nodes-ssh.txt --listen $h:9001 $options
-    else
-	ssh -f $h $dir/bin/ctdb_bench --nlist $dir/nodes-ssh.txt --listen $h:9001 $options
-    fi
-    i=`expr $i + 1`
-done
-
-wait
diff --git a/source4/cluster/ctdb/tests/bench.sh b/source4/cluster/ctdb/tests/bench.sh
index 3d0696f171..48a4277591 100755
--- a/source4/cluster/ctdb/tests/bench.sh
+++ b/source4/cluster/ctdb/tests/bench.sh
@@ -1,9 +1,24 @@
 #!/bin/sh
 
-killall -q ctdb_bench
+killall -q ctdb_bench ctdbd
 
-echo "Trying 2 nodes"
-$VALGRIND bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
-$VALGRIND bin/ctdb_bench --nlist tests/nodes.txt --listen 127.0.0.1:9001 $*
-wait
+NUMNODES=2
+if [ $# -gt 0 ]; then
+    NUMNODES=$1
+fi
+
+rm -f nodes.txt
+for i in `seq 1 $NUMNODES`; do
+  echo 127.0.0.$i >> nodes.txt
+done
+
+tests/start_daemons.sh $NUMNODES nodes.txt || exit 1
 
+killall -9 ctdb_bench
+echo "Trying $NUMNODES nodes"
+for i in `seq 1 $NUMNODES`; do
+  valgrind -q $VALGRIND bin/ctdb_bench --socket sock.$i -n $NUMNODES $*  &
+done
+
+wait
+bin/ctdb shutdown --socket sock.1 -n all
diff --git a/source4/cluster/ctdb/tests/bench1.sh b/source4/cluster/ctdb/tests/bench1.sh
deleted file mode 100755
index 9adcf3198b..0000000000
--- a/source4/cluster/ctdb/tests/bench1.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_bench
-
-echo "Trying 1 nodes"
-bin/ctdb_bench --nlist tests/1node.txt --listen 127.0.0.2:9001 $*
-wait
diff --git a/source4/cluster/ctdb/tests/cmdline.c b/source4/cluster/ctdb/tests/cmdline.c
deleted file mode 100644
index 3c2f5ca19a..0000000000
--- a/source4/cluster/ctdb/tests/cmdline.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* 
-   common commandline code to ctdb test tools
-
-   Copyright (C) Andrew Tridgell  2007
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-
-/* Handle common command line options for ctdb test progs
- */
-
-static struct {
-	const char *nlist;
-	const char *transport;
-	const char *myaddress;
-	int self_connect;
-	int daemon_mode;
-} ctdb_cmdline = {
-	.nlist = NULL,
-	.transport = "tcp",
-	.myaddress = NULL,
-	.self_connect = 0,
-	.daemon_mode = 0
-};
-
-
-struct poptOption popt_ctdb_cmdline[] = {
-	{ "nlist", 0, POPT_ARG_STRING, &ctdb_cmdline.nlist, 0, "node list file", "filename" },
-	{ "listen", 0, POPT_ARG_STRING, &ctdb_cmdline.myaddress, 0, "address to listen on", "address" },
-	{ "transport", 0, POPT_ARG_STRING, &ctdb_cmdline.transport, 0, "protocol transport", NULL },
-	{ "self-connect", 0, POPT_ARG_NONE, &ctdb_cmdline.self_connect, 0, "enable self connect", "boolean" },
-	{ "daemon", 0, POPT_ARG_NONE, &ctdb_cmdline.daemon_mode, 0, "spawn a ctdb daemon", "boolean" },
-	{ NULL }
-};
-
-
-/*
-  startup daemon side of ctdb according to command line options
- */
-struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
-{
-	struct ctdb_context *ctdb;
-	int ret;
-
-	if (ctdb_cmdline.nlist == NULL || ctdb_cmdline.myaddress == NULL) {
-		printf("You must provide a node list with --nlist and an address with --listen\n");
-		exit(1);
-	}
-
-	/* initialise ctdb */
-	ctdb = ctdb_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	if (ctdb_cmdline.self_connect) {
-		ctdb_set_flags(ctdb, CTDB_FLAG_SELF_CONNECT);
-	}
-	if (ctdb_cmdline.daemon_mode) {
-		ctdb_set_flags(ctdb, CTDB_FLAG_DAEMON_MODE);
-	}
-
-	ret = ctdb_set_transport(ctdb, ctdb_cmdline.transport);
-	if (ret == -1) {
-		printf("ctdb_set_transport failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* tell ctdb what address to listen on */
-	ret = ctdb_set_address(ctdb, ctdb_cmdline.myaddress);
-	if (ret == -1) {
-		printf("ctdb_set_address failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* tell ctdb what nodes are available */
-	ret = ctdb_set_nlist(ctdb, ctdb_cmdline.nlist);
-	if (ret == -1) {
-		printf("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	return ctdb;
-}
diff --git a/source4/cluster/ctdb/tests/cmdline.h b/source4/cluster/ctdb/tests/cmdline.h
deleted file mode 100644
index 785595ee6c..0000000000
--- a/source4/cluster/ctdb/tests/cmdline.h
+++ /dev/null
@@ -1,7 +0,0 @@
-
-extern struct poptOption popt_ctdb_cmdline[];
-
-#define POPT_CTDB_CMDLINE { NULL, 0, POPT_ARG_INCLUDE_TABLE, popt_ctdb_cmdline, 0, "Common ctdb test options:", NULL },
-
-struct ctdb_context *ctdb_cmdline_init(struct event_context *ev);
-
diff --git a/source4/cluster/ctdb/tests/ctdb_bench.c b/source4/cluster/ctdb/tests/ctdb_bench.c
index a9cb2c2372..f1292ee275 100644
--- a/source4/cluster/ctdb/tests/ctdb_bench.c
+++ b/source4/cluster/ctdb/tests/ctdb_bench.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -43,7 +43,7 @@ static double end_timer(void)
 
 static int timelimit = 10;
 static int num_records = 10;
-static int num_msgs = 1;
+static int num_nodes;
 
 enum my_functions {FUNC_INCR=1, FUNC_FETCH=2};
 
@@ -83,14 +83,14 @@ static int msg_plus, msg_minus;
 /*
   handler for messages in bench_ring()
 */
-static void ring_message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+static void ring_message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 				 TDB_DATA data, void *private_data)
 {
 	int incr = *(int *)data.dptr;
 	int *count = (int *)private_data;
 	int dest;
 	(*count)++;
-	dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
+	dest = (ctdb_get_vnn(ctdb) + incr) % num_nodes;
 	ctdb_send_message(ctdb, dest, srvid, data);
 	if (incr == 1) {
 		msg_plus++;
@@ -107,7 +107,6 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 	int vnn=ctdb_get_vnn(ctdb);
 
 	if (vnn == 0) {
-		int i;
 		/* two messages are injected into the ring, moving
 		   in opposite directions */
 		int dest, incr;
@@ -116,15 +115,13 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 		data.dptr = (uint8_t *)&incr;
 		data.dsize = sizeof(incr);
 
-		for (i=0;i<num_msgs;i++) {
-			incr = 1;
-			dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
-			ctdb_send_message(ctdb, dest, 0, data);
-
-			incr = -1;
-			dest = (ctdb_get_vnn(ctdb) + incr) % ctdb_get_num_nodes(ctdb);
-			ctdb_send_message(ctdb, dest, 0, data);
-		}
+		incr = 1;
+		dest = (ctdb_get_vnn(ctdb) + incr) % num_nodes;
+		ctdb_send_message(ctdb, dest, 0, data);
+		
+		incr = -1;
+		dest = (ctdb_get_vnn(ctdb) + incr) % num_nodes;
+		ctdb_send_message(ctdb, dest, 0, data);
 	}
 	
 	start_timer();
@@ -142,6 +139,16 @@ static void bench_ring(struct ctdb_context *ctdb, struct event_context *ev)
 	       msg_count/end_timer(), msg_plus, msg_minus);
 }
 
+/*
+  handler for reconfigure message
+*/
+static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+				TDB_DATA data, void *private_data)
+{
+	int *ready = (int *)private_data;
+	*ready = 1;
+}
+
 
 /*
   main program
@@ -156,7 +163,7 @@ int main(int argc, const char *argv[])
 		POPT_CTDB_CMDLINE
 		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
 		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
-		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
+		{ NULL, 'n', POPT_ARG_INT, &num_nodes, 0, "num_nodes", "integer" },
 		POPT_TABLEEND
 	};
 	int opt;
@@ -165,6 +172,7 @@ int main(int argc, const char *argv[])
 	int ret;
 	poptContext pc;
 	struct event_context *ev;
+	int cluster_ready=0;
 
 	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
 
@@ -187,10 +195,13 @@ int main(int argc, const char *argv[])
 	ev = event_context_init(NULL);
 
 	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
+	ctdb = ctdb_cmdline_client(ev);
+
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler, 
+				 &cluster_ready);
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	ctdb_db = ctdb_attach(ctdb, "test.tdb");
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
@@ -200,19 +211,19 @@ int main(int argc, const char *argv[])
 	ret = ctdb_set_call(ctdb_db, incr_func,  FUNC_INCR);
 	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
 
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-	ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count);
+	if (ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count))
+		goto error;
 
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
+	printf("Waiting for cluster\n");
+	while (1) {
+		uint32_t recmode=1;
+		ctdb_ctrl_getrecmode(ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode);
+		if (recmode == 0) break;
+		event_loop_once(ev);
+	}
 
 	bench_ring(ctdb, ev);
        
-	/* shut it down */
-	ctdb_shutdown(ctdb);
-
+error:
 	return 0;
 }
diff --git a/source4/cluster/ctdb/tests/ctdb_fetch.c b/source4/cluster/ctdb/tests/ctdb_fetch.c
index 38d652bf5f..f57d05d099 100644
--- a/source4/cluster/ctdb/tests/ctdb_fetch.c
+++ b/source4/cluster/ctdb/tests/ctdb_fetch.c
@@ -3,18 +3,18 @@
 
    Copyright (C) Andrew Tridgell  2006
 
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
 #include "includes.h"
@@ -43,8 +43,7 @@ static double end_timer(void)
 
 static int timelimit = 10;
 static int num_records = 10;
-static int num_msgs = 1;
-
+static int num_nodes;
 static int msg_count;
 
 #define TESTKEY "testkey"
@@ -99,14 +98,14 @@ static void bench_fetch_1node(struct ctdb_context *ctdb)
 	nulldata.dptr = NULL;
 	nulldata.dsize = 0;
 
-	dest = (ctdb_get_vnn(ctdb) + 1) % ctdb_get_num_nodes(ctdb);
+	dest = (ctdb_get_vnn(ctdb) + 1) % num_nodes;
 	ctdb_send_message(ctdb, dest, 0, nulldata);
 }
 
 /*
   handler for messages in bench_ring()
 */
-static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
+static void message_handler(struct ctdb_context *ctdb, uint64_t srvid, 
 			    TDB_DATA data, void *private_data)
 {
 	msg_count++;
@@ -126,7 +125,7 @@ static void bench_fetch(struct ctdb_context *ctdb, struct event_context *ev)
 {
 	int vnn=ctdb_get_vnn(ctdb);
 
-	if (vnn == ctdb_get_num_nodes(ctdb)-1) {
+	if (vnn == num_nodes - 1) {
 		bench_fetch_1node(ctdb);
 	}
 	
@@ -141,10 +140,6 @@ static void bench_fetch(struct ctdb_context *ctdb, struct event_context *ev)
 			printf("Event loop failed!\n");
 			break;
 		}
-
-		if (LogLevel > 9) {
-			talloc_report_null_full();
-		}
 	}
 
 	printf("Fetch: %.2f msgs/sec\n", msg_count/end_timer());
@@ -162,6 +157,16 @@ static int fetch_func(struct ctdb_call_info *call)
 }
 
 /*
+  handler for reconfigure message
+*/
+static void reconfigure_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+				TDB_DATA data, void *private_data)
+{
+	int *ready = (int *)private_data;
+	*ready = 1;
+}
+
+/*
   main program
 */
 int main(int argc, const char *argv[])
@@ -174,7 +179,7 @@ int main(int argc, const char *argv[])
 		POPT_CTDB_CMDLINE
 		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
 		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
-		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
+		{ NULL, 'n', POPT_ARG_INT, &num_nodes, 0, "num_nodes", "integer" },
 		POPT_TABLEEND
 	};
 	int opt;
@@ -184,6 +189,7 @@ int main(int argc, const char *argv[])
 	poptContext pc;
 	struct event_context *ev;
 	struct ctdb_call call;
+	int cluster_ready=0;
 
 	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
 
@@ -207,10 +213,13 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_init(ev);
+	ctdb = ctdb_cmdline_client(ev);
+
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_RECONFIGURE, reconfigure_handler, 
+				 &cluster_ready);
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
+	ctdb_db = ctdb_attach(ctdb, "test.tdb");
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
@@ -218,14 +227,15 @@ int main(int argc, const char *argv[])
 
 	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
 
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
 	ctdb_set_message_handler(ctdb, 0, message_handler, &msg_count);
 
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
+	printf("Waiting for cluster\n");
+	while (1) {
+		uint32_t recmode=1;
+		ctdb_ctrl_getrecmode(ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode);
+		if (recmode == 0) break;
+		event_loop_once(ev);
+	}
 
 	bench_fetch(ctdb, ev);
 
@@ -233,6 +243,8 @@ int main(int argc, const char *argv[])
 	call.key.dptr = discard_const(TESTKEY);
 	call.key.dsize = strlen(TESTKEY);
 
+	printf("Fetching final record\n");
+
 	/* fetch the record */
 	call.call_id = FUNC_FETCH;
 	call.call_data.dptr = NULL;
@@ -246,8 +258,5 @@ int main(int argc, const char *argv[])
 
 	printf("DATA:\n%s\n", (char *)call.reply_data.dptr);
 
-	/* go into a wait loop to allow other nodes to complete */
-	ctdb_shutdown(ctdb);
-
 	return 0;
 }
diff --git a/source4/cluster/ctdb/tests/ctdb_fetch1.c b/source4/cluster/ctdb/tests/ctdb_fetch1.c
deleted file mode 100644
index a4cb53dd26..0000000000
--- a/source4/cluster/ctdb/tests/ctdb_fetch1.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/* 
-   simple ctdb fetch test
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "ctdb.h"
-#include "ctdb_private.h"
-#include "cmdline.h"
-#include <sys/time.h>
-
-#define PARENT_SRVID	0
-#define CHILD1_SRVID	1
-#define CHILD2_SRVID	2
-
-int num_msg=0;
-
-static struct timeval tp1,tp2;
-
-static void start_timer(void)
-{
-	gettimeofday(&tp1,NULL);
-}
-
-static double end_timer(void)
-{
-	gettimeofday(&tp2,NULL);
-	return (tp2.tv_sec + (tp2.tv_usec*1.0e-6)) - 
-		(tp1.tv_sec + (tp1.tv_usec*1.0e-6));
-}
-
-static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-			    TDB_DATA data, void *private_data)
-{
-	num_msg++;
-}
-static void child_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-			    TDB_DATA data, void *private_data)
-{
-	num_msg++;
-}
-
-void test1(struct ctdb_db_context *ctdb_db)
-{
-	TDB_DATA key, data, data2, store_data;
-	int ret;
-	struct ctdb_record_handle *h;
- 
-	/* 
-	   test 1 : write data and read it back.   should all be the same
-	 */
-	printf("Test1: write and verify we can read it back: ");
-	key.dptr  = discard_const("Record");
-	key.dsize = strlen((const char *)key.dptr)+1;
-	h = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data);
-	if (h == NULL) {
-		printf("test1: ctdb_fetch_lock() failed\n");
-		exit(1);
-	}
-
-	store_data.dptr  = discard_const("data to store");
-	store_data.dsize = strlen((const char *)store_data.dptr)+1;
-	ret = ctdb_record_store(h, store_data);
-	talloc_free(h);
-	if (ret!=0) {
-		printf("test1: ctdb_record_store() failed\n");
-		exit(1);
-	}
-
-	h = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data2);
-	if (h == NULL) {
-		printf("test1: ctdb_fetch_lock() failed\n");
-		exit(1);
-	}
-
-	/* hopefully   data2 will now contain the record written above */
-	if (!strcmp("data to store", (const char *)data2.dptr)) {
-		printf("SUCCESS\n");
-	} else {
-		printf("FAILURE\n");
-		exit(10);
-	}
-	
-	/* just write it back to unlock it */
-	ret = ctdb_record_store(h, store_data);
-	talloc_free(h);
-	if (ret!=0) {
-		printf("test1: ctdb_record_store() failed\n");
-		exit(1);
-	}
-}
-
-void child(int srvid, struct event_context *ev, struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db)
-{
-	TDB_DATA data;
-	TDB_DATA key, data2;
-	struct ctdb_record_handle *h;
-
-	data.dptr=discard_const("dummy message");
-	data.dsize=strlen((const char *)data.dptr)+1;
-
-	ctdb_set_message_handler(ctdb, srvid, child_handler, NULL);
-
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), PARENT_SRVID, data);
-	while (num_msg==0) {
-		event_loop_once(ev);
-	}
-
-
-	/* fetch and lock the record */
-	key.dptr  = discard_const("Record");
-	key.dsize = strlen((const char *)key.dptr)+1;
-	printf("client:%d fetching the record\n",srvid);
-	h = ctdb_fetch_lock(ctdb_db, ctdb_db, key, &data2);
-	printf("client:%d the record is fetched and locked\n",srvid);
-	if (h == NULL) {
-		printf("client: ctdb_fetch_lock() failed\n");
-		exit(1);
-	}
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), PARENT_SRVID, data);
-
-	/* wait until parent tells us to release the lock */
-	while (num_msg==1) {
-		event_loop_once(ev);
-	}
-
-	printf("child %d terminating\n",srvid);
-	exit(10);
-	   
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-	TDB_DATA data;
-
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int ret;
-	poptContext pc;
-	struct event_context *ev;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
-	if (!ctdb_db) {
-		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-#if 0
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
-#endif
-
-	/*
-	   start two child processes
-	 */
-	if(fork()){
-		/* 
-		   set up a message handler so our child processes can talk to us
-		 */
-		ctdb_set_message_handler(ctdb, PARENT_SRVID, message_handler, NULL);
-	} else {
-		sleep(3);
-		if(!fork()){
-			child(CHILD1_SRVID, ev, ctdb, ctdb_db);
-		} else {
-			child(CHILD2_SRVID, ev, ctdb, ctdb_db);
-		}
-	}
-
-	/* 
-	   test 1 : write data and read it back.
-	 */
-	test1(ctdb_db); 
-
-	/* 
-	   wait until both children have sent us a message they have started
-	 */
-	printf("Wait for both child processes to start: ");
-	while (num_msg!=2) {
-		event_loop_once(ev);
-	}
-	printf("STARTED\n");
-
-
-	/*
-	   send message to child 1 to make it to fetch and lock the record 
-	 */
-	data.dptr=discard_const("dummy message");
-	data.dsize=strlen((const char *)data.dptr)+1;
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD1_SRVID, data);
-
-	/* wait for child 1 to complete fetching and locking the record */
-	while (num_msg!=3) {
-		event_loop_once(ev);
-	}
-
-	/* now tell child 2 to fetch and lock the same record */
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD2_SRVID, data);
-
-	/* wait a while for child 2 to complete fetching and locking the 
-	   record, this should fail since the record is already locked
-	   by the first child */
-	start_timer();
-	while ( (end_timer() < 1.0) && (num_msg!=4) ) {
-		event_loop_once(ev);
-	}
-	if (num_msg!=4) {
-		printf("Child 2 did not get the lock since it is held by client 1:SUCCESS\n");
-	} else {
-		printf("Child 2 did get the lock:FAILURE\n");
-		exit(10);
-	}
-
-	/* send message to child 1 to terminate, which should let child 2
-	   get the lock.
-	 */
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD1_SRVID, data);
-
-
-	/* wait for a final message from child 2 it has received the lock
-	   which indicates success */
-	while (num_msg!=4) {
-		event_loop_once(ev);
-	}
-	printf("child 2 aquired the lock after child 1 terminated:SUCCESS\n");
-
-	/* send a message to child 2 to tell it to terminate too */
-	ctdb_send_message(ctdb, ctdb_get_vnn(ctdb), CHILD2_SRVID, data);
-
-
-	printf("Test was SUCCESSFUL\n");
-
-	/* shut it down */
-	talloc_free(ctdb);
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/ctdb_messaging.c b/source4/cluster/ctdb/tests/ctdb_messaging.c
deleted file mode 100644
index 309b9ebd32..0000000000
--- a/source4/cluster/ctdb/tests/ctdb_messaging.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/* 
-   test of messaging
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "cmdline.h"
-
-static int timelimit = 10;
-static int num_records = 10;
-static int num_msgs = 1;
-static int num_clients = 2;
-
-
-/*
-  handler for messages in bench_ring()
-*/
-static void message_handler(struct ctdb_context *ctdb, uint32_t srvid, 
-				 TDB_DATA data, void *private_data)
-{
-	printf("client vnn:%d received a message to srvid:%d [%s]\n",ctdb_get_vnn(ctdb),srvid,data.dptr);
-	fflush(stdout);
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-	char buf[256];
-
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
-		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
-		{ "num-msgs", 'n', POPT_ARG_INT, &num_msgs, 0, "num_msgs", "integer" },
-		{ "num-clients", 0, POPT_ARG_INT, &num_clients, 0, "num_clients", "integer" },
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int ret, i, j;
-	poptContext pc;
-	struct event_context *ev;
-	pid_t pid;
-	int srvid;
-	TDB_DATA data;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
-	if (!ctdb_db) {
-		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-	srvid = -1;
-	for (i=0;i<num_clients-1;i++) {
-		pid=fork();
-		if (pid) {
-			srvid = i;
-			break;
-		}
-	}
-	if (srvid == -1) {
-		srvid = num_clients-1;
-	}
-
-	ctdb_set_message_handler(ctdb, srvid, message_handler, NULL);
-
-	/* wait until all nodes are connected (should not be needed
-	   outside of test code) */
-	ctdb_connect_wait(ctdb);
-
-	sleep(3);
-
-	printf("sending message from vnn:%d to vnn:%d/srvid:%d\n",ctdb_get_vnn(ctdb),ctdb_get_vnn(ctdb), 1-srvid);
-	for (i=0;i<ctdb_get_num_nodes(ctdb);i++) {
-		for (j=0;j<num_clients;j++) {
-			printf("sending message to %d:%d\n", i, j);
-			sprintf(buf,"Message from %d to vnn:%d srvid:%d",ctdb_get_vnn(ctdb),i,j);
-			data.dptr = (unsigned char *)buf;
-			data.dsize=strlen(buf)+1;
-			ctdb_send_message(ctdb, i, j, data);
-		}
-	}
-
-	while (1) {
-		event_loop_once(ev);
-	}
-       
-	/* shut it down */
-	ctdb_shutdown(ctdb);
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/ctdb_store.c b/source4/cluster/ctdb/tests/ctdb_store.c
new file mode 100644
index 0000000000..a60e009b91
--- /dev/null
+++ b/source4/cluster/ctdb/tests/ctdb_store.c
@@ -0,0 +1,156 @@
+/* 
+   simple tool to create a lot of records on a tdb and to read them out
+
+   Copyright (C) Andrew Tridgell  2006
+	Ronnie sahlberg 2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "popt.h"
+#include "cmdline.h"
+
+#include <sys/time.h>
+#include <time.h>
+
+static int num_records = 10;
+
+
+static void store_records(struct ctdb_context *ctdb, struct event_context *ev)
+{
+	TDB_DATA key, data;
+	struct ctdb_db_context *ctdb_db;
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+	int ret;
+	struct ctdb_record_handle *h;
+	uint32_t i;
+	
+	ctdb_db = ctdb_db_handle(ctdb, "test.tdb");
+
+	printf("creating %d records\n", num_records);
+	for (i=0;i<num_records;i++) {
+		key.dptr = (uint8_t *)&i;
+		key.dsize = sizeof(uint32_t); 
+
+		h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
+		if (h == NULL) {
+			printf("Failed to fetch record '%s' on node %d\n", 
+			       (const char *)key.dptr, ctdb_get_vnn(ctdb));
+			talloc_free(tmp_ctx);
+			return;
+		}
+
+		data.dptr = (uint8_t *)&i;
+		data.dsize = sizeof(uint32_t);
+
+		ret = ctdb_record_store(h, data);
+		talloc_free(h);
+		if (ret != 0) {
+			printf("Failed to store record\n");
+		}
+		if (i % 1000 == 0) {
+			printf("%u\r", i);
+			fflush(stdout);
+		}
+	}
+
+	printf("fetching all %d records\n", num_records);
+	while (1) {
+		for (i=0;i<num_records;i++) {
+			key.dptr = (uint8_t *)&i;
+			key.dsize = sizeof(uint32_t); 
+
+			h = ctdb_fetch_lock(ctdb_db, tmp_ctx, key, &data);
+			if (h == NULL) {
+				printf("Failed to fetch record '%s' on node %d\n", 
+				       (const char *)key.dptr, ctdb_get_vnn(ctdb));
+				talloc_free(tmp_ctx);
+				return;
+			}
+			talloc_free(h);
+		}
+		sleep(1);
+		printf(".");
+		fflush(stdout);
+	}
+
+	talloc_free(tmp_ctx);
+}
+
+/*
+  main program
+*/
+int main(int argc, const char *argv[])
+{
+	struct ctdb_context *ctdb;
+	struct ctdb_db_context *ctdb_db;
+
+	struct poptOption popt_options[] = {
+		POPT_AUTOHELP
+		POPT_CTDB_CMDLINE
+		{ "num-records", 'r', POPT_ARG_INT, &num_records, 0, "num_records", "integer" },
+		POPT_TABLEEND
+	};
+	int opt;
+	const char **extra_argv;
+	int extra_argc = 0;
+	poptContext pc;
+	struct event_context *ev;
+
+	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
+
+	while ((opt = poptGetNextOpt(pc)) != -1) {
+		switch (opt) {
+		default:
+			fprintf(stderr, "Invalid option %s: %s\n", 
+				poptBadOption(pc, 0), poptStrerror(opt));
+			exit(1);
+		}
+	}
+
+	/* talloc_enable_leak_report_full(); */
+
+	/* setup the remaining options for the main program to use */
+	extra_argv = poptGetArgs(pc);
+	if (extra_argv) {
+		extra_argv++;
+		while (extra_argv[extra_argc]) extra_argc++;
+	}
+
+	ev = event_context_init(NULL);
+
+	ctdb = ctdb_cmdline_client(ev);
+
+	/* attach to a specific database */
+	ctdb_db = ctdb_attach(ctdb, "test.tdb");
+	if (!ctdb_db) {
+		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
+		exit(1);
+	}
+
+	printf("Waiting for cluster\n");
+	while (1) {
+		uint32_t recmode=1;
+		ctdb_ctrl_getrecmode(ctdb, timeval_zero(), CTDB_CURRENT_NODE, &recmode);
+		if (recmode == 0) break;
+		event_loop_once(ev);
+	}
+
+	store_records(ctdb, ev);
+
+	return 0;
+}
diff --git a/source4/cluster/ctdb/tests/ctdb_test.c b/source4/cluster/ctdb/tests/ctdb_test.c
deleted file mode 100644
index bb888b5c7c..0000000000
--- a/source4/cluster/ctdb/tests/ctdb_test.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/* 
-   ctdb test harness
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "cmdline.h"
-
-enum my_functions {FUNC_SORT=1, FUNC_FETCH=2};
-
-static int int_compare(int *i1, int *i2)
-{
-	return *i1 - *i2;
-}
-
-/*
-  add an integer into a record in sorted order
-*/
-static int sort_func(struct ctdb_call_info *call)
-{
-	if (call->call_data == NULL ||
-	    call->call_data->dsize != sizeof(int)) {
-		return CTDB_ERR_INVALID;
-	}
-	call->new_data = talloc(call, TDB_DATA);
-	if (call->new_data == NULL) {
-		return CTDB_ERR_NOMEM;
-	}
-	call->new_data->dptr = talloc_size(call, 
-					   call->record_data.dsize + 
-					   call->call_data->dsize);
-	if (call->new_data->dptr == NULL) {
-		return CTDB_ERR_NOMEM;
-	}
-	call->new_data->dsize = call->record_data.dsize + call->call_data->dsize;
-	memcpy(call->new_data->dptr,
-	       call->record_data.dptr, call->record_data.dsize);
-	memcpy(call->new_data->dptr+call->record_data.dsize,
-	       call->call_data->dptr, call->call_data->dsize);
-
-	qsort(call->new_data->dptr, call->new_data->dsize / sizeof(int),
-	      sizeof(int), (comparison_fn_t)int_compare);
-
-	return 0;
-}
-
-/*
-  ctdb call function to fetch a record
-*/
-static int fetch_func(struct ctdb_call_info *call)
-{
-	call->reply_data = &call->record_data;
-	return 0;
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int i, ret;
-	poptContext pc;
-	struct event_context *ev;
-	struct ctdb_call call;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_init(ev);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", TDB_DEFAULT, O_RDWR|O_CREAT|O_TRUNC, 0666);
-	if (!ctdb_db) {
-		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	/* setup a ctdb call function */
-	ret = ctdb_set_call(ctdb_db, sort_func,  FUNC_SORT);
-	ret = ctdb_set_call(ctdb_db, fetch_func, FUNC_FETCH);
-
-	/* start the protocol running */
-	ret = ctdb_start(ctdb);
-
-	ctdb_connect_wait(ctdb);
-
-	ZERO_STRUCT(call);
-	call.key.dptr = discard_const("test");
-	call.key.dsize = strlen("test")+1;
-
-	/* add some random data */
-	for (i=0;i<10;i++) {
-		int v = random() % 1000;
-
-		call.call_id = FUNC_SORT;
-		call.call_data.dptr = (uint8_t *)&v;
-		call.call_data.dsize = sizeof(v);
-
-		ret = ctdb_call(ctdb_db, &call);
-		if (ret == -1) {
-			printf("ctdb_call FUNC_SORT failed - %s\n", ctdb_errstr(ctdb));
-			exit(1);
-		}
-	}
-
-	/* fetch the record */
-	call.call_id = FUNC_FETCH;
-	call.call_data.dptr = NULL;
-	call.call_data.dsize = 0;
-
-	ret = ctdb_call(ctdb_db, &call);
-	if (ret == -1) {
-		printf("ctdb_call FUNC_FETCH failed - %s\n", ctdb_errstr(ctdb));
-		exit(1);
-	}
-
-	for (i=0;i<call.reply_data.dsize/sizeof(int);i++) {
-		printf("%3d\n", ((int *)call.reply_data.dptr)[i]);
-	}
-	talloc_free(call.reply_data.dptr);
-
-	/* go into a wait loop to allow other nodes to complete */
-	ctdb_shutdown(ctdb);
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/ctdbd.sh b/source4/cluster/ctdb/tests/ctdbd.sh
new file mode 100755
index 0000000000..4a1e9976b2
--- /dev/null
+++ b/source4/cluster/ctdb/tests/ctdbd.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+killall -q ctdbd
+
+tests/start_daemons.sh 2 tests/nodes.txt || exit 1
+
+echo "Testing ping"
+$VALGRIND bin/ctdb ping || exit 1
+
+echo "Testing status"
+$VALGRIND bin/ctdb status || exit 1
+
+echo "Testing statistics"
+$VALGRIND bin/ctdb -n all statistics || exit 1
+
+echo "Testing statisticsreset"
+$VALGRIND bin/ctdb -n all statisticsreset || exit 1
+
+echo "Testing debug"
+$VALGRIND bin/ctdb -n all setdebug 3 || exit 1
+$VALGRIND bin/ctdb -n all getdebug || exit 1
+$VALGRIND bin/ctdb -n all setdebug 0 || exit 1
+$VALGRIND bin/ctdb -n all getdebug || exit 1
+
+echo "Attaching to some databases"
+$VALGRIND bin/ctdb attach test1.tdb || exit 1
+$VALGRIND bin/ctdb attach test2.tdb || exit 1
+
+echo "Testing getdbmap"
+$VALGRIND bin/ctdb getdbmap || exit 1
+
+echo "Testing status"
+$VALGRIND bin/ctdb status || exit 1
+
+echo "Testing variables"
+$VALGRIND bin/ctdb listvars || exit 1
+$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
+$VALGRIND bin/ctdb setvar TraverseTimeout 10 || exit 1
+$VALGRIND bin/ctdb getvar TraverseTimeout || exit 1
+
+sleep 1
+
+echo "Testing shutdown"
+$VALGRIND bin/ctdb shutdown -n all || exit 1
+
+sleep 1
+
+echo "All done"
+killall -q ctdbd
+exit 0
diff --git a/source4/cluster/ctdb/tests/events b/source4/cluster/ctdb/tests/events
new file mode 100755
index 0000000000..fb319bc426
--- /dev/null
+++ b/source4/cluster/ctdb/tests/events
@@ -0,0 +1,68 @@
+#!/bin/sh
+# event script for 'make test'
+
+cmd="$1"
+shift
+
+case $cmd in 
+    monitor)
+	echo "`date` monitor event"
+	exit 0
+	;;
+     startup)
+	echo "`date` ctdb startup event"
+	exit 0;	
+	;;
+	
+     takeip)
+	if [ $# != 3 ]; then
+	   echo "must supply interface, IP and maskbits"
+	   exit 1
+	fi
+	iface=$1
+	ip=$2
+	maskbits=$3
+
+	[ `id -u` = 0 ] && {
+	    /sbin/ip addr add $ip/$maskbits dev $iface || {
+		echo "`/bin/date` Failed to add $ip/$maskbits on dev $iface"
+		exit 1
+	    }
+	}
+	exit 0;
+	;;
+
+
+     ##################################################
+     # called when ctdbd wants to release an IP address
+     releaseip)
+	if [ $# != 3 ]; then
+	   echo "`/bin/date` must supply interface, IP and maskbits"
+	   exit 1
+	fi
+	iface=$1
+	ip=$2
+	maskbits=$3
+	[ `id -u` = 0 ] && {
+	    /sbin/ip addr del $ip/$maskbits dev $iface || {
+		echo "`/bin/date` Failed to del $ip on dev $iface"
+		exit 1
+	    }
+	}
+	echo "`date` ctdb takeip event for $1 $2 $3"
+	exit 0
+	;;
+
+     recovered)
+	echo "`date` ctdb recovered event"
+	exit 0
+	;;
+
+     shutdown)
+	echo "`date` ctdb shutdown event"
+	exit 0
+	;;
+esac
+
+echo "`/bin/date` Invalid command $cmd"
+exit 1
diff --git a/source4/cluster/ctdb/tests/fetch.sh b/source4/cluster/ctdb/tests/fetch.sh
index 73192e70ae..325957e63f 100755
--- a/source4/cluster/ctdb/tests/fetch.sh
+++ b/source4/cluster/ctdb/tests/fetch.sh
@@ -1,15 +1,24 @@
 #!/bin/sh
 
-killall -q ctdb_fetch
+NUMNODES=2
+if [ $# -gt 0 ]; then
+    NUMNODES=$1
+fi
 
-echo "Trying 2 nodes"
-$VALGRIND bin/ctdb_fetch --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/nodes.txt --listen 127.0.0.1:9001 $* 
-wait
+rm -f nodes.txt
+for i in `seq 1 $NUMNODES`; do
+  echo 127.0.0.$i >> nodes.txt
+done
+
+tests/start_daemons.sh $NUMNODES nodes.txt || exit 1
 
-echo "Trying 4 nodes"
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.4:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.3:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.2:9001 $* &
-$VALGRIND bin/ctdb_fetch --nlist tests/4nodes.txt --listen 127.0.0.1:9001 $* 
+
+killall -9 -q ctdb_fetch
+for i in `seq 1 $NUMNODES`; do
+  $VALGRIND bin/ctdb_fetch --socket sock.$i -n $NUMNODES $* &
+done
 wait
+
+echo "Shutting down"
+bin/ctdb shutdown -n all --socket=sock.1
+exit 0
diff --git a/source4/cluster/ctdb/tests/fetch1.sh b/source4/cluster/ctdb/tests/fetch1.sh
deleted file mode 100755
index db584ec012..0000000000
--- a/source4/cluster/ctdb/tests/fetch1.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_fetch1
-
-echo "Trying node"
-bin/ctdb_fetch1 --nlist tests/1node.txt --listen 127.0.0.1:9001 $* 
-wait
diff --git a/source4/cluster/ctdb/tests/lockwait.c b/source4/cluster/ctdb/tests/lockwait.c
deleted file mode 100644
index 6cefe5680a..0000000000
--- a/source4/cluster/ctdb/tests/lockwait.c
+++ /dev/null
@@ -1,244 +0,0 @@
-/* 
-   test a lock wait idea
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "system/wait.h"
-#include "popt.h"
-#include "cmdline.h"
-
-
-struct lockwait_handle {
-	struct fd_event *fde;
-	int fd[2];
-	pid_t child;
-	void *private_data;
-	void (*callback)(void *);
-};
-
-static void lockwait_handler(struct event_context *ev, struct fd_event *fde, 
-			     uint16_t flags, void *private_data)
-{
-	struct lockwait_handle *h = talloc_get_type(private_data, 
-						     struct lockwait_handle);
-	void (*callback)(void *) = h->callback;
-	void *p = h->private_data;
-	talloc_set_destructor(h, NULL);
-	close(h->fd[0]);
-	talloc_free(h);	
-	callback(p);
-	waitpid(h->child, NULL, 0);
-}
-
-static int lockwait_destructor(struct lockwait_handle *h)
-{
-	close(h->fd[0]);
-	kill(h->child, SIGKILL);
-	waitpid(h->child, NULL, 0);
-	return 0;
-}
-
-
-static struct lockwait_handle *lockwait(struct event_context *ev, 
-					TALLOC_CTX *mem_ctx,
-					int fd, off_t ofs, size_t len,
-					void (*callback)(void *), void *private_data)
-{
-	struct lockwait_handle *h;
-	int ret;
-
-	h = talloc_zero(mem_ctx, struct lockwait_handle);
-	if (h == NULL) {
-		return NULL;
-	}
-
-	ret = pipe(h->fd);
-	if (ret != 0) {
-		talloc_free(h);
-		return NULL;
-	}
-
-	h->child = fork();
-	if (h->child == (pid_t)-1) {
-		close(h->fd[0]);
-		close(h->fd[1]);
-		talloc_free(h);
-		return NULL;
-	}
-
-	h->callback = callback;
-	h->private_data = private_data;
-
-	if (h->child == 0) {
-		/* in child */
-		struct flock lock;
-		close(h->fd[0]);
-		lock.l_type = F_WRLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start = ofs;
-		lock.l_len = len;
-		lock.l_pid = 0;
-		fcntl(fd,F_SETLKW,&lock);
-		_exit(0);
-	}
-
-	close(h->fd[1]);
-	talloc_set_destructor(h, lockwait_destructor);
-
-	h->fde = event_add_fd(ev, h, h->fd[0], EVENT_FD_READ, lockwait_handler, h);
-	if (h->fde == NULL) {
-		talloc_free(h);
-		return NULL;
-	}
-
-	return h;
-}
-
-
-
-
-static void fcntl_lock_callback(void *p)
-{
-	int *got_lock = (int *)p;
-	*got_lock = 1;
-}
-
-/*
-  get an fcntl lock - waiting if necessary
- */
-static int fcntl_lock(struct event_context *ev,
-		      int fd, int op, off_t offset, off_t count, int type)
-{
-	struct flock lock;
-	int ret;
-	int use_lockwait = (op == F_SETLKW);
-	int got_lock = 0;
-
-	lock.l_type = type;
-	lock.l_whence = SEEK_SET;
-	lock.l_start = offset;
-	lock.l_len = count;
-	lock.l_pid = 0;
-
-	do {
-		ret = fcntl(fd,use_lockwait?F_SETLK:op,&lock);
-		if (ret == 0) {
-			return 0;
-		}
-		if (ret == -1 && 
-		    (errno == EACCES || errno == EAGAIN || errno == EDEADLK)) {
-			struct lockwait_handle *h;
-			h = lockwait(ev, ev, fd, offset, count, 
-				     fcntl_lock_callback, &got_lock);
-			if (h == NULL) {
-				errno = ENOLCK;
-				return -1;
-			}
-			/* in real code we would return to the event loop */
-			while (!got_lock) {
-				event_loop_once(ev);
-			}
-			got_lock = 0;
-		}
-	} while (!got_lock);
-
-	return ret;
-}
-
-static void child(struct event_context *ev, int n)
-{
-	int fd;
-	int count=0;
-	struct timeval tv;
-	fd = open("test.dat", O_CREAT|O_RDWR, 0666);
-	if (fd == -1) {
-		perror("test.dat");
-		exit(1);
-	}
-
-	tv = timeval_current();
-
-	while (timeval_elapsed(&tv) < 10) {
-		int ret;
-		ret = fcntl_lock(ev, fd, F_SETLKW, 0, 1, F_WRLCK);
-		if (ret != 0) {
-			printf("Failed to get lock in child %d!\n", n);
-			break;
-		}
-		fcntl_lock(ev, fd, F_SETLK, 0, 1, F_UNLCK);
-		count++;
-	}
-
-	printf("child %2d %.0f ops/sec\n", n, count/timeval_elapsed(&tv));
-	_exit(0);
-}
-
-static int timelimit = 10;
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	pid_t *pids;
-	int nprogs = 2;
-	int i;
-	struct event_context *ev;
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		{ "timelimit", 't', POPT_ARG_INT, &timelimit, 0, "timelimit", "integer" },
-		{ "num-progs", 'n', POPT_ARG_INT, &nprogs, 0, "num_progs", "integer" },
-		POPT_TABLEEND
-	};
-	poptContext pc;
-	int opt;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	ev = event_context_init(NULL);
-
-	pids = talloc_array(ev, pid_t, nprogs);
-
-	/* create N processes fighting over the same lock */
-	for (i=0;i<nprogs;i++) {
-		pids[i] = fork();
-		if (pids[i] == 0) {
-			child(ev, i);
-		}
-	}
-
-	printf("Waiting for %d children ...\n", nprogs);
-
-	/* wait for our kids to finish playing */
-	for (i=0;i<nprogs;i++) {
-		waitpid(pids[i], NULL, 0);
-	}	
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tests/messaging.sh b/source4/cluster/ctdb/tests/messaging.sh
deleted file mode 100755
index 46f3e4dc77..0000000000
--- a/source4/cluster/ctdb/tests/messaging.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_messaging
-
-echo "Trying 2 nodes"
-bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.2:9001 $* &
-bin/ctdb_messaging --nlist tests/nodes.txt --listen 127.0.0.1:9001 $*
-wait
diff --git a/source4/cluster/ctdb/tests/nodes.txt b/source4/cluster/ctdb/tests/nodes.txt
index e1198b59ac..99b07328b3 100644
--- a/source4/cluster/ctdb/tests/nodes.txt
+++ b/source4/cluster/ctdb/tests/nodes.txt
@@ -1,2 +1,4 @@
-127.0.0.1:9001
-127.0.0.2:9001
+127.0.0.1
+127.0.0.2
+127.0.0.3
+127.0.0.4
diff --git a/source4/cluster/ctdb/tests/public_addresses b/source4/cluster/ctdb/tests/public_addresses
new file mode 100644
index 0000000000..97c85af7a8
--- /dev/null
+++ b/source4/cluster/ctdb/tests/public_addresses
@@ -0,0 +1,4 @@
+10.99.99.0/24
+10.99.99.1/24
+10.99.99.2/24
+10.99.99.3/24
diff --git a/source4/cluster/ctdb/tests/recover.sh b/source4/cluster/ctdb/tests/recover.sh
new file mode 100755
index 0000000000..c626441786
--- /dev/null
+++ b/source4/cluster/ctdb/tests/recover.sh
@@ -0,0 +1,107 @@
+#!/bin/sh
+
+killall -q ctdbd
+
+echo "Starting 4 ctdb daemons"
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.2 --socket=/tmp/ctdb.socket.127.0.0.2
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.3 --socket=/tmp/ctdb.socket.127.0.0.3
+bin/ctdbd --recovery-daemon --nlist tests/4nodes.txt --listen=127.0.0.4 --socket=/tmp/ctdb.socket.127.0.0.4
+
+echo
+echo "Attaching to some databases"
+bin/ctdb_control attach test1.tdb || exit 1
+bin/ctdb_control attach test2.tdb || exit 1
+bin/ctdb_control attach test3.tdb || exit 1
+bin/ctdb_control attach test4.tdb || exit 1
+
+echo "Clearing all databases to make sure they are all empty"
+bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^dbid://" -e "s/ .*$//" | while read DB; do
+	seq 0 3 | while read NODE; do
+		bin/ctdb_control cleardb $NODE $DB
+	done
+done
+
+
+echo
+echo
+echo "Printing all databases on all nodes. they should all be empty"
+echo "============================================================="
+bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^.*name://" -e "s/ .*$//" | while read DBNAME; do
+	seq 0 3 | while read NODE; do
+		echo "Content of DBNAME:$DBNAME NODE:$NODE :"
+		bin/ctdb_control catdb $DBNAME $NODE
+	done
+done
+
+echo
+echo
+echo "Populating the databases"
+./bin/ctdb_control writerecord 0 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 0 0x220c2a7b 1
+
+./bin/ctdb_control writerecord 1 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 1 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 1 0x220c2a7b 2
+
+./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 2 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 2 0x220c2a7b 3
+
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control writerecord 3 0x220c2a7b testkey1 testdata1
+./bin/ctdb_control setdmaster 3 0x220c2a7b 3
+
+
+echo
+echo
+echo "Printing all databases on all nodes. there should be a record there"
+echo "============================================================="
+bin/ctdb_control getdbmap 0 | egrep "^dbid:" | sed -e "s/^.*name://" -e "s/ .*$//" | while read DBNAME; do
+	seq 0 3 | while read NODE; do
+		echo "Content of DBNAME:$DBNAME NODE:$NODE :"
+		bin/ctdb_control catdb $DBNAME $NODE
+	done
+done
+
+echo
+echo
+echo "killing off node #2"
+echo "==================="
+CTDBPID=`./bin/ctdb_control getpid 2 | sed -e "s/Pid://"`
+kill $CTDBPID
+sleep 1
+
+
+echo
+echo
+echo "wait 3 seconds to let the recovery daemon do its job"
+echo "===================================================="
+sleep 3
+
+echo
+echo
+echo "Printing all databases on all nodes."
+echo "The databases should be the same now on all nodes"
+echo "and the record will have been migrated to node 0"
+echo "================================================="
+echo "Node 0:"
+bin/ctdb_control catdb test4.tdb 0
+echo "Node 1:"
+bin/ctdb_control catdb test4.tdb 1
+echo "Node 3:"
+bin/ctdb_control catdb test4.tdb 3
+echo "nodemap:"
+bin/ctdb_control getnodemap 0
+
+echo
+echo
+echo "Traverse the cluster and dump the database"
+bin/ctdb_control catdb test4.tdb
+
+
+#leave the ctdb daemons running   so one can look at the box in more detail
+#killall -q ctdbd
diff --git a/source4/cluster/ctdb/tests/run_tests.sh b/source4/cluster/ctdb/tests/run_tests.sh
new file mode 100755
index 0000000000..356a9b21a0
--- /dev/null
+++ b/source4/cluster/ctdb/tests/run_tests.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+tests/fetch.sh 4 || exit 1
+tests/bench.sh 4 || exit 1
+tests/ctdbd.sh || exit 1
+
+echo "All OK"
+exit 0
diff --git a/source4/cluster/ctdb/tests/start_daemons.sh b/source4/cluster/ctdb/tests/start_daemons.sh
new file mode 100755
index 0000000000..d3c99d075a
--- /dev/null
+++ b/source4/cluster/ctdb/tests/start_daemons.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+NUMNODES="$1"
+NODES=$2
+shift
+shift
+
+killall -q ctdbd
+
+CTDB_OPTIONS="--reclock=rec.lock --nlist $NODES --event-script=tests/events --logfile=-  --dbdir=test.db $*"
+if [ `id -u` -eq 0 ]; then
+    CTDB_OPTIONS="$CTDB_OPTIONS --public-addresses=tests/public_addresses --public-interface=lo"
+fi
+
+echo "Starting $NUMNODES ctdb daemons"
+for i in `seq 1 $NUMNODES`; do
+    $VALGRIND bin/ctdbd --socket=sock.$i $CTDB_OPTIONS || exit 1
+done
+ln -sf $PWD/sock.1 /tmp/ctdb.socket || exit 1
+
+while bin/ctdb status | grep RECOVERY > /dev/null; do
+    echo "`date` Waiting for recovery"
+    sleep 1;
+done
+
+echo "$NUMNODES daemons started"
+
+exit 0
diff --git a/source4/cluster/ctdb/tests/test.sh b/source4/cluster/ctdb/tests/test.sh
deleted file mode 100755
index 23fdb8d6ce..0000000000
--- a/source4/cluster/ctdb/tests/test.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-
-killall -q ctdb_test
-
-
-echo "Trying 2 nodes ..."
-$VALGRIND bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.1:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/nodes.txt --listen 127.0.0.2:9001
-wait
-
-echo "Trying 4 nodes ..."
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.1:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.2:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.3:9001 &
-$VALGRIND bin/ctdb_test --nlist tests/4nodes.txt --listen 127.0.0.4:9001
-wait
-
diff --git a/source4/cluster/ctdb/tests/test1.sh b/source4/cluster/ctdb/tests/test1.sh
deleted file mode 100755
index 42256f22f9..0000000000
--- a/source4/cluster/ctdb/tests/test1.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-
-echo "Testing daemon mode"
-bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001
-wait
-
-echo "Testing self connect"
-bin/ctdb_test --nlist tests/1node.txt --listen 127.0.0.1:9001 --self-connect
-wait
diff --git a/source4/cluster/ctdb/tools/ctdb.c b/source4/cluster/ctdb/tools/ctdb.c
new file mode 100644
index 0000000000..2308261d00
--- /dev/null
+++ b/source4/cluster/ctdb/tools/ctdb.c
@@ -0,0 +1,1024 @@
+/* 
+   ctdb control tool
+
+   Copyright (C) Andrew Tridgell  2007
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "system/filesys.h"
+#include "system/network.h"
+#include "popt.h"
+#include "cmdline.h"
+#include "../include/ctdb.h"
+#include "../include/ctdb_private.h"
+
+static void usage(void);
+
+static struct {
+	int timelimit;
+	uint32_t vnn;
+	int machinereadable;
+} options;
+
+#define TIMELIMIT() timeval_current_ofs(options.timelimit, 0)
+
+/*
+  see if a process exists
+ */
+static int control_process_exists(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	uint32_t vnn, pid;
+	int ret;
+	if (argc < 1) {
+		usage();
+	}
+
+	if (sscanf(argv[0], "%u:%u", &vnn, &pid) != 2) {
+		printf("Badly formed vnn:pid\n");
+		return -1;
+	}
+
+	ret = ctdb_ctrl_process_exists(ctdb, vnn, pid);
+	if (ret == 0) {
+		printf("%u:%u exists\n", vnn, pid);
+	} else {
+		printf("%u:%u does not exist\n", vnn, pid);
+	}
+	return ret;
+}
+
+/*
+  display statistics structure
+ */
+static void show_statistics(struct ctdb_statistics *s)
+{
+	TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+	int i;
+	const char *prefix=NULL;
+	int preflen=0;
+	const struct {
+		const char *name;
+		uint32_t offset;
+	} fields[] = {
+#define STATISTICS_FIELD(n) { #n, offsetof(struct ctdb_statistics, n) }
+		STATISTICS_FIELD(num_clients),
+		STATISTICS_FIELD(frozen),
+		STATISTICS_FIELD(recovering),
+		STATISTICS_FIELD(client_packets_sent),
+		STATISTICS_FIELD(client_packets_recv),
+		STATISTICS_FIELD(node_packets_sent),
+		STATISTICS_FIELD(node_packets_recv),
+		STATISTICS_FIELD(keepalive_packets_sent),
+		STATISTICS_FIELD(keepalive_packets_recv),
+		STATISTICS_FIELD(node.req_call),
+		STATISTICS_FIELD(node.reply_call),
+		STATISTICS_FIELD(node.req_dmaster),
+		STATISTICS_FIELD(node.reply_dmaster),
+		STATISTICS_FIELD(node.reply_error),
+		STATISTICS_FIELD(node.req_message),
+		STATISTICS_FIELD(node.req_control),
+		STATISTICS_FIELD(node.reply_control),
+		STATISTICS_FIELD(client.req_call),
+		STATISTICS_FIELD(client.req_message),
+		STATISTICS_FIELD(client.req_control),
+		STATISTICS_FIELD(timeouts.call),
+		STATISTICS_FIELD(timeouts.control),
+		STATISTICS_FIELD(timeouts.traverse),
+		STATISTICS_FIELD(total_calls),
+		STATISTICS_FIELD(pending_calls),
+		STATISTICS_FIELD(lockwait_calls),
+		STATISTICS_FIELD(pending_lockwait_calls),
+		STATISTICS_FIELD(memory_used),
+		STATISTICS_FIELD(max_hop_count),
+	};
+	printf("CTDB version %u\n", CTDB_VERSION);
+	for (i=0;i<ARRAY_SIZE(fields);i++) {
+		if (strchr(fields[i].name, '.')) {
+			preflen = strcspn(fields[i].name, ".")+1;
+			if (!prefix || strncmp(prefix, fields[i].name, preflen) != 0) {
+				prefix = fields[i].name;
+				printf(" %*.*s\n", preflen-1, preflen-1, fields[i].name);
+			}
+		} else {
+			preflen = 0;
+		}
+		printf(" %*s%-22s%*s%10u\n", 
+		       preflen?4:0, "",
+		       fields[i].name+preflen, 
+		       preflen?0:4, "",
+		       *(uint32_t *)(fields[i].offset+(uint8_t *)s));
+	}
+	printf(" %-30s     %.6f sec\n", "max_call_latency", s->max_call_latency);
+	printf(" %-30s     %.6f sec\n", "max_lockwait_latency", s->max_lockwait_latency);
+	talloc_free(tmp_ctx);
+}
+
+/*
+  display remote ctdb statistics combined from all nodes
+ */
+static int control_statistics_all(struct ctdb_context *ctdb)
+{
+	int ret, i;
+	struct ctdb_statistics statistics;
+	uint32_t *nodes;
+	uint32_t num_nodes;
+
+	nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
+	CTDB_NO_MEMORY(ctdb, nodes);
+	
+	ZERO_STRUCT(statistics);
+
+	for (i=0;i<num_nodes;i++) {
+		struct ctdb_statistics s1;
+		int j;
+		uint32_t *v1 = (uint32_t *)&s1;
+		uint32_t *v2 = (uint32_t *)&statistics;
+		uint32_t num_ints = 
+			offsetof(struct ctdb_statistics, __last_counter) / sizeof(uint32_t);
+		ret = ctdb_ctrl_statistics(ctdb, nodes[i], &s1);
+		if (ret != 0) {
+			printf("Unable to get statistics from node %u\n", nodes[i]);
+			return ret;
+		}
+		for (j=0;j<num_ints;j++) {
+			v2[j] += v1[j];
+		}
+		statistics.max_hop_count = 
+			MAX(statistics.max_hop_count, s1.max_hop_count);
+		statistics.max_call_latency = 
+			MAX(statistics.max_call_latency, s1.max_call_latency);
+		statistics.max_lockwait_latency = 
+			MAX(statistics.max_lockwait_latency, s1.max_lockwait_latency);
+	}
+	talloc_free(nodes);
+	printf("Gathered statistics for %u nodes\n", num_nodes);
+	show_statistics(&statistics);
+	return 0;
+}
+
+/*
+  display remote ctdb statistics
+ */
+static int control_statistics(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	struct ctdb_statistics statistics;
+
+	if (options.vnn == CTDB_BROADCAST_ALL) {
+		return control_statistics_all(ctdb);
+	}
+
+	ret = ctdb_ctrl_statistics(ctdb, options.vnn, &statistics);
+	if (ret != 0) {
+		printf("Unable to get statistics from node %u\n", options.vnn);
+		return ret;
+	}
+	show_statistics(&statistics);
+	return 0;
+}
+
+
+/*
+  reset remote ctdb statistics
+ */
+static int control_statistics_reset(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_statistics_reset(ctdb, options.vnn);
+	if (ret != 0) {
+		printf("Unable to reset statistics on node %u\n", options.vnn);
+		return ret;
+	}
+	return 0;
+}
+
+
+/*
+  display remote ctdb status
+ */
+static int control_status(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int i, ret;
+	struct ctdb_vnn_map *vnnmap=NULL;
+	struct ctdb_node_map *nodemap=NULL;
+	uint32_t recmode, recmaster;
+	uint32_t myvnn;
+
+	myvnn = ctdb_ctrl_getvnn(ctdb, TIMELIMIT(), options.vnn);
+
+	ret = ctdb_ctrl_getnodemap(ctdb, TIMELIMIT(), options.vnn, ctdb, &nodemap);
+	if (ret != 0) {
+		printf("Unable to get nodemap from node %u\n", options.vnn);
+		return ret;
+	}
+
+	if(options.machinereadable){
+		printf(":Node:IP:Disonnected:Disabled:Permanently Disabled:\n");
+		for(i=0;i<nodemap->num;i++){
+			printf(":%d:%s:%d:%d:%d:\n", nodemap->nodes[i].vnn,
+				inet_ntoa(nodemap->nodes[i].sin.sin_addr),
+			       !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
+			       !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
+			       !!(nodemap->nodes[i].flags&NODE_FLAGS_PERMANENTLY_DISABLED));
+		}
+		return 0;
+	}
+
+	printf("Number of nodes:%d\n", nodemap->num);
+	for(i=0;i<nodemap->num;i++){
+		static const struct {
+			uint32_t flag;
+			const char *name;
+		} flag_names[] = {
+			{ NODE_FLAGS_DISCONNECTED,          "DISCONNECTED" },
+			{ NODE_FLAGS_PERMANENTLY_DISABLED,  "DISABLED" },
+			{ NODE_FLAGS_BANNED,                "BANNED" },
+			{ NODE_FLAGS_UNHEALTHY,             "UNHEALTHY" },
+		};
+		char *flags_str = NULL;
+		int j;
+		for (j=0;j<ARRAY_SIZE(flag_names);j++) {
+			if (nodemap->nodes[i].flags & flag_names[j].flag) {
+				if (flags_str == NULL) {
+					flags_str = talloc_strdup(ctdb, flag_names[j].name);
+				} else {
+					flags_str = talloc_asprintf_append(flags_str, "|%s",
+									   flag_names[j].name);
+				}
+				CTDB_NO_MEMORY_FATAL(ctdb, flags_str);
+			}
+		}
+		if (flags_str == NULL) {
+			flags_str = talloc_strdup(ctdb, "OK");
+			CTDB_NO_MEMORY_FATAL(ctdb, flags_str);
+		}
+		printf("vnn:%d %-16s %s%s\n", nodemap->nodes[i].vnn,
+		       inet_ntoa(nodemap->nodes[i].sin.sin_addr),
+		       flags_str,
+		       nodemap->nodes[i].vnn == myvnn?" (THIS NODE)":"");
+		talloc_free(flags_str);
+	}
+
+	ret = ctdb_ctrl_getvnnmap(ctdb, TIMELIMIT(), options.vnn, ctdb, &vnnmap);
+	if (ret != 0) {
+		printf("Unable to get vnnmap from node %u\n", options.vnn);
+		return ret;
+	}
+	printf("Generation:%d\n",vnnmap->generation);
+	printf("Size:%d\n",vnnmap->size);
+	for(i=0;i<vnnmap->size;i++){
+		printf("hash:%d lmaster:%d\n", i, vnnmap->map[i]);
+	}
+
+	ret = ctdb_ctrl_getrecmode(ctdb, TIMELIMIT(), options.vnn, &recmode);
+	if (ret != 0) {
+		printf("Unable to get recmode from node %u\n", options.vnn);
+		return ret;
+	}
+	printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, TIMELIMIT(), options.vnn, &recmaster);
+	if (ret != 0) {
+		printf("Unable to get recmaster from node %u\n", options.vnn);
+		return ret;
+	}
+	printf("Recovery master:%d\n",recmaster);
+
+	return 0;
+}
+
+/*
+  kill a tcp connection
+ */
+static int kill_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int i, ret, numrst;
+	struct sockaddr_in src, dst;
+
+	if (argc < 3) {
+		usage();
+	}
+
+	if (!parse_ip_port(argv[0], &src)) {
+		printf("Bad IP:port '%s'\n", argv[0]);
+		return -1;
+	}
+
+	if (!parse_ip_port(argv[1], &dst)) {
+		printf("Bad IP:port '%s'\n", argv[1]);
+		return -1;
+	}
+
+	numrst = strtoul(argv[2], NULL, 0);
+
+	for (i=0;i<numrst;i++) {
+		ret = ctdb_sys_kill_tcp(ctdb->ev, &src, &dst);
+
+		printf("ret:%d\n", ret);
+		if (ret==0) {
+			return 0;
+		}
+	}
+
+	return -1;
+}
+
+/*
+  send a tcp tickle ack
+ */
+static int tickle_tcp(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	struct sockaddr_in src, dst;
+
+	if (argc < 2) {
+		usage();
+	}
+
+	if (!parse_ip_port(argv[0], &src)) {
+		printf("Bad IP:port '%s'\n", argv[0]);
+		return -1;
+	}
+
+	if (!parse_ip_port(argv[1], &dst)) {
+		printf("Bad IP:port '%s'\n", argv[1]);
+		return -1;
+	}
+
+	ret = ctdb_sys_send_tcp(&src, &dst, 0, 0, 0);
+	if (ret==0) {
+		return 0;
+	}
+	printf("Error while sending tickle ack\n");
+
+	return -1;
+}
+
+/*
+  display public ip status
+ */
+static int control_ip(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int i, ret;
+	struct ctdb_all_public_ips *ips;
+	uint32_t myvnn;
+
+	myvnn = ctdb_ctrl_getvnn(ctdb, TIMELIMIT(), options.vnn);
+
+	ret = ctdb_ctrl_get_public_ips(ctdb, TIMELIMIT(), options.vnn, ctdb, &ips);
+	if (ret != 0) {
+		printf("Unable to get public ips from node %u\n", options.vnn);
+		return ret;
+	}
+
+	if(options.machinereadable){
+		printf(":Public IP:Node:\n");
+		for(i=0;i<ips->num;i++){
+			printf(":%s:%d:\n",
+			inet_ntoa(ips->ips[i].sin.sin_addr),
+			ips->ips[i].takeover_vnn);
+		}
+		return 0;
+	}
+
+
+	printf("Number of nodes:%d\n", ips->num);
+	for(i=0;i<ips->num;i++){
+		printf("%-16s %d\n",
+			inet_ntoa(ips->ips[i].sin.sin_addr),
+			ips->ips[i].takeover_vnn);
+	}
+
+	return 0;
+}
+
+/*
+  display pid of a ctdb daemon
+ */
+static int control_getpid(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	uint32_t pid;
+	int ret;
+
+	ret = ctdb_ctrl_getpid(ctdb, TIMELIMIT(), options.vnn, &pid);
+	if (ret != 0) {
+		printf("Unable to get daemon pid from node %u\n", options.vnn);
+		return ret;
+	}
+	printf("Pid:%d\n", pid);
+
+	return 0;
+}
+
+/*
+  disable a remote node
+ */
+static int control_disable(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.vnn, NODE_FLAGS_PERMANENTLY_DISABLED, 0);
+	if (ret != 0) {
+		printf("Unable to disable node %u\n", options.vnn);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+  enable a disabled remote node
+ */
+static int control_enable(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_ctrl_modflags(ctdb, TIMELIMIT(), options.vnn, 0, NODE_FLAGS_PERMANENTLY_DISABLED);
+	if (ret != 0) {
+		printf("Unable to enable node %u\n", options.vnn);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+  ban a node from the cluster
+ */
+static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	uint32_t recmaster;
+	struct ctdb_ban_info b;
+	TDB_DATA data;
+	uint32_t ban_time;
+
+	if (argc < 1) {
+		usage();
+	}
+
+	ban_time = strtoul(argv[0], NULL, 0);
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, TIMELIMIT(), options.vnn, &recmaster);
+	if (ret != 0) {
+		DEBUG(0,("Failed to find the recmaster\n"));
+		return -1;
+	}
+
+	b.vnn = options.vnn;
+	b.ban_time = ban_time;
+
+	data.dptr = (uint8_t *)&b;
+	data.dsize = sizeof(b);
+
+	ret = ctdb_send_message(ctdb, recmaster, CTDB_SRVID_BAN_NODE, data);
+	if (ret != 0) {
+		DEBUG(0,("Failed to tell the recmaster to ban node %u\n", options.vnn));
+		return -1;
+	}
+	
+	return 0;
+}
+
+
+/*
+  unban a node from the cluster
+ */
+static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	uint32_t recmaster;
+	TDB_DATA data;
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, TIMELIMIT(), options.vnn, &recmaster);
+	if (ret != 0) {
+		DEBUG(0,("Failed to find the recmaster\n"));
+		return -1;
+	}
+
+	data.dptr = (uint8_t *)&options.vnn;
+	data.dsize = sizeof(uint32_t);
+
+	ret = ctdb_send_message(ctdb, recmaster, CTDB_SRVID_UNBAN_NODE, data);
+	if (ret != 0) {
+		DEBUG(0,("Failed to tell the recmaster to unban node %u\n", options.vnn));
+		return -1;
+	}
+	
+	return 0;
+}
+
+
+/*
+  shutdown a daemon
+ */
+static int control_shutdown(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_ctrl_shutdown(ctdb, TIMELIMIT(), options.vnn);
+	if (ret != 0) {
+		printf("Unable to shutdown node %u\n", options.vnn);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+  trigger a recovery
+ */
+static int control_recover(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_ctrl_freeze(ctdb, TIMELIMIT(), options.vnn);
+	if (ret != 0) {
+		printf("Unable to freeze node\n");
+		return ret;
+	}
+
+	ret = ctdb_ctrl_setrecmode(ctdb, TIMELIMIT(), options.vnn, CTDB_RECOVERY_ACTIVE);
+	if (ret != 0) {
+		printf("Unable to set recovery mode\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+
+/*
+  display monitoring mode of a remote node
+ */
+static int control_getmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	uint32_t monmode;
+	int ret;
+
+	ret = ctdb_ctrl_getmonmode(ctdb, TIMELIMIT(), options.vnn, &monmode);
+	if (ret != 0) {
+		printf("Unable to get monmode from node %u\n", options.vnn);
+		return ret;
+	}
+	printf("Monitoring mode:%s (%d)\n",monmode==CTDB_MONITORING_ACTIVE?"ACTIVE":"DISABLED",monmode);
+
+	return 0;
+}
+
+/*
+  set the monitoring mode of a remote node
+ */
+static int control_setmonmode(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	uint32_t monmode;
+	int ret;
+
+	if (argc < 1) {
+		usage();
+	}
+
+	monmode = strtoul(argv[0], NULL, 0);
+
+	ret = ctdb_ctrl_setmonmode(ctdb, TIMELIMIT(), options.vnn, monmode);
+	if (ret != 0) {
+		printf("Unable to set monmode on node %u\n", options.vnn);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+  display remote list of keys/data for a db
+ */
+static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	const char *db_name;
+	struct ctdb_db_context *ctdb_db;
+	int ret;
+
+	if (argc < 1) {
+		usage();
+	}
+
+	db_name = argv[0];
+	ctdb_db = ctdb_attach(ctdb, db_name);
+
+	if (ctdb_db == NULL) {
+		DEBUG(0,("Unable to attach to database '%s'\n", db_name));
+		return -1;
+	}
+
+	/* traverse and dump the cluster tdb */
+	ret = ctdb_dump_db(ctdb_db, stdout);
+	if (ret == -1) {
+		printf("Unable to dump database\n");
+		return -1;
+	}
+	talloc_free(ctdb_db);
+
+	printf("Dumped %d records\n", ret);
+	return 0;
+}
+
+
+/*
+  display a list of the databases on a remote ctdb
+ */
+static int control_getdbmap(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int i, ret;
+	struct ctdb_dbid_map *dbmap=NULL;
+
+	ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.vnn, ctdb, &dbmap);
+	if (ret != 0) {
+		printf("Unable to get dbids from node %u\n", options.vnn);
+		return ret;
+	}
+
+	printf("Number of databases:%d\n", dbmap->num);
+	for(i=0;i<dbmap->num;i++){
+		const char *path;
+		const char *name;
+
+		ctdb_ctrl_getdbpath(ctdb, TIMELIMIT(), options.vnn, dbmap->dbids[i], ctdb, &path);
+		ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.vnn, dbmap->dbids[i], ctdb, &name);
+		printf("dbid:0x%08x name:%s path:%s\n", dbmap->dbids[i], name, path);
+	}
+
+	return 0;
+}
+
+/*
+  ping a node
+ */
+static int control_ping(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	struct timeval tv = timeval_current();
+	ret = ctdb_ctrl_ping(ctdb, options.vnn);
+	if (ret == -1) {
+		printf("Unable to get ping response from node %u\n", options.vnn);
+	} else {
+		printf("response from %u time=%.6f sec  (%d clients)\n", 
+		       options.vnn, timeval_elapsed(&tv), ret);
+	}
+	return 0;
+}
+
+
+/*
+  get a tunable
+ */
+static int control_getvar(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	const char *name;
+	uint32_t value;
+	int ret;
+
+	if (argc < 1) {
+		usage();
+	}
+
+	name = argv[0];
+	ret = ctdb_ctrl_get_tunable(ctdb, TIMELIMIT(), options.vnn, name, &value);
+	if (ret == -1) {
+		printf("Unable to get tunable variable '%s'\n", name);
+		return -1;
+	}
+
+	printf("%-19s = %u\n", name, value);
+	return 0;
+}
+
+/*
+  set a tunable
+ */
+static int control_setvar(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	const char *name;
+	uint32_t value;
+	int ret;
+
+	if (argc < 2) {
+		usage();
+	}
+
+	name = argv[0];
+	value = strtoul(argv[1], NULL, 0);
+
+	ret = ctdb_ctrl_set_tunable(ctdb, TIMELIMIT(), options.vnn, name, value);
+	if (ret == -1) {
+		printf("Unable to set tunable variable '%s'\n", name);
+		return -1;
+	}
+	return 0;
+}
+
+/*
+  list all tunables
+ */
+static int control_listvars(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	uint32_t count;
+	const char **list;
+	int ret, i;
+
+	ret = ctdb_ctrl_list_tunables(ctdb, TIMELIMIT(), options.vnn, ctdb, &list, &count);
+	if (ret == -1) {
+		printf("Unable to list tunable variables\n");
+		return -1;
+	}
+
+	for (i=0;i<count;i++) {
+		control_getvar(ctdb, 1, &list[i]);
+	}
+
+	talloc_free(list);
+	
+	return 0;
+}
+
+/*
+  display debug level on a node
+ */
+static int control_getdebug(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	uint32_t level;
+
+	ret = ctdb_ctrl_get_debuglevel(ctdb, options.vnn, &level);
+	if (ret != 0) {
+		printf("Unable to get debuglevel response from node %u\n", 
+		       options.vnn);
+	} else {
+		printf("Node %u is at debug level %u\n", options.vnn, level);
+	}
+	return 0;
+}
+
+
+/*
+  set debug level on a node or all nodes
+ */
+static int control_setdebug(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	uint32_t level;
+
+	if (argc < 1) {
+		usage();
+	}
+
+	level = strtoul(argv[0], NULL, 0);
+
+	ret = ctdb_ctrl_set_debuglevel(ctdb, options.vnn, level);
+	if (ret != 0) {
+		printf("Unable to set debug level on node %u\n", options.vnn);
+	}
+	return 0;
+}
+
+
+/*
+  freeze a node
+ */
+static int control_freeze(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_ctrl_freeze(ctdb, TIMELIMIT(), options.vnn);
+	if (ret != 0) {
+		printf("Unable to freeze node %u\n", options.vnn);
+	}		
+	return 0;
+}
+
+/*
+  thaw a node
+ */
+static int control_thaw(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+
+	ret = ctdb_ctrl_thaw(ctdb, TIMELIMIT(), options.vnn);
+	if (ret != 0) {
+		printf("Unable to thaw node %u\n", options.vnn);
+	}		
+	return 0;
+}
+
+
+/*
+  attach to a database
+ */
+static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	const char *db_name;
+	struct ctdb_db_context *ctdb_db;
+
+	if (argc < 1) {
+		usage();
+	}
+	db_name = argv[0];
+
+	ctdb_db = ctdb_attach(ctdb, db_name);
+	if (ctdb_db == NULL) {
+		DEBUG(0,("Unable to attach to database '%s'\n", db_name));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  dump memory usage
+ */
+static int control_dumpmemory(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	return ctdb_control(ctdb, options.vnn, 0, CTDB_CONTROL_DUMP_MEMORY,
+			    CTDB_CTRL_FLAG_NOREPLY, tdb_null, NULL, NULL, NULL, NULL, NULL);
+}
+
+
+static const struct {
+	const char *name;
+	int (*fn)(struct ctdb_context *, int, const char **);
+	bool auto_all;
+	const char *msg;
+	const char *args;
+} ctdb_commands[] = {
+	{ "status",          control_status,            true,  "show node status" },
+	{ "ping",            control_ping,              true,  "ping all nodes" },
+	{ "getvar",          control_getvar,            true,  "get a tunable variable",               "<name>"},
+	{ "setvar",          control_setvar,            true,  "set a tunable variable",               "<name> <value>"},
+	{ "listvars",        control_listvars,          true,  "list tunable variables"},
+	{ "statistics",      control_statistics,        false, "show statistics" },
+	{ "statisticsreset", control_statistics_reset,  true,  "reset statistics"},
+	{ "ip",              control_ip,                true,  "show which public ip's that ctdb manages" },
+	{ "process-exists",  control_process_exists,    true,  "check if a process exists on a node",  "<pid>"},
+	{ "getdbmap",        control_getdbmap,          true,  "show the database map" },
+	{ "catdb",           control_catdb,             true,  "dump a database" ,                     "<dbname>"},
+	{ "getmonmode",      control_getmonmode,        true,  "show monitoring mode" },
+	{ "setmonmode",      control_setmonmode,        true,  "set monitoring mode", "<0|1>" },
+	{ "setdebug",        control_setdebug,          true,  "set debug level",                      "<debuglevel>" },
+	{ "getdebug",        control_getdebug,          true,  "get debug level" },
+	{ "attach",          control_attach,            true,  "attach to a database",                 "<dbname>" },
+	{ "dumpmemory",      control_dumpmemory,        true,  "dump memory map to logs" },
+	{ "getpid",          control_getpid,            true,  "get ctdbd process ID" },
+	{ "disable",         control_disable,           true,  "disable a nodes public IP" },
+	{ "enable",          control_enable,            true,  "enable a nodes public IP" },
+	{ "ban",             control_ban,               true,  "ban a node from the cluster",          "<bantime|0>"},
+	{ "unban",           control_unban,             true,  "unban a node from the cluster" },
+	{ "shutdown",        control_shutdown,          true,  "shutdown ctdbd" },
+	{ "recover",         control_recover,           true,  "force recovery" },
+	{ "freeze",          control_freeze,            true,  "freeze all databases" },
+	{ "thaw",            control_thaw,              true,  "thaw all databases" },
+	{ "killtcp",         kill_tcp,                  false, "kill a tcp connection. Try <num> times.", "<srcip:port> <dstip:port> <num>" },
+	{ "tickle",          tickle_tcp,                false, "send a tcp tickle ack", "<srcip:port> <dstip:port>" },
+};
+
+/*
+  show usage message
+ */
+static void usage(void)
+{
+	int i;
+	printf(
+"Usage: ctdb [options] <control>\n" \
+"Options:\n" \
+"   -n <node>          choose node number, or 'all' (defaults to local node)\n"
+"   -Y                 generate machinereadable output\n"
+"   -t <timelimit>     set timelimit for control in seconds (default %u)\n", options.timelimit);
+	printf("Controls:\n");
+	for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
+		printf("  %-15s %-27s  %s\n", 
+		       ctdb_commands[i].name, 
+		       ctdb_commands[i].args?ctdb_commands[i].args:"",
+		       ctdb_commands[i].msg);
+	}
+	exit(1);
+}
+
+
+/*
+  main program
+*/
+int main(int argc, const char *argv[])
+{
+	struct ctdb_context *ctdb;
+	char *nodestring = NULL;
+	struct poptOption popt_options[] = {
+		POPT_AUTOHELP
+		POPT_CTDB_CMDLINE
+		{ "timelimit", 't', POPT_ARG_INT, &options.timelimit, 0, "timelimit", "integer" },
+		{ "node",      'n', POPT_ARG_STRING, &nodestring, 0, "node", "integer|all" },
+		{ "machinereadable", 'Y', POPT_ARG_NONE, &options.machinereadable, 0, "enable machinereadable output", NULL },
+		POPT_TABLEEND
+	};
+	int opt;
+	const char **extra_argv;
+	int extra_argc = 0;
+	int ret=-1, i;
+	poptContext pc;
+	struct event_context *ev;
+	const char *control;
+
+	/* set some defaults */
+	options.timelimit = 3;
+	options.vnn = CTDB_CURRENT_NODE;
+
+	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
+
+	while ((opt = poptGetNextOpt(pc)) != -1) {
+		switch (opt) {
+		default:
+			fprintf(stderr, "Invalid option %s: %s\n", 
+				poptBadOption(pc, 0), poptStrerror(opt));
+			exit(1);
+		}
+	}
+
+	/* setup the remaining options for the main program to use */
+	extra_argv = poptGetArgs(pc);
+	if (extra_argv) {
+		extra_argv++;
+		while (extra_argv[extra_argc]) extra_argc++;
+	}
+
+	if (extra_argc < 1) {
+		usage();
+	}
+
+	/* setup the node number to contact */
+	if (nodestring != NULL) {
+		if (strcmp(nodestring, "all") == 0) {
+			options.vnn = CTDB_BROADCAST_ALL;
+		} else {
+			options.vnn = strtoul(nodestring, NULL, 0);
+		}
+	}
+
+	control = extra_argv[0];
+
+	ev = event_context_init(NULL);
+
+	/* initialise ctdb */
+	ctdb = ctdb_cmdline_client(ev);
+	if (ctdb == NULL) {
+		printf("Failed to init ctdb\n");
+		exit(1);
+	}
+
+	for (i=0;i<ARRAY_SIZE(ctdb_commands);i++) {
+		if (strcmp(control, ctdb_commands[i].name) == 0) {
+			int j;
+
+			if (options.vnn == CTDB_CURRENT_NODE) {
+				options.vnn = ctdb_ctrl_getvnn(ctdb, TIMELIMIT(), options.vnn);		
+			}
+
+			if (ctdb_commands[i].auto_all && 
+			    options.vnn == CTDB_BROADCAST_ALL) {
+				uint32_t *nodes;
+				uint32_t num_nodes;
+				ret = 0;
+
+				nodes = ctdb_get_connected_nodes(ctdb, TIMELIMIT(), ctdb, &num_nodes);
+				CTDB_NO_MEMORY(ctdb, nodes);
+	
+				for (j=0;j<num_nodes;j++) {
+					options.vnn = nodes[j];
+					ret |= ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
+				}
+				talloc_free(nodes);
+			} else {
+				ret = ctdb_commands[i].fn(ctdb, extra_argc-1, extra_argv+1);
+			}
+			break;
+		}
+	}
+
+	if (i == ARRAY_SIZE(ctdb_commands)) {
+		printf("Unknown control '%s'\n", control);
+		exit(1);
+	}
+
+	return ret;
+}
diff --git a/source4/cluster/ctdb/tools/ctdb_status.c b/source4/cluster/ctdb/tools/ctdb_status.c
deleted file mode 100644
index ffb1fc7716..0000000000
--- a/source4/cluster/ctdb/tools/ctdb_status.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/* 
-   ctdb status tool
-
-   Copyright (C) Andrew Tridgell  2007
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "lib/events/events.h"
-#include "system/filesys.h"
-#include "popt.h"
-#include "cmdline.h"
-#include "../include/ctdb_private.h"
-
-
-/*
-  display status structure
- */
-static void show_status(struct ctdb_status *s)
-{
-	printf(" client_packets_sent     %u\n", s->client_packets_sent);
-	printf(" client_packets_recv     %u\n", s->client_packets_recv);
-	printf("   req_call              %u\n", s->client.req_call);
-	printf("   req_message           %u\n", s->client.req_message);
-	printf("   req_finished          %u\n", s->client.req_finished);
-	printf("   req_register          %u\n", s->client.req_register);
-	printf("   req_connect_wait      %u\n", s->client.req_connect_wait);
-	printf("   req_shutdown          %u\n", s->client.req_shutdown);
-	printf("   req_status            %u\n", s->client.req_status);
-	printf(" node_packets_sent       %u\n", s->node_packets_sent);
-	printf(" node_packets_recv       %u\n", s->node_packets_recv);
-	printf("   req_call              %u\n", s->client.req_call);
-	printf("   reply_call            %u\n", s->count.reply_call);
-	printf("   reply_redirect        %u\n", s->count.reply_redirect);
-	printf("   req_dmaster           %u\n", s->count.req_dmaster);
-	printf("   reply_dmaster         %u\n", s->count.reply_dmaster);
-	printf("   reply_error           %u\n", s->count.reply_error);
-	printf("   reply_redirect        %u\n", s->count.reply_redirect);
-	printf("   req_message           %u\n", s->count.req_message);
-	printf("   req_finished          %u\n", s->count.req_finished);
-	printf(" total_calls             %u\n", s->total_calls);
-	printf(" pending_calls           %u\n", s->pending_calls);
-	printf(" lockwait_calls          %u\n", s->lockwait_calls);
-	printf(" pending_lockwait_calls  %u\n", s->pending_lockwait_calls);
-	printf(" max_call_latency        %.6f seconds\n", s->max_call_latency);
-	printf(" max_lockwait_latency    %.6f seconds\n", s->max_lockwait_latency);
-}
-
-/*
-  show usage message
- */
-static void usage(void)
-{
-	printf("Usage: ctdb_status <socketpath>\n");
-	exit(1);
-}
-
-/*
-  main program
-*/
-int main(int argc, const char *argv[])
-{
-	struct ctdb_context *ctdb;
-	struct poptOption popt_options[] = {
-		POPT_AUTOHELP
-		POPT_CTDB_CMDLINE
-		POPT_TABLEEND
-	};
-	int opt;
-	const char **extra_argv;
-	int extra_argc = 0;
-	int ret;
-	poptContext pc;
-	struct event_context *ev;
-	const char *ctdb_socket;
-	struct ctdb_status status;
-
-	pc = poptGetContext(argv[0], argc, argv, popt_options, POPT_CONTEXT_KEEP_FIRST);
-
-	while ((opt = poptGetNextOpt(pc)) != -1) {
-		switch (opt) {
-		default:
-			fprintf(stderr, "Invalid option %s: %s\n", 
-				poptBadOption(pc, 0), poptStrerror(opt));
-			exit(1);
-		}
-	}
-
-	/* setup the remaining options for the main program to use */
-	extra_argv = poptGetArgs(pc);
-	if (extra_argv) {
-		extra_argv++;
-		while (extra_argv[extra_argc]) extra_argc++;
-	}
-
-	if (extra_argc < 1) {
-		usage();
-	}
-
-	ctdb_socket = extra_argv[0];
-
-	ev = event_context_init(NULL);
-
-	/* initialise ctdb */
-	ctdb = ctdb_cmdline_client(ev, ctdb_socket);
-	if (ctdb == NULL) {
-		printf("Failed to init ctdb\n");
-		exit(1);
-	}
-
-	ret = ctdb_status(ctdb, &status);
-	if (ret != 0) {
-		printf("Failed to get ctdb status\n");
-		exit(1);
-	}
-
-	show_status(&status);
-
-	return 0;
-}
diff --git a/source4/cluster/ctdb/tools/onnode.rsh b/source4/cluster/ctdb/tools/onnode.rsh
new file mode 100644
index 0000000000..cdda3256f7
--- /dev/null
+++ b/source4/cluster/ctdb/tools/onnode.rsh
@@ -0,0 +1,43 @@
+#!/bin/sh
+# onnode script for rsh
+
+if [ $# -lt 2 ]; then
+cat <<EOF
+Usage: onnode <nodenum|all> <command>
+EOF
+exit 1
+fi
+
+NODE="$1"
+shift
+SCRIPT="$*"
+
+NODES=/etc/ctdb/nodes
+
+NUMNODES=`egrep '^[[:alnum:]]' $NODES | wc -l`
+MAXNODE=`expr $NUMNODES - 1`
+
+if [ $NODE = "all" ]; then
+    for a in `egrep '^[[:alnum:]]' $NODES`; do
+	if [ -f "$SCRIPT" ]; then
+	    rsh $a at -f $SCRIPT now
+	else
+	    rsh $a $SCRIPT
+	fi
+    done
+    exit 0
+fi
+
+if [ $NODE -gt $MAXNODE ]; then
+    echo "Node $NODE doesn't exist"
+    exit 1
+fi
+
+NODEPLUSONE=`expr $NODE + 1`
+a=`egrep '^[[:alnum:]]' $NODES | head -$NODEPLUSONE | tail -1`
+
+if [ -f "$SCRIPT" ]; then
+    exec rsh $a at -f $SCRIPT now
+else
+    exec rsh $a $SCRIPT
+fi
diff --git a/source4/cluster/ctdb/tools/onnode.ssh b/source4/cluster/ctdb/tools/onnode.ssh
new file mode 100755
index 0000000000..ec1231616e
--- /dev/null
+++ b/source4/cluster/ctdb/tools/onnode.ssh
@@ -0,0 +1,43 @@
+#!/bin/sh
+# onnode script for ssh
+
+if [ $# -lt 2 ]; then
+cat <<EOF
+Usage: onnode <nodenum|all> <command>
+EOF
+exit 1
+fi
+
+NODE="$1"
+shift
+SCRIPT="$*"
+
+NODES=/etc/ctdb/nodes
+
+NUMNODES=`egrep '^[[:alnum:]]' $NODES | wc -l`
+MAXNODE=`expr $NUMNODES - 1`
+
+if [ $NODE = "all" ]; then
+    for a in `egrep '^[[:alnum:]]' $NODES`; do
+	if [ -f "$SCRIPT" ]; then
+	    ssh $a at -f $SCRIPT now
+	else
+	    ssh $a $SCRIPT
+	fi
+    done
+    exit 0
+fi
+
+if [ $NODE -gt $MAXNODE ]; then
+    echo "Node $NODE doesn't exist"
+    exit 1
+fi
+
+NODEPLUSONE=`expr $NODE + 1`
+a=`egrep '^[[:alnum:]]' $NODES | head -$NODEPLUSONE | tail -1`
+
+if [ -f "$SCRIPT" ]; then
+    exec ssh $a at -f $SCRIPT now
+else
+    exec ssh $a $SCRIPT
+fi