Add current headers from ctdb master.

(ctdb.h, ctdb_protocol.h, ctdb_private.h, ctdb_typesafe_cb.h, ctdb_client.h)

This will allow us to check for build regressions against ctdb from autobuild.

5 files changed, 4236 insertions, 0 deletions

diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h
new file mode 100644
index 0000000000..93224cbdaa
--- /dev/null
+++ b/ctdb/include/ctdb.h
@@ -0,0 +1,1186 @@
+/*
+   ctdb database library
+
+   Copyright (C) Ronnie sahlberg 2010
+   Copyright (C) Rusty Russell 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _CTDB_H
+#define _CTDB_H
+#include <sys/types.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <tdb.h>
+#include <netinet/in.h>
+#include <ctdb_protocol.h>
+
+/**
+ * ctdb - a library for accessing tdbs controlled by ctdbd
+ *
+ * ctdbd (clustered tdb daemon) is a daemon designed to syncronize TDB
+ * databases across a cluster.  Using this library, you can communicate with
+ * the daemon to access the databases, pass messages across the cluster, and
+ * control the daemon itself.
+ *
+ * The general API is event-driven and asynchronous: you call the
+ * *_send functions, supplying callbacks, then when the ctdbd file
+ * descriptor is usable, call ctdb_service() to perform read from it
+ * and call your callbacks, which use the *_recv functions to unpack
+ * the replies from ctdbd.
+ *
+ * There is also a synchronous wrapper for each function for trivial
+ * programs; these can be found in the section marked "Synchronous API".
+ */
+
+/**
+ * ctdb_log_fn_t - logging function for ctdbd
+ * @log_priv: private (typesafe) arg via ctdb_connect
+ * @severity: syslog-style severity
+ * @format: printf-style format string.
+ * @ap: arguments for formatting.
+ *
+ * The severity passed to log() are as per syslog(3).  In particular,
+ * LOG_DEBUG is used for tracing, LOG_WARNING is used for unusual
+ * conditions which don't necessarily return an error through the API,
+ * LOG_ERR is used for errors such as lost communication with ctdbd or
+ * out-of-memory, LOG_ALERT is used for library usage bugs, LOG_CRIT is
+ * used for libctdb internal consistency checks.
+ *
+ * The log() function can be typesafe: the @log_priv arg to
+ * ctdb_donnect and signature of log() should match.
+ */
+typedef void (*ctdb_log_fn_t)(void *log_priv,
+			      int severity, const char *format, va_list ap);
+
+/**
+ * ctdb_connect - connect to ctdb using the specified domain socket.
+ * @addr: the socket address, or NULL for default
+ * @log: the logging function
+ * @log_priv: the private argument to the logging function.
+ *
+ * Returns a ctdb context if successful or NULL.  Use ctdb_disconnect() to
+ * release the returned ctdb_connection when finished.
+ *
+ * See Also:
+ *	ctdb_log_fn_t, ctdb_log_file()
+ */
+struct ctdb_connection *ctdb_connect(const char *addr,
+				     ctdb_log_fn_t log_fn, void *log_priv);
+
+/**
+ * ctdb_log_file - example logging function
+ *
+ * Logs everything at priority LOG_WARNING or above to the file given (via
+ * the log_priv argument, usually stderr).
+ */
+void ctdb_log_file(FILE *, int, const char *, va_list);
+
+/**
+ * ctdb_log_level - level at which to call logging function
+ *
+ * This variable globally controls filtering on the logging function.
+ * It is initialized to LOG_WARNING, meaning that strange but nonfatal
+ * events, as well as errors and API misuses are reported.
+ *
+ * Set it to LOG_DEBUG to receive all messages.
+ */
+extern int ctdb_log_level;
+
+/**
+ * ctdb_disconnect - close down a connection to ctdbd.
+ * @ctdb: the ctdb connectio returned from ctdb_connect.
+ *
+ * The @ctdb arg will be freed by this call, and must not be used again.
+ */
+void ctdb_disconnect(struct ctdb_connection *ctdb);
+
+/***
+ *
+ *  Asynchronous API
+ *
+ ***/
+
+/**
+ * ctdb_num_active - get the number of active commands
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ *
+ * This command can be used to find the number of active commands we have
+ * issued. An active command is a command we have queued, or sent
+ * to the ctdb daemon but which we have not yet received a reply to.
+ *
+ * See Also:
+ *	ctdb_num_in_flight(), ctdb_num_out_queue()
+ */
+int ctdb_num_active(struct ctdb_connection *ctdb);
+
+/**
+ * ctdb_num_in_flight - get the number of commands in flight.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ *
+ * This command can be used to find the number of commands we have
+ * sent to the ctdb daemon to which we have not yet received/processed
+ * the reply.
+ *
+ * See Also:
+ *	ctdb_num_out_queue(), ctdb_num_active()
+ */
+int ctdb_num_in_flight(struct ctdb_connection *ctdb);
+
+/**
+ * ctdb_num_out_queue - get the number of commands in the out queue
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ *
+ * This command can be used to find the number of commands we have
+ * queued for delivery to the ctdb daemon but have not yet been
+ * written to the domain socket.
+ *
+ * See Also:
+ *	ctdb_num_in_flight(), ctdb_num_active()
+ */
+int ctdb_num_out_queue(struct ctdb_connection *ctdb);
+
+/**
+ * ctdb_get_fd - get the filedescriptor to select/poll on
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ *
+ * By using poll or select on this file descriptor, you will know when to call
+ * ctdb_service().
+ *
+ * See Also:
+ *	ctdb_which_events(), ctdb_service()
+ */
+int ctdb_get_fd(struct ctdb_connection *ctdb);
+
+/**
+ * ctdb_which_events - determine which events ctdb_service wants to see
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ *
+ * This returns POLLIN, possibly or'd with POLLOUT if there are writes
+ * pending.  You can set this straight into poll.events.
+ *
+ * See Also:
+ *	ctdb_service()
+ */
+int ctdb_which_events(struct ctdb_connection *ctdb);
+
+/**
+ * ctdb_service - service any I/O and callbacks from ctdbd communication
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @revents: which events are available.
+ *
+ * This is the core of the library: it read and writes to the ctdbd
+ * socket.  It may call callbacks registered with the various _send
+ * functions.
+ *
+ * revents is a bitset: POLLIN and/or POLLOUT may be set to indicate
+ * it is worth attempting to read/write the (nonblocking)
+ * filedescriptor respectively.
+ *
+ * Note that the synchronous functions call this internally.
+ * Returns false on catastrophic failure.
+ */
+bool ctdb_service(struct ctdb_connection *ctdb, int revents);
+
+/**
+ * struct ctdb_request - handle for an outstanding request
+ *
+ * This opaque structure returned from various *_send functions gives
+ * you a handle by which you can cancel a request.  You can't do
+ * anything else with it until the request is completed and it is
+ * handed to your callback function.
+ */
+struct ctdb_request;
+
+/**
+ * ctdb_request_free - free a completed request
+ *
+ * This frees a request: you should only call it once it has been
+ * handed to your callback.  For incomplete requests, see ctdb_cancel().
+ */
+void ctdb_request_free(struct ctdb_request *req);
+
+/**
+ * ctdb_callback_t - callback for completed requests.
+ *
+ * This would normally unpack the request using ctdb_*_recv().  You
+ * must free the request using ctdb_request_free().
+ *
+ * Note that due to macro magic, actual your callback can be typesafe:
+ * instead of taking a void *, it can take a type which matches the
+ * actual private parameter.
+ */
+typedef void (*ctdb_callback_t)(struct ctdb_connection *ctdb,
+				struct ctdb_request *req, void *private_data);
+
+/**
+ * struct ctdb_db - connection to a particular open TDB
+ *
+ * This represents a particular open database: you receive it from
+ * ctdb_attachdb or ctdb_attachdb_recv to manipulate a database.
+ *
+ * You have to free the handle with ctdb_detachdb() when finished with it.
+ */
+struct ctdb_db;
+
+/**
+ * ctdb_attachdb_send - open a clustered TDB
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @name: the filename of the database (no /).
+ * @persistent: whether the database is persistent across ctdbd's life
+ * @tdb_flags: the flags to pass to tdb_open.
+ * @callback: the callback when we're attached or failed (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * This function connects to a TDB controlled by ctdbd.  It can create
+ * a new TDB if it does not exist, depending on tdb_flags.  Returns
+ * the pending request, or NULL on error.
+ */
+struct ctdb_request *
+ctdb_attachdb_send(struct ctdb_connection *ctdb,
+		   const char *name, bool persistent, uint32_t tdb_flags,
+		   ctdb_callback_t callback, void *cbdata);
+
+/**
+ * ctdb_attachdb_recv - read an ctdb_attach reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ *
+ * This returns NULL if something went wrong, or otherwise the open database.
+ */
+struct ctdb_db *ctdb_attachdb_recv(struct ctdb_connection *ctdb,
+				   struct ctdb_request *req);
+
+
+/**
+ * struct ctdb_lock - a record lock on a clustered TDB database
+ *
+ * This locks a subset of the database across the entire cluster; it
+ * is the fundamental sychronization element for ctdb.  You cannot have
+ * more than one lock at once.
+ *
+ * You MUST NOT block during holding this lock and MUST release it
+ * quickly by performing ctdb_release_lock(lock).
+ * Do NOT make any system calls that may block while holding the lock.
+ *
+ * Try to release the lock as quickly as possible.
+ */
+struct ctdb_lock;
+
+/**
+ * ctdb_rrl_callback_t - callback for ctdb_readrecordlock_async
+ *
+ * This is not the standard ctdb_callback_t, because there is often no
+ * request required to access a database record (ie. if it is local already).
+ * So the callback is handed the lock directly: it might be NULL if there
+ * was an error obtaining the lock.
+ *
+ * See Also:
+ *	ctdb_readrecordlock_async(), ctdb_readrecordlock()
+ */
+typedef void (*ctdb_rrl_callback_t)(struct ctdb_db *ctdb_db,
+				    struct ctdb_lock *lock,
+				    TDB_DATA data,
+				    void *private_data);
+
+/**
+ * ctdb_readrecordlock_async - read and lock a record
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @key: the key of the record to lock.
+ * @callback: the callback once the record is locked (typesafe).
+ * @cbdata: the argument to callback()
+ *
+ * This returns true on success.  Commonly, we can obtain the record
+ * immediately and so the callback will be invoked.  Otherwise a request
+ * will be queued to ctdbd for the record.
+ *
+ * If failure is immediate, false is returned.  Otherwise, the callback
+ * may receive a NULL lock arg to indicate asynchronous failure.
+ */
+bool ctdb_readrecordlock_async(struct ctdb_db *ctdb_db, TDB_DATA key,
+			       ctdb_rrl_callback_t callback, void *cbdata);
+
+/**
+ * ctdb_readonlyrecordlock_async - read and lock a record for read-only access
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @key: the key of the record to lock.
+ * @callback: the callback once the record is locked (typesafe).
+ * @cbdata: the argument to callback()
+ *
+ * This returns true on success.  Commonly, we can obtain the record
+ * immediately and so the callback will be invoked.  Otherwise a request
+ * will be queued to ctdbd for the record.
+ *
+ * If failure is immediate, false is returned.  Otherwise, the callback
+ * may receive a NULL lock arg to indicate asynchronous failure.
+ */
+bool ctdb_readonlyrecordlock_async(struct ctdb_db *ctdb_db, TDB_DATA key,
+			       ctdb_rrl_callback_t callback, void *cbdata);
+
+
+/**
+ * ctdb_writerecord - write a locked record in a TDB
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @lock: the lock from ctdb_readrecordlock/ctdb_readrecordlock_recv
+ * @data: the new data to place in the record.
+ */
+bool ctdb_writerecord(struct ctdb_db *ctdb_db,
+		      struct ctdb_lock *lock, TDB_DATA data);
+
+/**
+ * ctdb_release_lock - release a record lock on a TDB
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @lock: the lock from ctdb_readrecordlock/ctdb_readrecordlock_async
+ */
+void ctdb_release_lock(struct ctdb_db *ctdb_db, struct ctdb_lock *lock);
+
+
+
+/**
+ * ctdb_traverse_callback_t - callback for ctdb_traverse_async.
+ * return 0 - to continue traverse
+ * return 1 - to abort the traverse
+ *
+ * See Also:
+ *	ctdb_traverse_async()
+ */
+#define TRAVERSE_STATUS_RECORD		0
+#define TRAVERSE_STATUS_FINISHED	1
+#define TRAVERSE_STATUS_ERROR		2
+typedef int (*ctdb_traverse_callback_t)(struct ctdb_connection *ctdb,
+				    struct ctdb_db *ctdb_db,
+				    int status,
+				    TDB_DATA key,
+				    TDB_DATA data,
+				    void *private_data);
+
+/**
+ * ctdb_traverse_async - traverse a database.
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @callback: the callback once the record is locked (typesafe).
+ * @cbdata: the argument to callback()
+ *
+ * This returns true on success.
+ * when successfull, the callback will be invoked for each record
+ * until the traversal is finished.
+ *
+ * status == 
+ * TRAVERSE_STATUS_RECORD         key/data contains a record.
+ * TRAVERSE_STATUS_FINISHED       traverse is finished. key/data is undefined.
+ * TRAVERSE_STATUS_ERROR          an error occured during traverse.
+ *                                key/data is undefined.
+ *
+ * If failure is immediate, false is returned.
+ */
+bool ctdb_traverse_async(struct ctdb_db *ctdb_db,
+			 ctdb_traverse_callback_t callback, void *cbdata);
+
+/**
+ * ctdb_message_fn_t - messaging callback for ctdb messages
+ *
+ * ctdbd provides a simple messaging API; you can register for a particular
+ * 64-bit id on which you want to send messages, and send to other ids.
+ *
+ * See Also:
+ *	ctdb_set_message_handler_send()
+ */
+typedef void (*ctdb_message_fn_t)(struct ctdb_connection *,
+				  uint64_t srvid, TDB_DATA data, void *);
+
+/**
+ * ctdb_set_message_handler_send - register for messages to a srvid
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @srvid: the 64 bit identifier for our messages.
+ * @handler: the callback when we receive such a message (typesafe)
+ * @handler_data: the argument to handler()
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * Note: our callback will always be called before handler.
+ *
+ * See Also:
+ *	ctdb_set_message_handler_recv(), ctdb_remove_message_handler_send()
+ */
+struct ctdb_request *
+ctdb_set_message_handler_send(struct ctdb_connection *ctdb, uint64_t srvid,
+			      ctdb_message_fn_t handler,
+			      void *handler_data,
+			      ctdb_callback_t callback,
+			      void *cbdata);
+
+/**
+ * ctdb_set_message_handler_recv - read a set_message_handler result
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request
+ *
+ * If this returns true, the registered handler may be called from the next
+ * ctdb_service().  If this returns false, the registration failed.
+ */
+bool ctdb_set_message_handler_recv(struct ctdb_connection *ctdb,
+				   struct ctdb_request *handle);
+
+/**
+ * ctdb_remove_message_handler_send - unregister for messages to a srvid
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @srvid: the 64 bit identifier for our messages.
+ * @handler: the callback when we receive such a message (typesafe)
+ * @handler_data: the argument to handler()
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * This undoes a successful ctdb_set_message_handler or
+ * ctdb_set_message_handler_recv.
+ */
+struct ctdb_request *
+ctdb_remove_message_handler_send(struct ctdb_connection *ctdb, uint64_t srvid,
+				 ctdb_message_fn_t handler, void *handler_data,
+				 ctdb_callback_t callback, void *cbdata);
+
+/**
+ * ctdb_remove_message_handler_recv - read a remove_message_handler result
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request
+ *
+ * After this returns true, the registered handler will no longer be called.
+ * If this returns false, the de-registration failed.
+ */
+bool ctdb_remove_message_handler_recv(struct ctdb_connection *ctdb,
+				      struct ctdb_request *req);
+
+
+/**
+ * ctdb_send_message - send a message via ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @pnn: the physical node number to send to
+ * @srvid: the 64 bit identifier for this message type.
+ * @data: the data to send
+ *
+ * This allows arbitrary messages to be sent across the cluster to those
+ * listening (via ctdb_set_message_handler et al).
+ *
+ * This queues a message to be sent: you will need to call
+ * ctdb_service() to actually send the message.  There is no callback
+ * because there is no acknowledgement.
+ *
+ * See Also:
+ *	ctdb_getpnn_send(), ctdb_getpnn()
+ */
+bool ctdb_send_message(struct ctdb_connection *ctdb, uint32_t pnn, uint64_t srvid, TDB_DATA data);
+
+/**
+ * ctdb_getpnn_send - read the pnn number of a node.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getpnn_send(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 ctdb_callback_t callback,
+		 void *cbdata);
+/**
+ * ctdb_getpnn_recv - read an ctdb_getpnn reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @pnn: a pointer to the pnn to fill in
+ *
+ * This returns false if something went wrong, or otherwise fills in pnn.
+ */
+bool ctdb_getpnn_recv(struct ctdb_connection *ctdb,
+		      struct ctdb_request *req, uint32_t *pnn);
+
+
+/**
+ * ctdb_getdbstat_send - read statistics for a db
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @db_id:    the database to collect the statistics from
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getdbstat_send(struct ctdb_connection *ctdb,
+		     uint32_t destnode,
+		     uint32_t db_id,
+		     ctdb_callback_t callback,
+		     void *cbdata);
+/**
+ * ctdb_getdbstat_recv - read an ctdb_getdbstat reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @stat: a pointer to the *stat to fill in
+ *
+ * This returns false if something went wrong, or otherwise fills in **stats
+ * stats must be freed later by calling ctdb_free_dbstat();
+ */
+bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
+			 struct ctdb_request *req,
+			 struct ctdb_db_statistics **stat);
+
+void ctdb_free_dbstat(struct ctdb_db_statistics *stat);
+
+/**
+ * ctdb_check_message_handlers_send - check a list of message_handlers
+ * if they are registered
+ * message_handlers are registered on the daemon using the
+ *   ctdb_set_message_handler_send() call
+ *
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @num: number of srvids to check
+ * @mhs: @num message_handlers values to check
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_check_message_handlers_send(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 uint32_t num,
+		 uint64_t *mhs,
+		 ctdb_callback_t callback,
+		 void *cbdata);
+/**
+ * ctdb_check_message_handlers_recv - read a ctdb_check_message_handlers
+ * reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @num: number of message_handlers to check
+ * @result: an array of @num uint8_t fields containing the result of the check
+ *     0: message_handler does not exist
+ *     1: message_handler exists
+ *
+ * This returns false if something went wrong, or otherwise fills in result.
+ */
+bool
+ctdb_check_message_handlers_recv(struct ctdb_connection *ctdb,
+				  struct ctdb_request *req, uint32_t num,
+				  uint8_t *result);
+
+
+/**
+ * ctdb_getdbseqnum_send - read the sequence number off a db
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @dbid: database id
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getdbseqnum_send(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 uint32_t dbid,
+		 ctdb_callback_t callback,
+		 void *cbdata);
+/**
+ * ctdb_getdbseqnum_recv - read the sequence number off a database
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @seqnum: a pointer to the seqnum to fill in
+ *
+ * This returns false if something went wrong, or otherwise fills in pnn.
+ */
+bool ctdb_getdbseqnum_recv(struct ctdb_connection *ctdb,
+		      struct ctdb_request *req, uint64_t *seqnum);
+
+/**
+ * ctdb_getnodemap_send - read the nodemap number from a node.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getnodemap_send(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 ctdb_callback_t callback,
+		 void *cbdata);
+/**
+ * ctdb_getnodemap_recv - read an ctdb_getnodemap reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @nodemap: a pointer to the returned nodemap structure
+ *
+ * This returns false if something went wrong.
+ * If the command failed, it guarantees to set nodemap to NULL.
+ * A non-NULL value for nodemap means the command was successful.
+ *
+ * A non-NULL value of the nodemap must be release released/freed
+ * by ctdb_free_nodemap().
+ */
+bool ctdb_getnodemap_recv(struct ctdb_connection *ctdb,
+		      struct ctdb_request *req, struct ctdb_node_map **nodemap);
+
+/**
+ * ctdb_getifaces_send - read the list of interfaces from a node.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getifaces_send(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 ctdb_callback_t callback,
+		 void *cbdata);
+/**
+ * ctdb_getifaces_recv - read an ctdb_getifaces reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @ifaces: the list of interfaces 
+ *
+ * This returns false if something went wrong.
+ * If the command failed, it guarantees to set ifaces to NULL.
+ * A non-NULL value for ifaces means the command was successful.
+ *
+ * A non-NULL value of the ifaces must be release released/freed
+ * by ctdb_free_ifaces().
+ */
+bool ctdb_getifaces_recv(struct ctdb_connection *ctdb,
+		      struct ctdb_request *req, struct ctdb_ifaces_list **ifaces);
+
+/* Free a datastructure returned by ctdb_getifaces[_recv] */
+void ctdb_free_ifaces(struct ctdb_ifaces_list *ifaces);
+
+/**
+ * ctdb_getpublicips_send - read the public ip list from a node.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * This control returns the list of public ips known to the local node.
+ * Deamons only know about those ips that are listed in the local
+ * public addresses file, which means the returned list of ips may
+ * be only a subset of all ips across the entire cluster.
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getpublicips_send(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 ctdb_callback_t callback,
+		 void *cbdata);
+/**
+ * ctdb_getpublicips_recv - read the public ip list from a node
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @ips: a pointer to the returned public ip list
+ *
+ * This returns false if something went wrong.
+ * If the command failed, it guarantees to set ips to NULL.
+ * A non-NULL value for nodemap means the command was successful.
+ *
+ * A non-NULL value of the nodemap must be release released/freed
+ * by ctdb_free_publicips().
+ */
+bool ctdb_getpublicips_recv(struct ctdb_connection *ctdb,
+		      struct ctdb_request *req, struct ctdb_all_public_ips **ips);
+
+
+/**
+ * ctdb_getrecmaster_send - read the recovery master of a node
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getrecmaster_send(struct ctdb_connection *ctdb,
+			uint32_t destnode,
+			ctdb_callback_t callback, void *cbdata);
+
+/**
+ * ctdb_getrecmaster_recv - read an ctdb_getrecmaster reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @recmaster: a pointer to the recmaster to fill in
+ *
+ * This returns false if something went wrong, or otherwise fills in
+ * recmaster.
+ */
+bool ctdb_getrecmaster_recv(struct ctdb_connection *ctdb,
+			    struct ctdb_request *handle,
+			    uint32_t *recmaster);
+
+/**
+ * ctdb_getrecmode_send - read the recovery mode of a node
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getrecmode_send(struct ctdb_connection *ctdb,
+		     uint32_t destnode,
+		     ctdb_callback_t callback, void *cbdata);
+
+/**
+ * ctdb_getrecmode_recv - read an ctdb_getrecmode reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @recmode: a pointer to the recmode to fill in
+ *
+ * This returns false if something went wrong, or otherwise fills in
+ * recmode.
+ */
+bool ctdb_getrecmode_recv(struct ctdb_connection *ctdb,
+			  struct ctdb_request *handle,
+			  uint32_t *recmode);
+
+/**
+ * ctdb_getvnnmap_send - read the vnn map from a node.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getvnnmap_send(struct ctdb_connection *ctdb,
+		    uint32_t destnode,
+		    ctdb_callback_t callback,
+		    void *cbdata);
+/**
+ * ctdb_getvnnmap_recv - read an ctdb_getvnnmap reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @vnnmap: the list of interfaces 
+ *
+ * This returns false if something went wrong.
+ * If the command failed, it guarantees to set vnnmap to NULL.
+ * A non-NULL value for vnnmap means the command was successful.
+ *
+ * A non-NULL value of the vnnmap must be released/freed
+ * by ctdb_free_vnnmap().
+ */
+bool ctdb_getvnnmap_recv(struct ctdb_connection *ctdb,
+			 struct ctdb_request *req, struct ctdb_vnn_map **vnnmap);
+
+/**
+ * ctdb_cancel - cancel an uncompleted request
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the uncompleted request.
+ *
+ * This cancels a request, returning true.  You may not cancel a
+ * request which has already been completed (ie. once its callback has
+ * been called); you should simply use ctdb_request_free() in that case.
+ */
+void ctdb_cancel(struct ctdb_connection *ctdb, struct ctdb_request *req);
+
+/***
+ *
+ *  Synchronous API
+ *
+ ***/
+
+/**
+ * ctdb_attachdb - open a clustered TDB (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @name: the filename of the database (no /).
+ * @persistent: whether the database is persistent across ctdbd's life
+ * @tdb_flags: the flags to pass to tdb_open.
+ *
+ * Do a ctdb_attachdb_send and wait for it to complete.
+ * Returns NULL on failure.
+ */
+struct ctdb_db *ctdb_attachdb(struct ctdb_connection *ctdb,
+			      const char *name, bool persistent,
+			      uint32_t tdb_flags);
+
+/**
+ * ctdb_detachdb - close a clustered TDB.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @db: the database from ctdb_attachdb/ctdb_attachdb_send
+ *
+ * Closes a clustered tdb.
+ */
+void ctdb_detachdb(struct ctdb_connection *ctdb, struct ctdb_db *db);
+
+/**
+ * ctdb_readrecordlock - read and lock a record (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @ctdb_db: the database handle from ctdb_attachdb/ctdb_attachdb_recv.
+ * @key: the key of the record to lock.
+ * @req: a pointer to the request, if one is needed.
+ *
+ * Do a ctdb_readrecordlock_send and wait for it to complete.
+ * Returns NULL on failure.
+ */
+struct ctdb_lock *ctdb_readrecordlock(struct ctdb_connection *ctdb,
+				      struct ctdb_db *ctdb_db, TDB_DATA key,
+				      TDB_DATA *data);
+
+
+/**
+ * ctdb_set_message_handler - register for messages to a srvid (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @srvid: the 64 bit identifier for our messages.
+ * @handler: the callback when we receive such a message (typesafe)
+ * @cbdata: the argument to handler()
+ *
+ * If this returns true, the message handler can be called from any
+ * ctdb_service() (which is also called indirectly by other
+ * synchronous functions).  If this returns false, the registration
+ * failed.
+ */
+bool ctdb_set_message_handler(struct ctdb_connection *ctdb, uint64_t srvid,
+			      ctdb_message_fn_t handler, void *cbdata);
+
+
+/**
+ * ctdb_remove_message_handler - deregister for messages (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @srvid: the 64 bit identifier for our messages.
+ * @handler: the callback when we receive such a message (typesafe)
+ * @handler_data: the argument to handler()
+ *
+ * If this returns true, the message handler will no longer be called.
+ * If this returns false, the deregistration failed.
+ */
+bool ctdb_remove_message_handler(struct ctdb_connection *ctdb, uint64_t srvid,
+				 ctdb_message_fn_t handler, void *handler_data);
+
+/**
+ * ctdb_getpnn - read the pnn number of a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @pnn: a pointer to the pnn to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *pnn on success.
+ */
+bool ctdb_getpnn(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 uint32_t *pnn);
+
+/**
+ * ctdb_getdbstat - read the db stat of a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @db_id:    the database to collect the statistics from
+ * @stat: a pointer to the *stat to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * This returns false if something went wrong, or otherwise fills in **stat
+ * stat must be freed later by calling ctdb_free_dbstat();
+ */
+bool ctdb_getdbstat(struct ctdb_connection *ctdb,
+		    uint32_t destnode,
+		    uint32_t db_id,
+		    struct ctdb_db_statistics **stat);
+
+
+/**
+ * ctdb_check_message_handlers - check a list of message_handlers (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @num: number of srvids to check
+ * @mhs: @num message_handlers to check
+ * @result: an array of @num uint8_t fields containing the result of the check
+ *     0: message_handler does not exist
+ *     1: message_handler exists
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+bool
+ctdb_check_message_handlers(struct ctdb_connection *ctdb,
+			   uint32_t destnode,
+			   uint32_t num,
+			   uint64_t *mhs,
+			   uint8_t *result);
+
+/**
+ * ctdb_getdbseqnum - read the seqnum of a database
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @dbid: database id
+ * @seqnum: sequence number for the database
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *pnn on success.
+ */
+bool
+ctdb_getdbseqnum(struct ctdb_connection *ctdb,
+		 uint32_t destnode,
+		 uint32_t dbid,
+		 uint64_t *seqnum);
+
+/**
+ * ctdb_getrecmaster - read the recovery master of a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @recmaster: a pointer to the recmaster to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *recmaster on success.
+ */
+bool ctdb_getrecmaster(struct ctdb_connection *ctdb,
+		       uint32_t destnode,
+		       uint32_t *recmaster);
+
+
+/**
+ * ctdb_getrecmode - read the recovery mode of a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @recmode: a pointer to the recmode to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *recmode on success.
+ */
+bool ctdb_getrecmode(struct ctdb_connection *ctdb,
+		     uint32_t destnode,
+		     uint32_t *recmode);
+
+
+/**
+ * ctdb_getnodemap - read the nodemap from a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @nodemap: a pointer to the nodemap to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *nodemap on success.
+ * A non-NULL nodemap must be freed by calling ctdb_free_nodemap.
+ */
+bool ctdb_getnodemap(struct ctdb_connection *ctdb,
+		     uint32_t destnode, struct ctdb_node_map **nodemap);
+
+/**
+ * ctdb_getifaces - read the list of interfaces from a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @ifaces: a pointer to the ifaces to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *ifaces on success.
+ * A non-NULL value of the ifaces must be release released/freed
+ * by ctdb_free_ifaces().
+ */
+bool ctdb_getifaces(struct ctdb_connection *ctdb,
+		    uint32_t destnode, struct ctdb_ifaces_list **ifaces);
+
+/*
+ * This function is used to release/free the nodemap structure returned
+ * by ctdb_getnodemap() and ctdb_getnodemap_recv()
+ */
+void ctdb_free_nodemap(struct ctdb_node_map *nodemap);
+
+
+/**
+ * ctdb_getpublicips - read the public ip list from a node.
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @ips: a pointer to the returned public ip list
+ *
+ * This control returns the list of public ips known to the local node.
+ * Deamons only know about those ips that are listed in the local
+ * public addresses file, which means the returned list of ips may
+ * be only a subset of all ips across the entire cluster.
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * This returns false if something went wrong.
+ * If the command failed, it guarantees to set ips to NULL.
+ * A non-NULL value for nodemap means the command was successful.
+ *
+ * A non-NULL value of the nodemap must be release released/freed
+ * by ctdb_free_publicips().
+ */
+bool ctdb_getpublicips(struct ctdb_connection *ctdb,
+		     uint32_t destnode, struct ctdb_all_public_ips **ips);
+
+/*
+ * This function is used to release/free the public ip structure returned
+ * by ctdb_getpublicips() and ctdb_getpublicips_recv()
+ */
+void ctdb_free_publicips(struct ctdb_all_public_ips *ips);
+
+
+/**
+ * ctdb_getvnnmap - read the vnn map from a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @vnnmap: a pointer to the vnnmap to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * Returns true and fills in *vnnmap on success.
+ * A non-NULL value of the vnnmap must be  released/freed
+ * by ctdb_free_vnnmap().
+ */
+bool ctdb_getvnnmap(struct ctdb_connection *ctdb,
+		    uint32_t destnode, struct ctdb_vnn_map **vnnmap);
+
+/*
+ * This function is used to release/free the vnnmap structure returned
+ * by ctdb_getvnnmap() and ctdb_getvnnmap_recv()
+ */
+void ctdb_free_vnnmap(struct ctdb_vnn_map *vnnmap);
+
+/* These ugly macro wrappers make the callbacks typesafe. */
+#include <ctdb_typesafe_cb.h>
+#define ctdb_sendcb(cb, cbdata)						\
+	 typesafe_cb_preargs(void, (cb), (cbdata),			\
+			     struct ctdb_connection *, struct ctdb_request *)
+
+#define ctdb_msgcb(cb, cbdata)						\
+	typesafe_cb_preargs(void, (cb), (cbdata),			\
+			    struct ctdb_connection *, uint64_t, TDB_DATA)
+
+#define ctdb_connect(addr, log, logpriv)				\
+	ctdb_connect((addr),						\
+		     typesafe_cb_postargs(void, (log), (logpriv),	\
+					  int, const char *, va_list),	\
+		     (logpriv))
+
+#define ctdb_set_message_handler(ctdb, srvid, handler, hdata)		\
+	ctdb_set_message_handler((ctdb), (srvid),			\
+				 ctdb_msgcb((handler), (hdata)), (hdata))
+
+#define ctdb_remove_message_handler(ctdb, srvid, handler, hdata)	\
+	ctdb_remove_message_handler((ctdb), (srvid),			\
+				    ctdb_msgcb((handler), (hdata)), (hdata))
+
+#define ctdb_attachdb_send(ctdb, name, persistent, tdb_flags, cb, cbdata) \
+	ctdb_attachdb_send((ctdb), (name), (persistent), (tdb_flags),	\
+			   ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_readrecordlock_async(_ctdb_db, key, cb, cbdata)		\
+	ctdb_readrecordlock_async((_ctdb_db), (key),			\
+		typesafe_cb_preargs(void, (cb), (cbdata),		\
+				    struct ctdb_db *, struct ctdb_lock *, \
+				    TDB_DATA), (cbdata))
+
+#define ctdb_set_message_handler_send(ctdb, srvid, handler, hdata, cb, cbdata) \
+	ctdb_set_message_handler_send((ctdb), (srvid),			\
+				      ctdb_msgcb((handler), (hdata)), (hdata), \
+				      ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_remove_message_handler_send(ctdb, srvid, handler, hdata, cb, cbdata) \
+	ctdb_remove_message_handler_send((ctdb), (srvid),		\
+	      ctdb_msgcb((handler), (hdata)), (hdata),			\
+	      ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getpnn_send(ctdb, destnode, cb, cbdata)			\
+	ctdb_getpnn_send((ctdb), (destnode),				\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getdbstat_send(ctdb, destnode, db_id, cb, cbdata)		\
+	ctdb_getdbstat_send((ctdb), (destnode), (db_id),		\
+			    ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_check_message_handlers_send(ctdb, destnode, num, mhs,	\
+			 cb, cbdata)					\
+	ctdb_check_message_handlers_send((ctdb), (destnode), (num), 	\
+			 (mhs),						\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getrecmaster_send(ctdb, destnode, cb, cbdata)		\
+	ctdb_getrecmaster_send((ctdb), (destnode),			\
+			       ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getrecmode_send(ctdb, destnode, cb, cbdata)		\
+	ctdb_getrecmode_send((ctdb), (destnode),			\
+			       ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getnodemap_send(ctdb, destnode, cb, cbdata)		\
+	ctdb_getnodemap_send((ctdb), (destnode),			\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getpublicips_send(ctdb, destnode, cb, cbdata)		\
+	ctdb_getpublicips_send((ctdb), (destnode),			\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getdbseqnum_send(ctdb, destnode, dbid, cb, cbdata)		\
+	ctdb_getdbseqnum_send((ctdb), (destnode), (dbid),		\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getifaces_send(ctdb, destnode, cb, cbdata)			\
+	ctdb_getifaces_send((ctdb), (destnode),				\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#define ctdb_getvnnmap_send(ctdb, destnode, cb, cbdata)			\
+	ctdb_getvnnmap_send((ctdb), (destnode),				\
+			 ctdb_sendcb((cb), (cbdata)), (cbdata))
+
+#endif
diff --git a/ctdb/include/ctdb_client.h b/ctdb/include/ctdb_client.h
new file mode 100644
index 0000000000..c14a395b9e
--- /dev/null
+++ b/ctdb/include/ctdb_client.h
@@ -0,0 +1,630 @@
+/*
+   ctdb database library: old client interface
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _CTDB_CLIENT_H
+#define _CTDB_CLIENT_H
+#include <ctdb_protocol.h>
+
+enum control_state {CTDB_CONTROL_WAIT, CTDB_CONTROL_DONE, CTDB_CONTROL_ERROR, CTDB_CONTROL_TIMEOUT};
+
+struct ctdb_client_control_state {
+	struct ctdb_context *ctdb;
+	uint32_t reqid;
+	int32_t status;
+	TDB_DATA outdata;
+	enum control_state state;
+	char *errormsg;
+	struct ctdb_req_control *c;
+
+	/* if we have a callback registered for the completion (or failure) of
+	   this control
+	   if a callback is used, it MUST talloc_free the cb_data passed to it
+	*/
+	struct {
+		void (*fn)(struct ctdb_client_control_state *);
+		void *private_data;
+	} async;
+};
+
+struct ctdb_client_notify_register {
+	uint64_t srvid;
+	uint32_t len;
+	uint8_t notify_data[1];
+};
+
+struct ctdb_client_notify_deregister {
+	uint64_t srvid;
+};
+
+struct tevent_context;
+
+/*
+  initialise ctdb subsystem
+*/
+struct ctdb_context *ctdb_init(struct tevent_context *ev);
+
+/*
+  choose the transport
+*/
+int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport);
+
+/*
+  set the directory for the local databases
+*/
+int ctdb_set_tdb_dir(struct ctdb_context *ctdb, const char *dir);
+int ctdb_set_tdb_dir_persistent(struct ctdb_context *ctdb, const char *dir);
+int ctdb_set_tdb_dir_state(struct ctdb_context *ctdb, const char *dir);
+
+/*
+  set some flags
+*/
+void ctdb_set_flags(struct ctdb_context *ctdb, unsigned flags);
+
+/*
+  tell ctdb what address to listen on, in transport specific format
+*/
+int ctdb_set_address(struct ctdb_context *ctdb, const char *address);
+
+int ctdb_set_socketname(struct ctdb_context *ctdb, const char *socketname);
+const char *ctdb_get_socketname(struct ctdb_context *ctdb);
+
+/*
+  tell ctdb what nodes are available. This takes a filename, which will contain
+  1 node address per line, in a transport specific format
+*/
+int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist);
+
+/*
+  Check that a specific ip address exists in the node list and returns
+  the id for the node or -1
+*/
+int ctdb_ip_to_nodeid(struct ctdb_context *ctdb, const char *nodeip);
+
+/*
+  start the ctdb protocol
+*/
+int ctdb_start(struct ctdb_context *ctdb);
+int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog, const char *public_address_list);
+
+/*
+  attach to a ctdb database
+*/
+struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb,
+				    struct timeval timeout,
+				    const char *name,
+				    bool persistent,
+				    uint32_t tdb_flags);
+
+
+/*
+  find an attached ctdb_db handle given a name
+ */
+struct ctdb_db_context *ctdb_db_handle(struct ctdb_context *ctdb, const char *name);
+
+/*
+  error string for last ctdb error
+*/
+const char *ctdb_errstr(struct ctdb_context *);
+
+/* a ctdb call function */
+typedef int (*ctdb_fn_t)(struct ctdb_call_info *);
+
+/*
+  setup a ctdb call function
+*/
+int ctdb_set_call(struct ctdb_db_context *ctdb_db, ctdb_fn_t fn, uint32_t id);
+
+
+
+/*
+  make a ctdb call. The associated ctdb call function will be called on the DMASTER
+  for the given record
+*/
+int ctdb_call(struct ctdb_db_context *ctdb_db, struct ctdb_call *call);
+
+/*
+  initiate an ordered ctdb cluster shutdown
+  this function will never return
+*/
+void ctdb_shutdown(struct ctdb_context *ctdb);
+
+/* return pnn of this node */
+uint32_t ctdb_get_pnn(struct ctdb_context *ctdb);
+
+/*
+  return the number of nodes
+*/
+uint32_t ctdb_get_num_nodes(struct ctdb_context *ctdb);
+
+/* setup a handler for ctdb messages */
+typedef void (*ctdb_msg_fn_t)(struct ctdb_context *, uint64_t srvid,
+				  TDB_DATA data, void *);
+int ctdb_client_set_message_handler(struct ctdb_context *ctdb, uint64_t srvid,
+			     ctdb_msg_fn_t handler,
+			     void *private_data);
+int ctdb_client_remove_message_handler(struct ctdb_context *ctdb,
+				       uint64_t srvid, void *private_data);
+
+
+int ctdb_call(struct ctdb_db_context *ctdb_db, struct ctdb_call *call);
+struct ctdb_client_call_state *ctdb_call_send(struct ctdb_db_context *ctdb_db, struct ctdb_call *call);
+int ctdb_call_recv(struct ctdb_client_call_state *state, struct ctdb_call *call);
+
+/* send a ctdb message */
+int ctdb_client_send_message(struct ctdb_context *ctdb, uint32_t pnn,
+		      uint64_t srvid, TDB_DATA data);
+
+
+/*
+   Fetch a ctdb record from a remote node
+ . Underneath this will force the
+   dmaster for the record to be moved to the local node.
+*/
+struct ctdb_record_handle *ctdb_fetch_lock(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx,
+					   TDB_DATA key, TDB_DATA *data);
+
+struct ctdb_record_handle *ctdb_fetch_readonly_lock(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx, TDB_DATA key, TDB_DATA *data, int read_only);
+
+int ctdb_record_store(struct ctdb_record_handle *h, TDB_DATA data);
+
+int ctdb_fetch(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx,
+	       TDB_DATA key, TDB_DATA *data);
+
+int ctdb_register_message_handler(struct ctdb_context *ctdb,
+				  TALLOC_CTX *mem_ctx,
+				  uint64_t srvid,
+				  ctdb_msg_fn_t handler,
+				  void *private_data);
+
+struct ctdb_db_context *find_ctdb_db(struct ctdb_context *ctdb, uint32_t id);
+
+
+struct ctdb_context *ctdb_cmdline_client(struct tevent_context *ev,
+					 struct timeval req_timeout);
+
+struct ctdb_statistics;
+int ctdb_ctrl_statistics(struct ctdb_context *ctdb, uint32_t destnode, struct ctdb_statistics *status);
+
+int ctdb_ctrl_shutdown(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+struct ctdb_vnn_map;
+int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb,
+		struct timeval timeout, uint32_t destnode,
+		TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap);
+int ctdb_ctrl_setvnnmap(struct ctdb_context *ctdb,
+		struct timeval timeout, uint32_t destnode,
+		TALLOC_CTX *mem_ctx, struct ctdb_vnn_map *vnnmap);
+
+/* table that contains a list of all dbids on a node
+ */
+struct ctdb_dbid_map {
+	uint32_t num;
+	struct ctdb_dbid {
+		uint32_t dbid;
+#define CTDB_DB_FLAGS_PERSISTENT	0x01
+#define CTDB_DB_FLAGS_READONLY		0x02
+#define CTDB_DB_FLAGS_STICKY		0x04
+		uint8_t flags;
+	} dbs[1];
+};
+int ctdb_ctrl_getdbmap(struct ctdb_context *ctdb,
+	struct timeval timeout, uint32_t destnode,
+	TALLOC_CTX *mem_ctx, struct ctdb_dbid_map **dbmap);
+
+
+struct ctdb_node_map;
+
+int ctdb_ctrl_getnodemap(struct ctdb_context *ctdb,
+		    struct timeval timeout, uint32_t destnode,
+		    TALLOC_CTX *mem_ctx, struct ctdb_node_map **nodemap);
+
+int ctdb_ctrl_getnodemapv4(struct ctdb_context *ctdb,
+		    struct timeval timeout, uint32_t destnode,
+		    TALLOC_CTX *mem_ctx, struct ctdb_node_map **nodemap);
+
+int ctdb_ctrl_reload_nodes_file(struct ctdb_context *ctdb,
+		    struct timeval timeout, uint32_t destnode);
+
+struct ctdb_key_list {
+	uint32_t dbid;
+	uint32_t num;
+	TDB_DATA *keys;
+	struct ctdb_ltdb_header *headers;
+	TDB_DATA *data;
+};
+
+int ctdb_ctrl_pulldb(
+       struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid,
+       uint32_t lmaster, TALLOC_CTX *mem_ctx,
+       struct timeval timeout, TDB_DATA *outdata);
+
+struct ctdb_client_control_state *ctdb_ctrl_pulldb_send(
+       struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid,
+       uint32_t lmaster, TALLOC_CTX *mem_ctx, struct timeval timeout);
+
+int ctdb_ctrl_pulldb_recv(
+       struct ctdb_context *ctdb,
+       TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state,
+       TDB_DATA *outdata);
+
+int ctdb_ctrl_pushdb(
+       struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid,
+       TALLOC_CTX *mem_ctx,
+       struct timeval timeout, TDB_DATA indata);
+
+struct ctdb_client_control_state *ctdb_ctrl_pushdb_send(
+       struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid,
+       TALLOC_CTX *mem_ctx, struct timeval timeout,
+       TDB_DATA indata);
+
+int ctdb_ctrl_pushdb_recv(
+       struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
+       struct ctdb_client_control_state *state);
+
+
+int ctdb_ctrl_copydb(struct ctdb_context *ctdb,
+	struct timeval timeout, uint32_t sourcenode,
+	uint32_t destnode, uint32_t dbid, uint32_t lmaster,
+	TALLOC_CTX *mem_ctx);
+
+int ctdb_ctrl_getdbpath(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t dbid, TALLOC_CTX *mem_ctx, const char **path);
+int ctdb_ctrl_getdbname(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t dbid, TALLOC_CTX *mem_ctx, const char **name);
+int ctdb_ctrl_getdbhealth(struct ctdb_context *ctdb,
+			  struct timeval timeout,
+			  uint32_t destnode,
+			  uint32_t dbid, TALLOC_CTX *mem_ctx,
+			  const char **reason);
+int ctdb_ctrl_createdb(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, const char *name, bool persistent);
+
+int ctdb_ctrl_process_exists(struct ctdb_context *ctdb, uint32_t destnode, pid_t pid);
+
+int ctdb_ctrl_ping(struct ctdb_context *ctdb, uint32_t destnode);
+
+int ctdb_ctrl_get_config(struct ctdb_context *ctdb);
+
+int ctdb_ctrl_get_debuglevel(struct ctdb_context *ctdb, uint32_t destnode, int32_t *level);
+int ctdb_ctrl_set_debuglevel(struct ctdb_context *ctdb, uint32_t destnode, int32_t level);
+
+/*
+  change dmaster for all keys in the database to the new value
+ */
+int ctdb_ctrl_setdmaster(struct ctdb_context *ctdb,
+	struct timeval timeout, uint32_t destnode,
+	TALLOC_CTX *mem_ctx, uint32_t dbid, uint32_t dmaster);
+
+/*
+  write a record on a specific db (this implicitely updates dmaster of the record to locally be the vnn of the node where the control is executed on)
+ */
+int ctdb_ctrl_write_record(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_CTX *mem_ctx, uint32_t dbid, TDB_DATA key, TDB_DATA data);
+
+#define CTDB_RECOVERY_NORMAL		0
+#define CTDB_RECOVERY_ACTIVE		1
+
+/*
+  get the recovery mode of a remote node
+ */
+int ctdb_ctrl_getrecmode(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, uint32_t *recmode);
+
+struct ctdb_client_control_state *ctdb_ctrl_getrecmode_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_getrecmode_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, uint32_t *recmode);
+
+
+/*
+  set the recovery mode of a remote node
+ */
+int ctdb_ctrl_setrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmode);
+/*
+  get the monitoring mode of a remote node
+ */
+int ctdb_ctrl_getmonmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *monmode);
+
+/*
+  set the monitoring mode of a remote node to active
+ */
+int ctdb_ctrl_enable_monmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+/*
+  set the monitoring mode of a remote node to disabled
+ */
+int ctdb_ctrl_disable_monmode(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+
+/*
+  get the recovery master of a remote node
+ */
+int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, uint32_t *recmaster);
+
+struct ctdb_client_control_state *ctdb_ctrl_getrecmaster_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_getrecmaster_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, uint32_t *recmaster);
+
+
+
+/*
+  set the recovery master of a remote node
+ */
+int ctdb_ctrl_setrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmaster);
+
+uint32_t *ctdb_get_connected_nodes(struct ctdb_context *ctdb,
+				   struct timeval timeout,
+				   TALLOC_CTX *mem_ctx,
+				   uint32_t *num_nodes);
+
+int ctdb_statistics_reset(struct ctdb_context *ctdb, uint32_t destnode);
+
+int ctdb_set_logfile(struct ctdb_context *ctdb, const char *logfile, bool use_syslog);
+
+typedef int (*ctdb_traverse_func)(struct ctdb_context *, TDB_DATA, TDB_DATA, void *);
+int ctdb_traverse(struct ctdb_db_context *ctdb_db, ctdb_traverse_func fn, void *private_data);
+
+struct ctdb_dump_db_context {
+	FILE *f;
+	bool printemptyrecords;
+	bool printdatasize;
+	bool printlmaster;
+	bool printhash;
+	bool printrecordflags;
+};
+
+int ctdb_dumpdb_record(struct ctdb_context *ctdb, TDB_DATA key, TDB_DATA data, void *p);
+int ctdb_dump_db(struct ctdb_db_context *ctdb_db,
+		 struct ctdb_dump_db_context *ctx);
+
+/*
+  get the pid of a ctdb daemon
+ */
+int ctdb_ctrl_getpid(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *pid);
+
+int ctdb_ctrl_freeze(struct ctdb_context *ctdb, struct timeval timeout,
+			uint32_t destnode);
+int ctdb_ctrl_freeze_priority(struct ctdb_context *ctdb, struct timeval timeout,
+			      uint32_t destnode, uint32_t priority);
+
+struct ctdb_client_control_state *
+ctdb_ctrl_freeze_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
+		      struct timeval timeout, uint32_t destnode,
+		      uint32_t priority);
+
+int ctdb_ctrl_freeze_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
+			struct ctdb_client_control_state *state);
+
+int ctdb_ctrl_thaw_priority(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t priority);
+int ctdb_ctrl_thaw(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_getpnn(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_get_tunable(struct ctdb_context *ctdb,
+			  struct timeval timeout,
+			  uint32_t destnode,
+			  const char *name, uint32_t *value);
+
+int ctdb_ctrl_set_tunable(struct ctdb_context *ctdb,
+			  struct timeval timeout,
+			  uint32_t destnode,
+			  const char *name, uint32_t value);
+
+int ctdb_ctrl_list_tunables(struct ctdb_context *ctdb,
+			    struct timeval timeout,
+			    uint32_t destnode,
+			    TALLOC_CTX *mem_ctx,
+			    const char ***list, uint32_t *count);
+
+int ctdb_ctrl_modflags(struct ctdb_context *ctdb,
+		       struct timeval timeout,
+		       uint32_t destnode,
+		       uint32_t set, uint32_t clear);
+
+enum ctdb_server_id_type {
+	SERVER_TYPE_SAMBA=1,
+	SERVER_TYPE_NFSD=2,
+	SERVER_TYPE_ISCSID=3
+};
+
+struct ctdb_server_id {
+	enum ctdb_server_id_type type;
+	uint32_t pnn;
+	uint32_t server_id;
+};
+
+struct ctdb_server_id_list {
+	uint32_t num;
+	struct ctdb_server_id server_ids[1];
+};
+
+
+int ctdb_ctrl_register_server_id(struct ctdb_context *ctdb,
+		struct timeval timeout,
+		struct ctdb_server_id *id);
+int ctdb_ctrl_unregister_server_id(struct ctdb_context *ctdb,
+		struct timeval timeout,
+		struct ctdb_server_id *id);
+int ctdb_ctrl_check_server_id(struct ctdb_context *ctdb,
+		struct timeval timeout, uint32_t destnode,
+		struct ctdb_server_id *id, uint32_t *status);
+int ctdb_ctrl_get_server_id_list(struct ctdb_context *ctdb,
+		TALLOC_CTX *mem_ctx,
+		struct timeval timeout, uint32_t destnode,
+		struct ctdb_server_id_list **svid_list);
+
+struct ctdb_uptime {
+	struct timeval current_time;
+	struct timeval ctdbd_start_time;
+	struct timeval last_recovery_started;
+	struct timeval last_recovery_finished;
+};
+
+/*
+  struct for tcp_client control
+  this is an ipv4 only version of this structure used by samba
+  samba will later be migrated over to use the
+  ctdb_control_tcp_addr structure instead
+ */
+struct ctdb_control_tcp {
+	struct sockaddr_in src; // samba uses this
+	struct sockaddr_in dest;// samba uses this
+};
+/* new style structure */
+struct ctdb_control_tcp_addr {
+	ctdb_sock_addr src;
+	ctdb_sock_addr dest;
+};
+
+int ctdb_socket_connect(struct ctdb_context *ctdb);
+
+/*
+  get the uptime of a remote node
+ */
+int ctdb_ctrl_uptime(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, struct ctdb_uptime **uptime);
+
+struct ctdb_client_control_state *ctdb_ctrl_uptime_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_uptime_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, struct ctdb_uptime **uptime);
+
+int ctdb_ctrl_end_recovery(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_getreclock(struct ctdb_context *ctdb,
+	struct timeval timeout, uint32_t destnode,
+	TALLOC_CTX *mem_ctx, const char **reclock);
+int ctdb_ctrl_setreclock(struct ctdb_context *ctdb,
+	struct timeval timeout, uint32_t destnode,
+	const char *reclock);
+
+
+uint32_t *list_of_connected_nodes(struct ctdb_context *ctdb,
+				struct ctdb_node_map *node_map,
+				TALLOC_CTX *mem_ctx,
+				bool include_self);
+uint32_t *list_of_active_nodes(struct ctdb_context *ctdb,
+				struct ctdb_node_map *node_map,
+				TALLOC_CTX *mem_ctx,
+				bool include_self);
+uint32_t *list_of_vnnmap_nodes(struct ctdb_context *ctdb,
+				struct ctdb_vnn_map *vnn_map,
+				TALLOC_CTX *mem_ctx,
+				bool include_self);
+uint32_t *list_of_active_nodes_except_pnn(struct ctdb_context *ctdb,
+				struct ctdb_node_map *node_map,
+				TALLOC_CTX *mem_ctx,
+				uint32_t pnn);
+
+int ctdb_read_pnn_lock(int fd, int32_t pnn);
+
+/*
+  get capabilities of a remote node
+ */
+int ctdb_ctrl_getcapabilities(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *capabilities);
+
+struct ctdb_client_control_state *ctdb_ctrl_getcapabilities_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_getcapabilities_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, uint32_t *capabilities);
+
+
+int32_t ctdb_ctrl_transaction_active(struct ctdb_context *ctdb,
+				     uint32_t destnode,
+				     uint32_t db_id);
+
+struct ctdb_marshall_buffer *ctdb_marshall_add(TALLOC_CTX *mem_ctx,
+					       struct ctdb_marshall_buffer *m,
+					       uint64_t db_id,
+					       uint32_t reqid,
+					       TDB_DATA key,
+					       struct ctdb_ltdb_header *header,
+					       TDB_DATA data);
+TDB_DATA ctdb_marshall_finish(struct ctdb_marshall_buffer *m);
+
+struct ctdb_transaction_handle *ctdb_transaction_start(struct ctdb_db_context *ctdb_db,
+						       TALLOC_CTX *mem_ctx);
+int ctdb_transaction_fetch(struct ctdb_transaction_handle *h,
+			   TALLOC_CTX *mem_ctx,
+			   TDB_DATA key, TDB_DATA *data);
+int ctdb_transaction_store(struct ctdb_transaction_handle *h,
+			   TDB_DATA key, TDB_DATA data);
+int ctdb_transaction_commit(struct ctdb_transaction_handle *h);
+
+int ctdb_ctrl_recd_ping(struct ctdb_context *ctdb);
+
+int switch_from_server_to_client(struct ctdb_context *ctdb, const char *fmt,
+				 ...);
+
+int ctdb_ctrl_getscriptstatus(struct ctdb_context *ctdb,
+		    struct timeval timeout, uint32_t destnode,
+		    TALLOC_CTX *mem_ctx, enum ctdb_eventscript_call type,
+		    struct ctdb_scripts_wire **script_status);
+
+
+struct debug_levels {
+	int32_t	level;
+	const char *description;
+};
+extern struct debug_levels debug_levels[];
+
+const char *get_debug_by_level(int32_t level);
+int32_t get_debug_by_desc(const char *desc);
+
+int ctdb_ctrl_stop_node(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+int ctdb_ctrl_continue_node(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode);
+
+int ctdb_ctrl_setnatgwstate(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t natgwstate);
+int ctdb_ctrl_setlmasterrole(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t lmasterrole);
+int ctdb_ctrl_setrecmasterrole(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t recmasterrole);
+
+int ctdb_ctrl_enablescript(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, const char *script);
+int ctdb_ctrl_disablescript(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, const char *script);
+
+struct ctdb_ban_time {
+	uint32_t pnn;
+	uint32_t time;
+};
+
+int ctdb_ctrl_set_ban(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, struct ctdb_ban_time *bantime);
+int ctdb_ctrl_get_ban(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_ban_time **bantime);
+
+struct ctdb_db_priority {
+	uint32_t db_id;
+	uint32_t priority;
+};
+
+int ctdb_ctrl_set_db_priority(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, struct ctdb_db_priority *db_prio);
+int ctdb_ctrl_get_db_priority(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t db_id, uint32_t *priority);
+
+int ctdb_ctrl_getstathistory(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_statistics_wire **stats);
+
+
+
+struct ctdb_client_control_state *
+ctdb_ctrl_updaterecord_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_ltdb_header *header, TDB_DATA data);
+
+int ctdb_ctrl_updaterecord_recv(struct ctdb_context *ctdb, struct ctdb_client_control_state *state);
+
+int
+ctdb_ctrl_updaterecord(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_ltdb_header *header, TDB_DATA data);
+
+
+struct ctdb_client_control_state *
+ctdb_ctrl_set_db_readonly_send(struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid);
+int ctdb_ctrl_set_db_readonly_recv(struct ctdb_context *ctdb, struct ctdb_client_control_state *state);
+int ctdb_ctrl_set_db_readonly(struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid);
+
+struct ctdb_client_control_state *
+ctdb_ctrl_set_db_sticky_send(struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid);
+int ctdb_ctrl_set_db_sticky_recv(struct ctdb_context *ctdb, struct ctdb_client_control_state *state);
+int ctdb_ctrl_set_db_sticky(struct ctdb_context *ctdb, uint32_t destnode, uint32_t dbid);
+
+#endif /* _CTDB_CLIENT_H */
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
new file mode 100644
index 0000000000..0f9da360c6
--- /dev/null
+++ b/ctdb/include/ctdb_private.h
@@ -0,0 +1,1502 @@
+/* 
+   ctdb database library
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _CTDB_PRIVATE_H
+#define _CTDB_PRIVATE_H
+
+#include "ctdb_client.h"
+#include <sys/socket.h>
+
+/*
+  recovery daemon memdump reply address
+ */
+struct rd_memdump_reply {
+	uint32_t pnn;
+	uint64_t srvid;
+};
+
+/*
+  description for a TAKEOVER_RUN message reply address
+ */
+struct takeover_run_reply {
+	uint32_t pnn;
+	uint64_t srvid;
+};
+
+/*
+ * pid of the ctdbd daemon
+ */
+extern pid_t ctdbd_pid;
+
+/*
+  a tcp connection description
+  also used by tcp_add and tcp_remove controls
+ */
+struct ctdb_tcp_connection {
+	ctdb_sock_addr src_addr;
+	ctdb_sock_addr dst_addr;
+};
+
+/* the wire representation for a tcp tickle array */
+struct ctdb_tcp_wire_array {
+	uint32_t num;
+	struct ctdb_tcp_connection connections[1];
+};	
+
+/* the list of tcp tickles used by get/set tcp tickle list */
+struct ctdb_control_tcp_tickle_list {
+	ctdb_sock_addr addr;
+	struct ctdb_tcp_wire_array tickles;
+};
+
+/*
+  array of tcp connections
+ */
+struct ctdb_tcp_array {
+	uint32_t num;
+	struct ctdb_tcp_connection *connections;
+};	
+
+
+/* all tunable variables go in here */
+struct ctdb_tunable {
+	uint32_t max_redirect_count;
+	uint32_t seqnum_interval; /* unit is ms */
+	uint32_t control_timeout;
+	uint32_t traverse_timeout;
+	uint32_t keepalive_interval;
+	uint32_t keepalive_limit;
+	uint32_t recover_timeout;
+	uint32_t recover_interval;
+	uint32_t election_timeout;
+	uint32_t takeover_timeout;
+	uint32_t monitor_interval;
+	uint32_t tickle_update_interval;
+	uint32_t script_timeout;
+	uint32_t script_timeout_count; /* allow dodgy scripts to hang this many times in a row before we mark the node unhealthy */
+	uint32_t script_unhealthy_on_timeout; /* obsolete */
+	uint32_t recovery_grace_period;
+	uint32_t recovery_ban_period;
+	uint32_t database_hash_size;
+	uint32_t database_max_dead;
+	uint32_t rerecovery_timeout;
+	uint32_t enable_bans;
+	uint32_t deterministic_public_ips;
+	uint32_t reclock_ping_period;
+	uint32_t no_ip_failback;
+	uint32_t disable_ip_failover;
+	uint32_t verbose_memory_names;
+	uint32_t recd_ping_timeout;
+	uint32_t recd_ping_failcount;
+	uint32_t log_latency_ms;
+	uint32_t reclock_latency_ms;
+	uint32_t recovery_drop_all_ips;
+	uint32_t verify_recovery_lock;
+	uint32_t vacuum_interval;
+	uint32_t vacuum_default_interval;
+	uint32_t vacuum_max_run_time;
+	uint32_t repack_limit;
+	uint32_t vacuum_limit;
+	uint32_t vacuum_min_interval;
+	uint32_t vacuum_max_interval;
+	uint32_t max_queue_depth_drop_msg;
+	uint32_t use_status_events_for_monitoring;
+	uint32_t allow_unhealthy_db_read;
+	uint32_t stat_history_interval;
+	uint32_t deferred_attach_timeout;
+	uint32_t vacuum_fast_path_count;
+	uint32_t lcp2_public_ip_assignment;
+	uint32_t allow_client_db_attach;
+	uint32_t recover_pdb_by_seqnum;
+	uint32_t deferred_rebalance_on_node_add;
+	uint32_t fetch_collapse;
+	uint32_t max_lacount;
+	uint32_t hopcount_make_sticky;
+	uint32_t sticky_duration;
+	uint32_t sticky_pindown;
+	uint32_t no_ip_takeover;
+	uint32_t db_record_count_warn;
+	uint32_t db_record_size_warn;
+	uint32_t db_size_warn;
+	uint32_t pulldb_preallocation_size;
+};
+
+/*
+  an installed ctdb remote call
+*/
+struct ctdb_registered_call {
+	struct ctdb_registered_call *next, *prev;
+	uint32_t id;
+	ctdb_fn_t fn;
+};
+
+/*
+  this address structure might need to be generalised later for some
+  transports
+*/
+struct ctdb_address {
+	const char *address;
+	int port;
+};
+
+/*
+  check that a pnn is valid
+ */
+#define ctdb_validate_pnn(ctdb, pnn) (((uint32_t)(pnn)) < (ctdb)->num_nodes)
+
+
+/* called from the queue code when a packet comes in. Called with data==NULL
+   on error */
+typedef void (*ctdb_queue_cb_fn_t)(uint8_t *data, size_t length,
+				   void *private_data);
+
+/* used for callbacks in ctdb_control requests */
+typedef void (*ctdb_control_callback_fn_t)(struct ctdb_context *,
+					   int32_t status, TDB_DATA data, 
+					   const char *errormsg,
+					   void *private_data);
+/*
+  structure describing a connected client in the daemon
+ */
+struct ctdb_client {
+	struct ctdb_context *ctdb;
+	int fd;
+	struct ctdb_queue *queue;
+	uint32_t client_id;
+	pid_t pid;
+	struct ctdb_tcp_list *tcp_list;
+	uint32_t db_id;
+	uint32_t num_persistent_updates;
+	struct ctdb_client_notify_list *notify;
+};
+
+struct ctdb_iface;
+
+/* state associated with a public ip address */
+struct ctdb_vnn {
+	struct ctdb_vnn *prev, *next;
+
+	struct ctdb_iface *iface;
+	const char **ifaces;
+	ctdb_sock_addr public_address;
+	uint8_t public_netmask_bits;
+
+	/* the node number that is serving this public address, if any. 
+	   If no node serves this ip it is set to -1 */
+	int32_t pnn;
+
+	/* List of clients to tickle for this public address */
+	struct ctdb_tcp_array *tcp_array;
+
+	/* whether we need to update the other nodes with changes to our list
+	   of connected clients */
+	bool tcp_update_needed;
+
+	/* a context to hang sending gratious arp events off */
+	TALLOC_CTX *takeover_ctx;
+
+	struct ctdb_kill_tcp *killtcp;
+};
+
+/*
+  state associated with one node
+*/
+struct ctdb_node {
+	struct ctdb_context *ctdb;
+	struct ctdb_address address;
+	const char *name; /* for debug messages */
+	void *private_data; /* private to transport */
+	uint32_t pnn;
+	uint32_t flags;
+
+	/* used by the dead node monitoring */
+	uint32_t dead_count;
+	uint32_t rx_cnt;
+	uint32_t tx_cnt;
+
+	/* used to track node capabilities, is only valid/tracked inside the
+	   recovery daemon.
+	*/
+	uint32_t capabilities;
+
+	/* a list of controls pending to this node, so we can time them out quickly
+	   if the node becomes disconnected */
+	struct daemon_control_state *pending_controls;
+
+	/* used by the recovery daemon when distributing ip addresses 
+	   across the nodes.  it needs to know which public ip's can be handled
+	   by each node.
+	*/
+	struct ctdb_all_public_ips *known_public_ips;
+	struct ctdb_all_public_ips *available_public_ips;
+	/* used by the recovery dameon to track when a node should be banned */
+	struct ctdb_banning_state *ban_state; 
+};
+
+/*
+  transport specific methods
+*/
+struct ctdb_methods {
+	int (*initialise)(struct ctdb_context *); /* initialise transport structures */	
+	int (*start)(struct ctdb_context *); /* start the transport */
+	int (*add_node)(struct ctdb_node *); /* setup a new node */	
+	int (*connect_node)(struct ctdb_node *); /* connect to node */
+	int (*queue_pkt)(struct ctdb_node *, uint8_t *data, uint32_t length);
+	void *(*allocate_pkt)(TALLOC_CTX *mem_ctx, size_t );
+	void (*shutdown)(struct ctdb_context *); /* shutdown transport */
+	void (*restart)(struct ctdb_node *); /* stop and restart the connection */
+};
+
+/*
+  transport calls up to the ctdb layer
+*/
+struct ctdb_upcalls {
+	/* recv_pkt is called when a packet comes in */
+	void (*recv_pkt)(struct ctdb_context *, uint8_t *data, uint32_t length);
+
+	/* node_dead is called when an attempt to send to a node fails */
+	void (*node_dead)(struct ctdb_node *);
+
+	/* node_connected is called when a connection to a node is established */
+	void (*node_connected)(struct ctdb_node *);
+};
+
+/* list of message handlers - needs to be changed to a more efficient data
+   structure so we can find a message handler given a srvid quickly */
+struct ctdb_message_list {
+	struct ctdb_context *ctdb;
+	struct ctdb_message_list *next, *prev;
+	uint64_t srvid;
+	ctdb_msg_fn_t message_handler;
+	void *message_private;
+};
+
+/* additional data required for the daemon mode */
+struct ctdb_daemon_data {
+	int sd;
+	char *name;
+	struct ctdb_queue *queue;
+};
+
+
+#define CTDB_UPDATE_STAT(ctdb, counter, value) \
+	{										\
+		if (value > ctdb->statistics.counter) {					\
+			ctdb->statistics.counter = c->hopcount;				\
+		}									\
+		if (value > ctdb->statistics_current.counter) {				\
+			ctdb->statistics_current.counter = c->hopcount;			\
+		}									\
+	}
+
+#define CTDB_INCREMENT_STAT(ctdb, counter) \
+	{										\
+		ctdb->statistics.counter++;						\
+		ctdb->statistics_current.counter++;					\
+	}
+
+#define CTDB_INCREMENT_DB_STAT(ctdb_db, counter) \
+	{										\
+		ctdb_db->statistics.counter++;						\
+	}
+
+#define CTDB_DECREMENT_STAT(ctdb, counter) \
+	{										\
+		if (ctdb->statistics.counter > 0)					\
+			ctdb->statistics.counter--;					\
+		if (ctdb->statistics_current.counter > 0)				\
+			ctdb->statistics_current.counter--;				\
+	}
+
+#define CTDB_UPDATE_RECLOCK_LATENCY(ctdb, name, counter, value) \
+	{										\
+		if (value > ctdb->statistics.counter.max)					\
+			ctdb->statistics.counter.max = value;				\
+		if (value > ctdb->statistics_current.counter.max)				\
+			ctdb->statistics_current.counter.max = value;			\
+											\
+		if (ctdb->statistics.counter.num == 0 || value < ctdb->statistics.counter.min)	\
+			ctdb->statistics.counter.min = value;				\
+		if (ctdb->statistics_current.counter.num == 0 || value < ctdb->statistics_current.counter.min)	\
+			ctdb->statistics_current.counter.min = value;			\
+											\
+		ctdb->statistics.counter.total += value;					\
+		ctdb->statistics_current.counter.total += value;				\
+											\
+		ctdb->statistics.counter.num++;						\
+		ctdb->statistics_current.counter.num++;					\
+											\
+		if (ctdb->tunable.reclock_latency_ms != 0) {				\
+			if (value*1000 > ctdb->tunable.reclock_latency_ms) {		\
+				DEBUG(DEBUG_ERR, ("High RECLOCK latency %fs for operation %s\n", value, name));	\
+			}								\
+		}									\
+	}
+
+
+#define CTDB_UPDATE_LATENCY(ctdb, db, operation, counter, t) \
+	{										\
+		double l = timeval_elapsed(&t);						\
+											\
+		if (l > ctdb->statistics.counter.max)					\
+			ctdb->statistics.counter.max = l;				\
+		if (l > ctdb->statistics_current.counter.max)				\
+			ctdb->statistics_current.counter.max = l;			\
+											\
+		if (ctdb->statistics.counter.num == 0 || l < ctdb->statistics.counter.min)	\
+			ctdb->statistics.counter.min = l;				\
+		if (ctdb->statistics_current.counter.num == 0 || l < ctdb->statistics_current.counter.min)	\
+			ctdb->statistics_current.counter.min = l;			\
+											\
+		ctdb->statistics.counter.total += l;					\
+		ctdb->statistics_current.counter.total += l;				\
+											\
+		ctdb->statistics.counter.num++;						\
+		ctdb->statistics_current.counter.num++;					\
+											\
+		if (ctdb->tunable.log_latency_ms !=0) {					\
+			if (l*1000 > ctdb->tunable.log_latency_ms) {			\
+				DEBUG(DEBUG_WARNING, ("High latency %.6fs for operation %s on database %s\n", l, operation, db->db_name));\
+			}								\
+		}									\
+	}
+
+
+
+/* a structure that contains the elements required for the write record
+   control
+*/
+struct ctdb_write_record {
+	uint32_t dbid;
+	uint32_t keylen;
+	uint32_t datalen;
+	unsigned char blob[1];
+};
+
+enum ctdb_freeze_mode {CTDB_FREEZE_NONE, CTDB_FREEZE_PENDING, CTDB_FREEZE_FROZEN};
+
+#define CTDB_MONITORING_ACTIVE		0
+#define CTDB_MONITORING_DISABLED	1
+
+/* The different capabilities of the ctdb daemon. */
+#define CTDB_CAP_RECMASTER		0x00000001
+#define CTDB_CAP_LMASTER		0x00000002
+/* This capability is set if CTDB_LVS_PUBLIC_IP is set */
+#define CTDB_CAP_LVS			0x00000004
+/* This capability is set if NATGW is enabled */
+#define CTDB_CAP_NATGW			0x00000008
+
+#define NUM_DB_PRIORITIES 3
+/* main state of the ctdb daemon */
+struct ctdb_context {
+	struct tevent_context *ev;
+	struct timeval ctdbd_start_time;
+	struct timeval last_recovery_started;
+	struct timeval last_recovery_finished;
+	uint32_t recovery_mode;
+	TALLOC_CTX *tickle_update_context;
+	TALLOC_CTX *keepalive_ctx;
+	TALLOC_CTX *check_public_ifaces_ctx;
+	struct ctdb_tunable tunable;
+	enum ctdb_freeze_mode freeze_mode[NUM_DB_PRIORITIES+1];
+	struct ctdb_freeze_handle *freeze_handles[NUM_DB_PRIORITIES+1];
+	bool freeze_transaction_started;
+	uint32_t freeze_transaction_id;
+	struct ctdb_address address;
+	const char *name;
+	const char *db_directory;
+	const char *db_directory_persistent;
+	const char *db_directory_state;
+	struct tdb_wrap *db_persistent_health;
+	uint32_t db_persistent_startup_generation;
+	uint64_t db_persistent_check_errors;
+	uint64_t max_persistent_check_errors;
+	const char *transport;
+	char *recovery_lock_file;
+	int recovery_lock_fd;
+	uint32_t pnn; /* our own pnn */
+	uint32_t num_nodes;
+	uint32_t num_connected;
+	unsigned flags;
+	uint32_t capabilities;
+	struct idr_context *idr;
+	int lastid;
+	struct ctdb_node **nodes; /* array of nodes in the cluster - indexed by vnn */
+	struct ctdb_vnn *vnn; /* list of public ip addresses and interfaces */
+	struct ctdb_vnn *single_ip_vnn; /* a structure for the single ip */
+	struct ctdb_iface *ifaces; /* list of local interfaces */
+	char *err_msg;
+	const struct ctdb_methods *methods; /* transport methods */
+	const struct ctdb_upcalls *upcalls; /* transport upcalls */
+	void *private_data; /* private to transport */
+	struct ctdb_db_context *db_list;
+	struct ctdb_message_list *message_list;
+	struct ctdb_daemon_data daemon;
+	struct ctdb_statistics statistics;
+	struct ctdb_statistics statistics_current;
+#define MAX_STAT_HISTORY 100
+	struct ctdb_statistics statistics_history[MAX_STAT_HISTORY];
+	struct ctdb_vnn_map *vnn_map;
+	uint32_t num_clients;
+	uint32_t recovery_master;
+	struct ctdb_call_state *pending_calls;
+	struct ctdb_client_ip *client_ip_list;
+	bool do_checkpublicip;
+	struct trbt_tree *server_ids; 
+	bool do_setsched;
+	void *saved_scheduler_param;
+	const char *event_script_dir;
+	const char *debug_hung_script;
+	const char *notification_script;
+	const char *default_public_interface;
+	pid_t ctdbd_pid;
+	pid_t recoverd_pid;
+	pid_t syslogd_pid;
+	bool done_startup;
+	const char *node_ip;
+	struct ctdb_monitor_state *monitor;
+	struct ctdb_log_state *log;
+	int start_as_disabled;
+	int start_as_stopped;
+	bool valgrinding;
+	uint32_t event_script_timeouts; /* counting how many consecutive times an eventscript has timedout */
+	uint32_t *recd_ping_count;
+	TALLOC_CTX *release_ips_ctx; /* a context used to automatically drop all IPs if we fail to recover the node */
+
+	TALLOC_CTX *event_script_ctx;
+
+	struct ctdb_event_script_state *current_monitor;
+	struct ctdb_scripts_wire *last_status[CTDB_EVENT_MAX];
+
+	TALLOC_CTX *banning_ctx;
+
+	struct ctdb_vacuum_child_context *vacuumers;
+
+	/* mapping from pid to ctdb_client * */
+	struct ctdb_client_pid_list *client_pids;
+
+	/* used in the recovery daemon to remember the ip allocation */
+	struct trbt_tree *ip_tree;
+
+	/* Used to defer db attach requests while in recovery mode */
+	struct ctdb_deferred_attach_context *deferred_attach;
+
+	/* if we are a child process, do we have a domain socket to send controls on */
+	bool can_send_controls;
+
+	/* list of event script callback functions that are active */
+	struct event_script_callback *script_callbacks;
+
+	struct ctdb_reloadips_handle *reload_ips;
+
+	const char *public_addresses_file;
+	struct trbt_tree *child_processes; 
+	TALLOC_CTX *debug_hung_script_ctx;
+};
+
+struct ctdb_db_context {
+	struct ctdb_db_context *next, *prev;
+	struct ctdb_context *ctdb;
+	uint32_t db_id;
+	uint32_t priority;
+	bool persistent;
+	bool readonly; /* Do we support read-only delegations ? */
+	bool sticky; /* Do we support sticky records ? */
+	const char *db_name;
+	const char *db_path;
+	struct tdb_wrap *ltdb;
+	struct tdb_context *rottdb; /* ReadOnly tracking TDB */
+	struct ctdb_registered_call *calls; /* list of registered calls */
+	uint32_t seqnum;
+	struct timed_event *seqnum_update;
+	struct ctdb_traverse_local_handle *traverse;
+	bool transaction_active;
+	struct ctdb_vacuum_handle *vacuum_handle;
+	char *unhealthy_reason;
+	int pending_requests;
+	struct lockwait_handle *lockwait_active;
+	struct lockwait_handle *lockwait_overflow;
+	struct revokechild_handle *revokechild_active;
+	struct ctdb_persistent_state *persistent_state;
+	struct trbt_tree *delete_queue;
+	struct trbt_tree *sticky_records; 
+	int (*ctdb_ltdb_store_fn)(struct ctdb_db_context *ctdb_db,
+				  TDB_DATA key,
+				  struct ctdb_ltdb_header *header,
+				  TDB_DATA data);
+
+	/* used to track which records we are currently fetching
+	   so we can avoid sending duplicate fetch requests
+	*/
+	struct trbt_tree *deferred_fetch;
+
+	struct ctdb_db_statistics statistics;
+};
+
+
+#define CTDB_NO_MEMORY(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
+          ctdb_set_error(ctdb, "Out of memory at %s:%d", __FILE__, __LINE__); \
+	  return -1; }} while (0)
+
+#define CTDB_NO_MEMORY_VOID(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
+          ctdb_set_error(ctdb, "Out of memory at %s:%d", __FILE__, __LINE__); \
+	  return; }} while (0)
+
+#define CTDB_NO_MEMORY_NULL(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
+          ctdb_set_error(ctdb, "Out of memory at %s:%d", __FILE__, __LINE__); \
+	  return NULL; }} while (0)
+
+#define CTDB_NO_MEMORY_FATAL(ctdb, p) do { if (!(p)) { \
+          DEBUG(0,("Out of memory for %s at %s\n", #p, __location__)); \
+          ctdb_fatal(ctdb, "Out of memory in " __location__ ); \
+	  }} while (0)
+
+/*
+  structure passed in set_call control
+ */
+struct ctdb_control_set_call {
+	uint32_t db_id;
+	ctdb_fn_t fn;
+	uint32_t id;
+};
+
+/*
+  struct for kill_tcp control
+ */
+struct ctdb_control_killtcp {
+	ctdb_sock_addr src_addr;
+	ctdb_sock_addr dst_addr;
+};
+
+/*
+  struct holding a ctdb_sock_addr and an interface name,
+  used to add/remove public addresses
+ */
+struct ctdb_control_ip_iface {
+	ctdb_sock_addr addr;
+	uint32_t mask;
+	uint32_t len;
+	char iface[1];
+};
+
+/*
+  struct holding a ctdb_sock_addr and an interface name,
+  used for send_gratious_arp
+ */
+struct ctdb_control_gratious_arp {
+	ctdb_sock_addr addr;
+	uint32_t mask;
+	uint32_t len;
+	char iface[1];
+};
+
+/*
+  persistent store control - update this record on all other nodes
+ */
+struct ctdb_control_persistent_store {
+	uint32_t db_id;
+	uint32_t len;
+	uint8_t  data[1];
+};
+
+/*
+  structure used for CTDB_SRVID_NODE_FLAGS_CHANGED
+ */
+struct ctdb_node_flag_change {
+	uint32_t pnn;
+	uint32_t new_flags;
+	uint32_t old_flags;
+};
+
+/*
+  struct for admin setting a ban
+ */
+struct ctdb_ban_info {
+	uint32_t pnn;
+	uint32_t ban_time;
+};
+
+enum call_state {CTDB_CALL_WAIT, CTDB_CALL_DONE, CTDB_CALL_ERROR};
+
+#define CTDB_LMASTER_ANY	0xffffffff
+
+/*
+  state of a in-progress ctdb call
+*/
+struct ctdb_call_state {
+	struct ctdb_call_state *next, *prev;
+	enum call_state state;
+	uint32_t reqid;
+	struct ctdb_req_call *c;
+	struct ctdb_db_context *ctdb_db;
+	const char *errmsg;
+	struct ctdb_call *call;
+	uint32_t generation;
+	struct {
+		void (*fn)(struct ctdb_call_state *);
+		void *private_data;
+	} async;
+};
+
+
+/* used for fetch_lock */
+struct ctdb_fetch_handle {
+	struct ctdb_db_context *ctdb_db;
+	TDB_DATA key;
+	TDB_DATA *data;
+	struct ctdb_ltdb_header header;
+};
+
+/* internal prototypes */
+void ctdb_set_error(struct ctdb_context *ctdb, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3);
+void ctdb_fatal(struct ctdb_context *ctdb, const char *msg);
+bool ctdb_same_address(struct ctdb_address *a1, struct ctdb_address *a2);
+int ctdb_parse_address(struct ctdb_context *ctdb,
+		       TALLOC_CTX *mem_ctx, const char *str,
+		       struct ctdb_address *address);
+bool ctdb_same_ip(const ctdb_sock_addr *ip1, const ctdb_sock_addr *ip2);
+bool ctdb_same_sockaddr(const ctdb_sock_addr *ip1, const ctdb_sock_addr *ip2);
+uint32_t ctdb_hash(const TDB_DATA *key);
+uint32_t ctdb_hash_string(const char *str);
+void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_request_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_request_message(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_reply_dmaster(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_reply_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_reply_error(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+
+uint32_t ctdb_lmaster(struct ctdb_context *ctdb, const TDB_DATA *key);
+int ctdb_ltdb_fetch(struct ctdb_db_context *ctdb_db, 
+		    TDB_DATA key, struct ctdb_ltdb_header *header, 
+		    TALLOC_CTX *mem_ctx, TDB_DATA *data);
+int ctdb_ltdb_store(struct ctdb_db_context *ctdb_db, TDB_DATA key, 
+		    struct ctdb_ltdb_header *header, TDB_DATA data);
+int ctdb_ltdb_delete(struct ctdb_db_context *ctdb_db, TDB_DATA key);
+int ctdb_ltdb_fetch_with_header(struct ctdb_db_context *ctdb_db, 
+		    TDB_DATA key, struct ctdb_ltdb_header *header, 
+		    TALLOC_CTX *mem_ctx, TDB_DATA *data);
+int32_t ctdb_control_start_persistent_update(struct ctdb_context *ctdb, 
+			struct ctdb_req_control *c,
+			TDB_DATA recdata);
+int32_t ctdb_control_cancel_persistent_update(struct ctdb_context *ctdb, 
+			struct ctdb_req_control *c,
+			TDB_DATA recdata);
+void ctdb_queue_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_queue_packet_opcode(struct ctdb_context *ctdb, struct ctdb_req_header *hdr, unsigned opcode);
+int ctdb_ltdb_lock_requeue(struct ctdb_db_context *ctdb_db, 
+			   TDB_DATA key, struct ctdb_req_header *hdr,
+			   void (*recv_pkt)(void *, struct ctdb_req_header *),
+			   void *recv_context, bool ignore_generation);
+int ctdb_ltdb_lock_fetch_requeue(struct ctdb_db_context *ctdb_db, 
+				 TDB_DATA key, struct ctdb_ltdb_header *header, 
+				 struct ctdb_req_header *hdr, TDB_DATA *data,
+				 void (*recv_pkt)(void *, struct ctdb_req_header *),
+				 void *recv_context, bool ignore_generation);
+void ctdb_input_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *);
+
+struct ctdb_call_state *ctdb_call_local_send(struct ctdb_db_context *ctdb_db, 
+					     struct ctdb_call *call,
+					     struct ctdb_ltdb_header *header,
+					     TDB_DATA *data);
+
+
+int ctdbd_start(struct ctdb_context *ctdb);
+struct ctdb_call_state *ctdbd_call_send(struct ctdb_db_context *ctdb_db, struct ctdb_call *call);
+int ctdbd_call_recv(struct ctdb_call_state *state, struct ctdb_call *call);
+
+/*
+  queue a packet for sending
+*/
+int ctdb_queue_send(struct ctdb_queue *queue, uint8_t *data, uint32_t length);
+
+/*
+  setup the fd used by the queue
+ */
+int ctdb_queue_set_fd(struct ctdb_queue *queue, int fd);
+
+/*
+  setup a packet queue on a socket
+ */
+struct ctdb_queue *ctdb_queue_setup(struct ctdb_context *ctdb,
+				    TALLOC_CTX *mem_ctx, int fd, int alignment,
+				    
+				    ctdb_queue_cb_fn_t callback,
+				    void *private_data, const char *fmt, ...)
+	PRINTF_ATTRIBUTE(7,8);
+
+/*
+  allocate a packet for use in client<->daemon communication
+ */
+struct ctdb_req_header *_ctdbd_allocate_pkt(struct ctdb_context *ctdb,
+					    TALLOC_CTX *mem_ctx, 
+					    enum ctdb_operation operation, 
+					    size_t length, size_t slength,
+					    const char *type);
+#define ctdbd_allocate_pkt(ctdb, mem_ctx, operation, length, type) \
+	(type *)_ctdbd_allocate_pkt(ctdb, mem_ctx, operation, length, sizeof(type), #type)
+
+struct ctdb_req_header *_ctdb_transport_allocate(struct ctdb_context *ctdb,
+						 TALLOC_CTX *mem_ctx, 
+						 enum ctdb_operation operation, 
+						 size_t length, size_t slength,
+						 const char *type);
+#define ctdb_transport_allocate(ctdb, mem_ctx, operation, length, type) \
+	(type *)_ctdb_transport_allocate(ctdb, mem_ctx, operation, length, sizeof(type), #type)
+
+int ctdb_queue_length(struct ctdb_queue *queue);
+
+/*
+  lock a record in the ltdb, given a key
+ */
+int ctdb_ltdb_lock(struct ctdb_db_context *ctdb_db, TDB_DATA key);
+
+/*
+  unlock a record in the ltdb, given a key
+ */
+int ctdb_ltdb_unlock(struct ctdb_db_context *ctdb_db, TDB_DATA key);
+
+
+/*
+  make a ctdb call to the local daemon - async send. Called from client context.
+
+  This constructs a ctdb_call request and queues it for processing. 
+  This call never blocks.
+*/
+struct ctdb_call_state *ctdb_client_call_send(struct ctdb_db_context *ctdb_db, 
+					      struct ctdb_call *call);
+
+/*
+  make a recv call to the local ctdb daemon - called from client context
+
+  This is called when the program wants to wait for a ctdb_call to complete and get the 
+  results. This call will block unless the call has already completed.
+*/
+int ctdb_client_call_recv(struct ctdb_call_state *state, struct ctdb_call *call);
+
+int ctdb_client_send_message(struct ctdb_context *ctdb, uint32_t vnn,
+			     uint64_t srvid, TDB_DATA data);
+
+/*
+  send a ctdb message
+*/
+int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t pnn,
+			     uint64_t srvid, TDB_DATA data);
+
+
+struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
+				      TDB_DATA key,
+				      void (*callback)(void *), void *private_data);
+
+struct ctdb_call_state *ctdb_daemon_call_send(struct ctdb_db_context *ctdb_db, 
+					      struct ctdb_call *call);
+
+int ctdb_daemon_call_recv(struct ctdb_call_state *state, struct ctdb_call *call);
+
+struct ctdb_call_state *ctdb_daemon_call_send_remote(struct ctdb_db_context *ctdb_db, 
+						     struct ctdb_call *call, 
+						     struct ctdb_ltdb_header *header);
+
+int ctdb_call_local(struct ctdb_db_context *ctdb_db, struct ctdb_call *call,
+		    struct ctdb_ltdb_header *header, TALLOC_CTX *mem_ctx,
+		    TDB_DATA *data, bool updatetdb, uint32_t caller);
+
+#define ctdb_reqid_find(ctdb, reqid, type)	(type *)_ctdb_reqid_find(ctdb, reqid, #type, __location__)
+
+void ctdb_recv_raw_pkt(void *p, uint8_t *data, uint32_t length);
+
+int ctdb_socket_connect(struct ctdb_context *ctdb);
+void ctdb_client_read_cb(uint8_t *data, size_t cnt, void *args);
+
+#define CTDB_BAD_REQID ((uint32_t)-1)
+uint32_t ctdb_reqid_new(struct ctdb_context *ctdb, void *state);
+void *_ctdb_reqid_find(struct ctdb_context *ctdb, uint32_t reqid, const char *type, const char *location);
+void ctdb_reqid_remove(struct ctdb_context *ctdb, uint32_t reqid);
+
+void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+void ctdb_reply_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr);
+
+int ctdb_daemon_send_control(struct ctdb_context *ctdb, uint32_t destnode,
+			     uint64_t srvid, uint32_t opcode, uint32_t client_id, uint32_t flags,
+			     TDB_DATA data,
+			     ctdb_control_callback_fn_t callback,
+			     void *private_data);
+
+int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata, 
+			       TDB_DATA *outdata, uint64_t tdb_flags,
+			       bool persistent, uint32_t client_id,
+			       struct ctdb_req_control *c,
+			       bool *async_reply);
+
+int ctdb_daemon_set_call(struct ctdb_context *ctdb, uint32_t db_id,
+			 ctdb_fn_t fn, int id);
+
+int ctdb_control(struct ctdb_context *ctdb, uint32_t destnode, uint64_t srvid, 
+		 uint32_t opcode, uint32_t flags, TDB_DATA data, 
+		 TALLOC_CTX *mem_ctx, TDB_DATA *outdata, int32_t *status,
+		 struct timeval *timeout, char **errormsg);
+int ctdb_control_recv(struct ctdb_context *ctdb, 
+		struct ctdb_client_control_state *state, 
+		TALLOC_CTX *mem_ctx,
+		TDB_DATA *outdata, int32_t *status, char **errormsg);
+
+struct ctdb_client_control_state *
+ctdb_control_send(struct ctdb_context *ctdb, 
+		uint32_t destnode, uint64_t srvid, 
+		uint32_t opcode, uint32_t flags, TDB_DATA data, 
+		TALLOC_CTX *mem_ctx,
+		struct timeval *timeout,
+		char **errormsg);
+
+
+
+
+#define CHECK_CONTROL_DATA_SIZE(size) do { \
+ if (indata.dsize != size) { \
+	 DEBUG(0,(__location__ " Invalid data size in opcode %u. Got %u expected %u\n", \
+		  opcode, (unsigned)indata.dsize, (unsigned)size));	\
+	 return -1; \
+ } \
+ } while (0)
+
+#define CHECK_CONTROL_MIN_DATA_SIZE(size) do { \
+ if (indata.dsize < size) { \
+	 DEBUG(0,(__location__ " Invalid data size in opcode %u. Got %u expected >= %u\n", \
+		  opcode, (unsigned)indata.dsize, (unsigned)size));	\
+	 return -1; \
+ } \
+ } while (0)
+
+int ctdb_control_getvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_setvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_getdbmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_getnodemapv4(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+int ctdb_control_writerecord(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata);
+
+
+/* structure used for pulldb control */
+struct ctdb_control_pulldb {
+	uint32_t db_id;
+	uint32_t lmaster;
+};
+
+/* structure used for sending lists of records */
+struct ctdb_marshall_buffer {
+	uint32_t db_id;
+	uint32_t count;
+	uint8_t data[1];
+};
+
+/* set dmaster control structure */
+struct ctdb_control_set_dmaster {
+	uint32_t db_id;
+	uint32_t dmaster;
+};
+
+/*
+  structure for setting a tunable
+ */
+struct ctdb_control_set_tunable {
+	uint32_t value;
+	uint32_t length;
+	uint8_t  name[1];
+};
+
+/*
+  structure for getting a tunable
+ */
+struct ctdb_control_get_tunable {
+	uint32_t length;
+	uint8_t  name[1];
+};
+
+/*
+  structure for listing tunables
+ */
+struct ctdb_control_list_tunable {
+	uint32_t length;
+	/* returns a : separated list of tunable names */
+	uint8_t  data[1];
+};
+
+
+struct ctdb_node_and_flagsv4 {
+	uint32_t pnn;
+	uint32_t flags;
+	struct sockaddr_in sin;
+};
+
+struct ctdb_node_mapv4 {
+	uint32_t num;
+	struct ctdb_node_and_flagsv4 nodes[1];
+};
+
+struct ctdb_control_wipe_database {
+	uint32_t db_id;
+	uint32_t transaction_id;
+};
+
+/*
+  state of a in-progress ctdb call in client
+*/
+struct ctdb_client_call_state {
+	enum call_state state;
+	uint32_t reqid;
+	struct ctdb_db_context *ctdb_db;
+	struct ctdb_call *call;
+	struct {
+		void (*fn)(struct ctdb_client_call_state *);
+		void *private_data;
+	} async;
+};
+
+
+int32_t ctdb_control_traverse_start_ext(struct ctdb_context *ctdb,
+					TDB_DATA indata,
+					TDB_DATA *outdata,
+					uint32_t srcnode,
+					uint32_t client_id);
+int32_t ctdb_control_traverse_start(struct ctdb_context *ctdb, TDB_DATA indata, 
+				    TDB_DATA *outdata, uint32_t srcnode, uint32_t client_id);
+int32_t ctdb_control_traverse_all(struct ctdb_context *ctdb, TDB_DATA data, TDB_DATA *outdata);
+int32_t ctdb_control_traverse_data(struct ctdb_context *ctdb, TDB_DATA data, TDB_DATA *outdata);
+int32_t ctdb_control_traverse_kill(struct ctdb_context *ctdb, TDB_DATA indata, 
+				    TDB_DATA *outdata, uint32_t srcnode);
+
+int ctdb_dispatch_message(struct ctdb_context *ctdb, uint64_t srvid, TDB_DATA data);
+
+int daemon_register_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid);
+int ctdb_deregister_message_handler(struct ctdb_context *ctdb, uint64_t srvid, void *private_data);
+int daemon_deregister_message_handler(struct ctdb_context *ctdb, uint32_t client_id, uint64_t srvid);
+int daemon_check_srvids(struct ctdb_context *ctdb, TDB_DATA indata,
+			TDB_DATA *outdata);
+
+int32_t ctdb_ltdb_enable_seqnum(struct ctdb_context *ctdb, uint32_t db_id);
+int32_t ctdb_ltdb_update_seqnum(struct ctdb_context *ctdb, uint32_t db_id, uint32_t srcnode);
+
+struct ctdb_rec_data *ctdb_marshall_record(TALLOC_CTX *mem_ctx, uint32_t reqid,	
+					   TDB_DATA key, struct ctdb_ltdb_header *, TDB_DATA data);
+
+struct ctdb_rec_data *ctdb_marshall_loop_next(struct ctdb_marshall_buffer *m, struct ctdb_rec_data *r,
+					      uint32_t *reqid,
+					      struct ctdb_ltdb_header *header,
+					      TDB_DATA *key, TDB_DATA *data);
+
+int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_set_dmaster(struct ctdb_context *ctdb, TDB_DATA indata);
+
+int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, bool *async_reply,
+				 const char **errormsg);
+void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_control *c,
+				TDB_DATA *outdata, int32_t status, const char *errormsg);
+
+int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply);
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority);
+
+int ctdb_start_recoverd(struct ctdb_context *ctdb);
+void ctdb_stop_recoverd(struct ctdb_context *ctdb);
+
+uint32_t ctdb_get_num_active_nodes(struct ctdb_context *ctdb);
+
+void ctdb_disable_monitoring(struct ctdb_context *ctdb);
+void ctdb_enable_monitoring(struct ctdb_context *ctdb);
+void ctdb_stop_monitoring(struct ctdb_context *ctdb);
+void ctdb_start_monitoring(struct ctdb_context *ctdb);
+void ctdb_start_tcp_tickle_update(struct ctdb_context *ctdb);
+void ctdb_send_keepalive(struct ctdb_context *ctdb, uint32_t destnode);
+void ctdb_start_keepalive(struct ctdb_context *ctdb);
+void ctdb_stop_keepalive(struct ctdb_context *ctdb);
+int32_t ctdb_run_eventscripts(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA data, bool *async_reply);
+
+
+void ctdb_daemon_cancel_controls(struct ctdb_context *ctdb, struct ctdb_node *node);
+void ctdb_call_resend_all(struct ctdb_context *ctdb);
+void ctdb_node_dead(struct ctdb_node *node);
+void ctdb_node_connected(struct ctdb_node *node);
+bool ctdb_blocking_freeze(struct ctdb_context *ctdb);
+void ctdb_set_scheduler(struct ctdb_context *ctdb);
+void ctdb_restore_scheduler(struct ctdb_context *ctdb);
+
+struct tevent_signal *ctdb_init_sigchld(struct ctdb_context *ctdb);
+pid_t ctdb_fork(struct ctdb_context *ctdb);
+int ctdb_kill(struct ctdb_context *ctdb, pid_t pid, int signum);
+
+int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply);
+int32_t ctdb_control_takeover_ipv4(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply);
+int32_t ctdb_control_release_ip(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply);
+int32_t ctdb_control_release_ipv4(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 TDB_DATA indata, 
+				 bool *async_reply);
+int32_t ctdb_control_start_recovery(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 bool *async_reply);
+int32_t ctdb_control_end_recovery(struct ctdb_context *ctdb, 
+				 struct ctdb_req_control *c,
+				 bool *async_reply);
+
+struct ctdb_public_ipv4 {
+	uint32_t pnn;
+	struct sockaddr_in sin;
+};
+
+int ctdb_ctrl_takeover_ip(struct ctdb_context *ctdb, struct timeval timeout, 
+			  uint32_t destnode, struct ctdb_public_ip *ip);
+int ctdb_ctrl_release_ip(struct ctdb_context *ctdb, struct timeval timeout, 
+			 uint32_t destnode, struct ctdb_public_ip *ip);
+
+struct ctdb_all_public_ipsv4 {
+	uint32_t num;
+	struct ctdb_public_ipv4 ips[1];
+};
+
+int32_t ctdb_control_get_public_ipsv4(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA *outdata);
+int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA *outdata);
+int ctdb_ctrl_get_public_ips(struct ctdb_context *ctdb, 
+			     struct timeval timeout,
+			     uint32_t destnode,
+			     TALLOC_CTX *mem_ctx,
+			     struct ctdb_all_public_ips **ips);
+#define CTDB_PUBLIC_IP_FLAGS_ONLY_AVAILABLE 0x00010000
+int ctdb_ctrl_get_public_ips_flags(struct ctdb_context *ctdb,
+				   struct timeval timeout, uint32_t destnode,
+				   TALLOC_CTX *mem_ctx,
+				   uint32_t flags,
+				   struct ctdb_all_public_ips **ips);
+int ctdb_ctrl_get_public_ipsv4(struct ctdb_context *ctdb, 
+			struct timeval timeout, uint32_t destnode, 
+			TALLOC_CTX *mem_ctx, struct ctdb_all_public_ips **ips);
+
+struct ctdb_control_iface_info {
+	char name[CTDB_IFACE_SIZE+2];
+	uint16_t link_state;
+	uint32_t references;
+};
+
+struct ctdb_control_public_ip_info {
+	struct ctdb_public_ip ip;
+	uint32_t active_idx;
+	uint32_t num;
+	struct ctdb_control_iface_info ifaces[1];
+};
+
+struct ctdb_control_get_ifaces {
+	uint32_t num;
+	struct ctdb_control_iface_info ifaces[1];
+};
+
+int32_t ctdb_control_get_public_ip_info(struct ctdb_context *ctdb,
+					struct ctdb_req_control *c,
+					TDB_DATA indata,
+					TDB_DATA *outdata);
+int32_t ctdb_control_get_ifaces(struct ctdb_context *ctdb,
+				struct ctdb_req_control *c,
+				TDB_DATA *outdata);
+int32_t ctdb_control_set_iface_link(struct ctdb_context *ctdb,
+				    struct ctdb_req_control *c,
+				    TDB_DATA indata);
+int ctdb_ctrl_get_public_ip_info(struct ctdb_context *ctdb,
+				 struct timeval timeout, uint32_t destnode,
+				 TALLOC_CTX *mem_ctx,
+				 const ctdb_sock_addr *addr,
+				 struct ctdb_control_public_ip_info **info);
+int ctdb_ctrl_get_ifaces(struct ctdb_context *ctdb,
+			 struct timeval timeout, uint32_t destnode,
+			 TALLOC_CTX *mem_ctx,
+			 struct ctdb_control_get_ifaces **ifaces);
+int ctdb_ctrl_set_iface_link(struct ctdb_context *ctdb,
+			     struct timeval timeout, uint32_t destnode,
+			     TALLOC_CTX *mem_ctx,
+			     const struct ctdb_control_iface_info *info);
+
+/* from takeover/system.c */
+uint32_t uint16_checksum(uint16_t *data, size_t n);
+int ctdb_sys_send_arp(const ctdb_sock_addr *addr, const char *iface);
+bool ctdb_sys_have_ip(ctdb_sock_addr *addr);
+char *ctdb_sys_find_ifname(ctdb_sock_addr *addr);
+bool ctdb_sys_check_iface_exists(const char *iface);
+int ctdb_get_peer_pid(const int fd, pid_t *peer_pid);
+int ctdb_sys_send_tcp(const ctdb_sock_addr *dest, 
+		      const ctdb_sock_addr *src,
+		      uint32_t seq, uint32_t ack, int rst);
+
+int ctdb_set_public_addresses(struct ctdb_context *ctdb, bool check_addresses);
+int ctdb_set_single_public_ip(struct ctdb_context *ctdb,
+			      const char *iface,
+			      const char *ip);
+int ctdb_set_event_script(struct ctdb_context *ctdb, const char *script);
+int ctdb_set_event_script_dir(struct ctdb_context *ctdb, const char *script_dir);
+int ctdb_set_debug_hung_script(struct ctdb_context *ctdb, const char *script);
+int ctdb_set_notification_script(struct ctdb_context *ctdb, const char *script);
+void lcp2_forcerebalance(struct ctdb_context *ctdb, uint32_t pnn);
+int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap);
+
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, 
+				TDB_DATA indata);
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata, bool tcp_update_needed);
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_startup(struct ctdb_context *ctdb, uint32_t vnn);
+int32_t ctdb_control_kill_tcp(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_send_gratious_arp(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_get_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int32_t ctdb_control_set_tcp_tickle_list(struct ctdb_context *ctdb, TDB_DATA indata);
+
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client);
+int ctdb_event_script(struct ctdb_context *ctdb, enum ctdb_eventscript_call call);
+int ctdb_event_script_args(struct ctdb_context *ctdb, enum ctdb_eventscript_call call,
+			   const char *fmt, ...) PRINTF_ATTRIBUTE(3,4);
+int ctdb_event_script_callback(struct ctdb_context *ctdb, 
+			       TALLOC_CTX *mem_ctx,
+			       void (*callback)(struct ctdb_context *, int, void *),
+			       void *private_data,
+			       bool from_user,
+			       enum ctdb_eventscript_call call,
+			       const char *fmt, ...) PRINTF_ATTRIBUTE(7,8);
+void ctdb_release_all_ips(struct ctdb_context *ctdb);
+
+void set_nonblocking(int fd);
+void set_close_on_exec(int fd);
+
+bool ctdb_recovery_lock(struct ctdb_context *ctdb, bool keep);
+
+int ctdb_set_recovery_lock_file(struct ctdb_context *ctdb, const char *file);
+
+int32_t ctdb_control_get_tunable(struct ctdb_context *ctdb, TDB_DATA indata, 
+				 TDB_DATA *outdata);
+int32_t ctdb_control_set_tunable(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_list_tunables(struct ctdb_context *ctdb, TDB_DATA *outdata);
+int32_t ctdb_control_try_delete_records(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DATA *outdata);
+int32_t ctdb_control_add_public_address(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_del_public_address(struct ctdb_context *ctdb, TDB_DATA indata);
+
+void ctdb_tunables_set_defaults(struct ctdb_context *ctdb);
+
+int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata);
+
+int ctdb_ctrl_get_all_tunables(struct ctdb_context *ctdb, 
+			       struct timeval timeout, 
+			       uint32_t destnode,
+			       struct ctdb_tunable *tunables);
+
+int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority);
+
+bool parse_ip_mask(const char *s, const char *iface, ctdb_sock_addr *addr, unsigned *mask);
+bool parse_ip_port(const char *s, ctdb_sock_addr *addr);
+bool parse_ip(const char *s, const char *iface, unsigned port, ctdb_sock_addr *addr);
+bool parse_ipv4(const char *s, unsigned port, struct sockaddr_in *sin);
+ 
+
+int ctdb_sys_open_capture_socket(const char *iface, void **private_data);
+int ctdb_sys_close_capture_socket(void *private_data);
+int ctdb_sys_read_tcp_packet(int s, void *private_data, ctdb_sock_addr *src, ctdb_sock_addr *dst, uint32_t *ack_seq, uint32_t *seq);
+
+int ctdb_ctrl_killtcp(struct ctdb_context *ctdb, 
+		      struct timeval timeout, 
+		      uint32_t destnode,
+		      struct ctdb_control_killtcp *killtcp);
+
+int ctdb_ctrl_add_public_ip(struct ctdb_context *ctdb, 
+		      struct timeval timeout, 
+		      uint32_t destnode,
+		      struct ctdb_control_ip_iface *pub);
+
+int ctdb_ctrl_del_public_ip(struct ctdb_context *ctdb, 
+		      struct timeval timeout, 
+		      uint32_t destnode,
+		      struct ctdb_control_ip_iface *pub);
+
+int ctdb_ctrl_gratious_arp(struct ctdb_context *ctdb, 
+		      struct timeval timeout, 
+		      uint32_t destnode,
+		      ctdb_sock_addr *addr,
+		      const char *ifname);
+
+int ctdb_ctrl_get_tcp_tickles(struct ctdb_context *ctdb, 
+		      struct timeval timeout, 
+		      uint32_t destnode,
+		      TALLOC_CTX *mem_ctx,
+		      ctdb_sock_addr *addr,
+		      struct ctdb_control_tcp_tickle_list **list);
+
+
+int32_t ctdb_control_register_server_id(struct ctdb_context *ctdb, 
+		      uint32_t client_id,
+		      TDB_DATA indata);
+int32_t ctdb_control_check_server_id(struct ctdb_context *ctdb, 
+		      TDB_DATA indata);
+int32_t ctdb_control_unregister_server_id(struct ctdb_context *ctdb, 
+		      TDB_DATA indata);
+int32_t ctdb_control_get_server_id_list(struct ctdb_context *ctdb, 
+		      TDB_DATA *outdata);
+int32_t ctdb_control_uptime(struct ctdb_context *ctdb, 
+		      TDB_DATA *outdata);
+
+int ctdb_attach_databases(struct ctdb_context *ctdb);
+
+int32_t ctdb_control_persistent_store(struct ctdb_context *ctdb, 
+				      struct ctdb_req_control *c, 
+				      TDB_DATA recdata, bool *async_reply);
+int32_t ctdb_control_update_record(struct ctdb_context *ctdb, 
+				   struct ctdb_req_control *c, TDB_DATA recdata, 
+				   bool *async_reply);
+int32_t ctdb_control_trans2_commit(struct ctdb_context *ctdb, 
+				   struct ctdb_req_control *c, 
+				   TDB_DATA recdata, bool *async_reply);
+
+int32_t ctdb_control_trans3_commit(struct ctdb_context *ctdb,
+				   struct ctdb_req_control *c,
+				   TDB_DATA recdata, bool *async_reply);
+
+void ctdb_persistent_finish_trans3_commits(struct ctdb_context *ctdb);
+
+int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id);
+int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id);
+int32_t ctdb_control_transaction_cancel(struct ctdb_context *ctdb);
+int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_db_set_healthy(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_db_get_health(struct ctdb_context *ctdb,
+				   TDB_DATA indata,
+				   TDB_DATA *outdata);
+
+
+int ctdb_vacuum(struct ctdb_context *ctdb, int argc, const char **argv);
+int ctdb_repack(struct ctdb_context *ctdb, int argc, const char **argv);
+
+void ctdb_block_signal(int signum);
+void ctdb_unblock_signal(int signum);
+int32_t ctdb_monitoring_mode(struct ctdb_context *ctdb);
+int ctdb_set_child_logging(struct ctdb_context *ctdb);
+void ctdb_lockdown_memory(struct ctdb_context *ctdb);
+
+typedef void (*client_async_callback)(struct ctdb_context *ctdb, uint32_t node_pnn, int32_t res, TDB_DATA outdata, void *callback_data);
+
+struct client_async_data {
+	enum ctdb_controls opcode;
+	bool dont_log_errors;
+	uint32_t count;
+	uint32_t fail_count;
+	client_async_callback callback;
+	client_async_callback fail_callback;
+	void *callback_data;
+};
+void ctdb_client_async_add(struct client_async_data *data, struct ctdb_client_control_state *state);
+int ctdb_client_async_wait(struct ctdb_context *ctdb, struct client_async_data *data);
+int ctdb_client_async_control(struct ctdb_context *ctdb,
+				enum ctdb_controls opcode,
+				uint32_t *nodes,
+			      	uint64_t srvid,
+				struct timeval timeout,
+				bool dont_log_errors,
+				TDB_DATA data,
+			      	client_async_callback client_callback,
+			        client_async_callback fail_callback,
+				void *callback_data);
+
+void ctdb_load_nodes_file(struct ctdb_context *ctdb);
+
+int ctdb_control_reload_nodes_file(struct ctdb_context *ctdb, uint32_t opcode);
+
+int32_t ctdb_dump_memory(struct ctdb_context *ctdb, TDB_DATA *outdata);
+int32_t ctdb_control_get_capabilities(struct ctdb_context *ctdb, TDB_DATA *outdata);
+
+int32_t ctdb_control_trans2_finished(struct ctdb_context *ctdb, 
+				     struct ctdb_req_control *c);
+int32_t ctdb_control_trans2_error(struct ctdb_context *ctdb, 
+				  struct ctdb_req_control *c);
+int32_t ctdb_control_trans2_active(struct ctdb_context *ctdb,
+				   struct ctdb_req_control *c,
+				   uint32_t db_id);
+
+char *ctdb_addr_to_str(ctdb_sock_addr *addr);
+unsigned ctdb_addr_to_port(ctdb_sock_addr *addr);
+void ctdb_canonicalize_ip(const ctdb_sock_addr *ip, ctdb_sock_addr *cip);
+
+int32_t ctdb_control_recd_ping(struct ctdb_context *ctdb);
+int32_t ctdb_control_set_recmaster(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata);
+
+extern int script_log_level;
+extern bool fast_start;
+
+int32_t ctdb_control_get_event_script_status(struct ctdb_context *ctdb,
+					     uint32_t call_type,
+					     TDB_DATA *outdata);
+
+int ctdb_log_event_script_output(struct ctdb_context *ctdb, char *str, uint16_t len);
+int ctdb_ctrl_report_recd_lock_latency(struct ctdb_context *ctdb, struct timeval timeout, double latency);
+
+int32_t ctdb_control_stop_node(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply);
+int32_t ctdb_control_continue_node(struct ctdb_context *ctdb);
+
+void ctdb_stop_vacuuming(struct ctdb_context *ctdb);
+int ctdb_vacuum_init(struct ctdb_db_context *ctdb_db);
+
+int32_t ctdb_control_enable_script(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_disable_script(struct ctdb_context *ctdb, TDB_DATA indata);
+
+int32_t ctdb_local_node_got_banned(struct ctdb_context *ctdb);
+int32_t ctdb_control_set_ban_state(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_get_ban_state(struct ctdb_context *ctdb, TDB_DATA *outdata);
+int32_t ctdb_control_set_db_priority(struct ctdb_context *ctdb, TDB_DATA indata);
+void ctdb_ban_self(struct ctdb_context *ctdb);
+
+int32_t ctdb_control_register_notify(struct ctdb_context *ctdb, uint32_t client_id, TDB_DATA indata);
+
+int32_t ctdb_control_deregister_notify(struct ctdb_context *ctdb, uint32_t client_id, TDB_DATA indata);
+
+int start_syslog_daemon(struct ctdb_context *ctdb);
+
+/* Where to send the log messages back to */
+struct ctdb_get_log_addr {
+	uint32_t pnn;
+	uint64_t srvid;
+	int32_t level;
+};
+
+extern int log_ringbuf_size;
+
+int32_t ctdb_control_get_log(struct ctdb_context *ctdb, TDB_DATA addr);
+int32_t ctdb_control_clear_log(struct ctdb_context *ctdb);
+
+
+struct ctdb_log_state *ctdb_fork_with_logging(TALLOC_CTX *mem_ctx,
+					      struct ctdb_context *ctdb,
+					      const char *log_prefix,
+					      void (*logfn)(const char *, uint16_t, void *),
+					      void *logfn_private, pid_t *pid);
+
+int32_t ctdb_control_process_exists(struct ctdb_context *ctdb, pid_t pid);
+struct ctdb_client *ctdb_find_client_by_pid(struct ctdb_context *ctdb, pid_t pid);
+
+int32_t ctdb_control_get_db_seqnum(struct ctdb_context *ctdb,
+				   TDB_DATA indata,
+				   TDB_DATA *outdata);
+
+int ctdb_load_persistent_health(struct ctdb_context *ctdb,
+				struct ctdb_db_context *ctdb_db);
+int ctdb_update_persistent_health(struct ctdb_context *ctdb,
+				  struct ctdb_db_context *ctdb_db,
+				  const char *reason,/* NULL means healthy */
+				  int num_healthy_nodes);
+int ctdb_recheck_persistent_health(struct ctdb_context *ctdb);
+
+void ctdb_run_notification_script(struct ctdb_context *ctdb, const char *event);
+
+void ctdb_fault_setup(void);
+
+int verify_remote_ip_allocation(struct ctdb_context *ctdb, 
+				struct ctdb_all_public_ips *ips);
+int update_ip_assignment_tree(struct ctdb_context *ctdb,
+				struct ctdb_public_ip *ip);
+
+int ctdb_init_tevent_logging(struct ctdb_context *ctdb);
+
+int ctdb_statistics_init(struct ctdb_context *ctdb);
+
+int32_t ctdb_control_get_stat_history(struct ctdb_context *ctdb,
+				      struct ctdb_req_control *c,
+				      TDB_DATA *outdata);
+
+int ctdb_deferred_drop_all_ips(struct ctdb_context *ctdb);
+
+int ctdb_process_deferred_attach(struct ctdb_context *ctdb);
+
+/**
+ * structure to pass to a schedule_for_deletion_control
+ */
+struct ctdb_control_schedule_for_deletion {
+	uint32_t db_id;
+	struct ctdb_ltdb_header hdr;
+	uint32_t keylen;
+	uint8_t key[1]; /* key[] */
+};
+
+int32_t ctdb_control_schedule_for_deletion(struct ctdb_context *ctdb,
+					   TDB_DATA indata);
+
+
+int32_t ctdb_local_schedule_for_deletion(struct ctdb_db_context *ctdb_db,
+					 const struct ctdb_ltdb_header *hdr,
+					 TDB_DATA key);
+
+void ctdb_local_remove_from_delete_queue(struct ctdb_db_context *ctdb_db,
+					 const struct ctdb_ltdb_header *hdr,
+					 const TDB_DATA key);
+
+struct ctdb_ltdb_header *ctdb_header_from_record_handle(struct ctdb_record_handle *h);
+
+/* For unit testing ctdb_transaction.c. */
+struct ctdb_public_ip_list {
+	struct ctdb_public_ip_list *next;
+	uint32_t pnn;
+	ctdb_sock_addr addr;
+};
+int ctdb_trackingdb_add_pnn(struct ctdb_context *ctdb, TDB_DATA *data, uint32_t pnn);
+
+typedef void (*ctdb_trackingdb_cb)(struct ctdb_context *ctdb, uint32_t pnn, void *private_data);
+
+void ctdb_trackingdb_traverse(struct ctdb_context *ctdb, TDB_DATA data, ctdb_trackingdb_cb cb, void *private_data);
+
+int ctdb_start_revoke_ro_record(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_ltdb_header *header, TDB_DATA data);
+
+typedef void (*deferred_requeue_fn)(void *call_context, struct ctdb_req_header *hdr);
+
+int ctdb_add_revoke_deferred_call(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db, TDB_DATA key, struct ctdb_req_header *hdr, deferred_requeue_fn fn, void *call_context);
+
+int ctdb_set_db_readonly(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db);
+
+int ctdb_null_func(struct ctdb_call_info *call);
+
+int ctdb_fetch_func(struct ctdb_call_info *call);
+
+int ctdb_fetch_with_header_func(struct ctdb_call_info *call);
+
+int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
+				uint32_t db_id,
+				TDB_DATA *outdata);
+
+int ctdb_set_db_sticky(struct ctdb_context *ctdb, struct ctdb_db_context *ctdb_db);
+
+/*
+  description for a message to reload all ips via recovery master/daemon
+ */
+struct reloadips_all_reply {
+	uint32_t pnn;
+	uint64_t srvid;
+};
+
+int32_t ctdb_control_reload_public_ips(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply);
+
+int ctdb_start_monitoring_interfaces(struct ctdb_context *ctdb);
+
+#endif
diff --git a/ctdb/include/ctdb_protocol.h b/ctdb/include/ctdb_protocol.h
new file mode 100644
index 0000000000..33187c7842
--- /dev/null
+++ b/ctdb/include/ctdb_protocol.h
@@ -0,0 +1,741 @@
+/*
+   ctdb database library
+
+   Copyright (C) Andrew Tridgell  2006
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef _CTDB_PROTOCOL_H
+#define _CTDB_PROTOCOL_H
+
+/* location of daemon socket */
+#define CTDB_PATH	"/tmp/ctdb.socket"
+
+/* default ctdb port number */
+#define CTDB_PORT 4379
+
+/* we must align packets to ensure ctdb works on all architectures (eg. sparc) */
+#define CTDB_DS_ALIGNMENT 8
+
+
+#define CTDB_NULL_FUNC                  0xFF000001
+#define CTDB_FETCH_FUNC                 0xFF000002
+#define CTDB_FETCH_WITH_HEADER_FUNC     0xFF000003
+
+
+struct ctdb_call {
+	int call_id;
+	TDB_DATA key;
+	TDB_DATA call_data;
+	TDB_DATA reply_data;
+	uint32_t status;
+#define CTDB_IMMEDIATE_MIGRATION		0x00000001
+#define CTDB_CALL_FLAG_VACUUM_MIGRATION		0x00000002
+#define CTDB_WANT_READONLY			0x00000004
+	uint32_t flags;
+};
+
+/*
+  structure passed to a ctdb call backend function
+*/
+struct ctdb_call_info {
+	TDB_DATA key;          /* record key */
+	struct ctdb_ltdb_header *header;
+	TDB_DATA record_data;  /* current data in the record */
+	TDB_DATA *new_data;    /* optionally updated record data */
+	TDB_DATA *call_data;   /* optionally passed from caller */
+	TDB_DATA *reply_data;  /* optionally returned by function */
+	uint32_t status;       /* optional reply status - defaults to zero */
+};
+
+#define CTDB_ERR_INVALID 1
+#define CTDB_ERR_NOMEM 2
+
+/*
+  ctdb flags
+*/
+#define CTDB_FLAG_TORTURE      (1<<1)
+
+/*
+   a message handler ID meaning "give me all messages"
+ */
+#define CTDB_SRVID_ALL (~(uint64_t)0)
+
+/*
+  srvid type : RECOVERY
+*/
+#define CTDB_SRVID_RECOVERY	0xF100000000000000LL
+
+/*
+   a message handler ID meaning that the cluster has been reconfigured
+ */
+#define CTDB_SRVID_RECONFIGURE 0xF200000000000000LL
+
+/*
+   a message handler ID meaning that an IP address has been released
+ */
+#define CTDB_SRVID_RELEASE_IP 0xF300000000000000LL
+
+/*
+   a message handler ID meaning that an IP address has been taken
+ */
+#define CTDB_SRVID_TAKE_IP 0xF301000000000000LL
+
+/*
+   a message ID to set the node flags in the recovery daemon
+ */
+#define CTDB_SRVID_SET_NODE_FLAGS 0xF400000000000000LL
+
+/*
+   a message ID to ask the recovery daemon to update the expected node
+   assignment for a public ip
+ */
+#define CTDB_SRVID_RECD_UPDATE_IP 0xF500000000000000LL
+
+/*
+  a message to tell the recovery daemon to fetch a set of records
+ */
+#define CTDB_SRVID_VACUUM_FETCH 0xF700000000000000LL
+
+/*
+  a message to tell the recovery daemon to write a talloc memdump
+  to the log
+ */
+#define CTDB_SRVID_MEM_DUMP 0xF800000000000000LL
+
+/*
+   a message ID to get the recovery daemon to push the node flags out
+ */
+#define CTDB_SRVID_PUSH_NODE_FLAGS 0xF900000000000000LL
+
+/*
+   a message ID to get the recovery daemon to reload the nodes file
+ */
+#define CTDB_SRVID_RELOAD_NODES 0xFA00000000000000LL
+
+/*
+   a message ID to get the recovery daemon to perform a takeover run
+ */
+#define CTDB_SRVID_TAKEOVER_RUN 0xFB00000000000000LL
+
+/* request recovery daemon to rebalance ips for a node.
+   input is uint32_t for the node id.
+*/
+#define CTDB_SRVID_REBALANCE_NODE 0xFB01000000000000LL
+
+/*
+   a message handler ID meaning to ask recovery master to reload all ips
+ */
+#define CTDB_SRVID_RELOAD_ALL_IPS 0xFB02000000000000LL
+
+/* A message id to ask the recovery daemon to temporarily disable the
+   public ip checks
+*/
+#define CTDB_SRVID_DISABLE_IP_CHECK  0xFC00000000000000LL
+
+/* A dummy port used for sending back ipreallocate resposnes to the main
+   daemon
+*/
+#define CTDB_SRVID_TAKEOVER_RUN_RESPONSE  0xFD00000000000000LL
+
+/* A range of ports reserved for registering a PID (top 8 bits)
+ * All ports matching the 8 top bits are reserved for exclusive use by
+ * registering a SRVID that matches the process-id of the requesting process
+ */
+#define CTDB_SRVID_PID_RANGE   0x0000000000000000LL
+
+/* A range of ports reserved for samba (top 8 bits)
+ * All ports matching the 8 top bits are reserved for exclusive use by
+ * CIFS server
+ */
+#define CTDB_SRVID_SAMBA_NOTIFY  0xFE00000000000000LL
+#define CTDB_SRVID_SAMBA_RANGE   0xFE00000000000000LL
+
+/* A range of ports reserved for a CTDB NFS server (top 8 bits)
+ * All ports matching the 8 top bits are reserved for exclusive use by
+ * NFS server
+ */
+#define CTDB_SRVID_NFSD_RANGE  0xEE00000000000000LL
+
+/* A range of ports reserved for a CTDB ISCSI server (top 8 bits)
+ * All ports matching the 8 top bits are reserved for exclusive use by
+ * ISCSI server
+ */
+#define CTDB_SRVID_ISCSID_RANGE  0xDE00000000000000LL
+
+/* A range of ports reserved for testing (top 8 bits)
+ * All ports matching the 8 top bits are reserved for exclusive use by
+ * test applications
+ */
+#define CTDB_SRVID_TEST_RANGE  0xCE00000000000000LL
+
+/* Range of ports reserved for traversals */
+#define CTDB_SRVID_TRAVERSE_RANGE  0xBE00000000000000LL
+
+/* used on the domain socket, send a pdu to the local daemon */
+#define CTDB_CURRENT_NODE     0xF0000001
+/* send a broadcast to all nodes in the cluster, active or not */
+#define CTDB_BROADCAST_ALL    0xF0000002
+/* send a broadcast to all nodes in the current vnn map */
+#define CTDB_BROADCAST_VNNMAP 0xF0000003
+/* send a broadcast to all connected nodes */
+#define CTDB_BROADCAST_CONNECTED 0xF0000004
+/* send a broadcast to selected connected nodes */
+#define CTDB_MULTICAST 0xF0000005
+
+/* the key used for transaction locking on persistent databases */
+#define CTDB_TRANSACTION_LOCK_KEY "__transaction_lock__"
+
+/* the key used to store persistent db sequence number */
+#define CTDB_DB_SEQNUM_KEY "__db_sequence_number__"
+
+#define MONITOR_SCRIPT_OK      0
+#define MONITOR_SCRIPT_TIMEOUT 1
+
+#define MAX_SCRIPT_NAME 31
+#define MAX_SCRIPT_OUTPUT 511
+struct ctdb_script_wire {
+	char name[MAX_SCRIPT_NAME+1];
+	struct timeval start;
+	struct timeval finished;
+	int32_t status;
+	char output[MAX_SCRIPT_OUTPUT+1];
+};
+
+struct ctdb_scripts_wire {
+	uint32_t num_scripts;
+	struct ctdb_script_wire scripts[1];
+};
+
+/* different calls to event scripts. */
+enum ctdb_eventscript_call {
+	CTDB_EVENT_INIT,		/* CTDB starting up: no args */
+	CTDB_EVENT_SETUP,		/* CTDB starting up after transport is readdy: no args. */
+	CTDB_EVENT_STARTUP,		/* CTDB starting up after initial recovery: no args. */
+	CTDB_EVENT_START_RECOVERY,	/* CTDB recovery starting: no args. */
+	CTDB_EVENT_RECOVERED,		/* CTDB recovery finished: no args. */
+	CTDB_EVENT_TAKE_IP,		/* IP taken: interface, IP address, netmask bits. */
+	CTDB_EVENT_RELEASE_IP,		/* IP released: interface, IP address, netmask bits. */
+	CTDB_EVENT_STOPPED,		/* This node is stopped: no args. */
+	CTDB_EVENT_MONITOR,		/* Please check if service is healthy: no args. */
+	CTDB_EVENT_STATUS,		/* Report service status: no args. */
+	CTDB_EVENT_SHUTDOWN,		/* CTDB shutting down: no args. */
+	CTDB_EVENT_RELOAD,		/* magic */
+	CTDB_EVENT_UPDATE_IP,		/* IP updating: old interface, new interface, IP address, netmask bits. */
+	CTDB_EVENT_IPREALLOCATED,	/* when a takeover_run() completes */
+	CTDB_EVENT_MAX
+};
+
+/* Mapping from enum to names. */
+extern const char *ctdb_eventscript_call_names[];
+
+/*
+  operation IDs
+*/
+enum ctdb_operation {
+	CTDB_REQ_CALL           = 0,
+	CTDB_REPLY_CALL         = 1,
+	CTDB_REQ_DMASTER        = 2,
+	CTDB_REPLY_DMASTER      = 3,
+	CTDB_REPLY_ERROR        = 4,
+	CTDB_REQ_MESSAGE        = 5,
+	/* #6 removed */
+	CTDB_REQ_CONTROL        = 7,
+	CTDB_REPLY_CONTROL      = 8,
+	CTDB_REQ_KEEPALIVE      = 9,
+};
+
+#define CTDB_MAGIC 0x43544442 /* CTDB */
+#define CTDB_VERSION 1
+
+enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0,
+		    CTDB_CONTROL_STATISTICS              = 1,
+		    /* #2 removed */
+		    CTDB_CONTROL_PING                    = 3,
+		    CTDB_CONTROL_GETDBPATH               = 4,
+		    CTDB_CONTROL_GETVNNMAP               = 5,
+		    CTDB_CONTROL_SETVNNMAP               = 6,
+		    CTDB_CONTROL_GET_DEBUG               = 7,
+		    CTDB_CONTROL_SET_DEBUG               = 8,
+		    CTDB_CONTROL_GET_DBMAP               = 9,
+		    CTDB_CONTROL_GET_NODEMAPv4           = 10, /* obsolete */
+		    CTDB_CONTROL_SET_DMASTER             = 11,
+		    /* #12 removed */
+		    CTDB_CONTROL_PULL_DB                 = 13,
+		    CTDB_CONTROL_PUSH_DB                 = 14,
+		    CTDB_CONTROL_GET_RECMODE             = 15,
+		    CTDB_CONTROL_SET_RECMODE             = 16,
+		    CTDB_CONTROL_STATISTICS_RESET        = 17,
+		    CTDB_CONTROL_DB_ATTACH               = 18,
+		    CTDB_CONTROL_SET_CALL                = 19,
+		    CTDB_CONTROL_TRAVERSE_START          = 20,
+		    CTDB_CONTROL_TRAVERSE_ALL            = 21,
+		    CTDB_CONTROL_TRAVERSE_DATA           = 22,
+		    CTDB_CONTROL_REGISTER_SRVID          = 23,
+		    CTDB_CONTROL_DEREGISTER_SRVID        = 24,
+		    CTDB_CONTROL_GET_DBNAME              = 25,
+		    CTDB_CONTROL_ENABLE_SEQNUM           = 26,
+		    CTDB_CONTROL_UPDATE_SEQNUM           = 27,
+		    /* #28 removed */
+		    CTDB_CONTROL_DUMP_MEMORY             = 29,
+		    CTDB_CONTROL_GET_PID                 = 30,
+		    CTDB_CONTROL_GET_RECMASTER           = 31,
+		    CTDB_CONTROL_SET_RECMASTER           = 32,
+		    CTDB_CONTROL_FREEZE                  = 33,
+		    CTDB_CONTROL_THAW                    = 34,
+		    CTDB_CONTROL_GET_PNN                 = 35,
+		    CTDB_CONTROL_SHUTDOWN                = 36,
+		    CTDB_CONTROL_GET_MONMODE             = 37,
+		    /* #38 removed */
+		    /* #39 removed */
+		    /* #40 removed */
+		    /* #41 removed */
+		    CTDB_CONTROL_TAKEOVER_IPv4           = 42, /* obsolete */
+		    CTDB_CONTROL_RELEASE_IPv4            = 43, /* obsolete */
+		    CTDB_CONTROL_TCP_CLIENT              = 44,
+		    CTDB_CONTROL_TCP_ADD                 = 45,
+		    CTDB_CONTROL_TCP_REMOVE              = 46,
+		    CTDB_CONTROL_STARTUP                 = 47,
+		    CTDB_CONTROL_SET_TUNABLE             = 48,
+		    CTDB_CONTROL_GET_TUNABLE             = 49,
+		    CTDB_CONTROL_LIST_TUNABLES           = 50,
+		    CTDB_CONTROL_GET_PUBLIC_IPSv4        = 51, /* obsolete */
+		    CTDB_CONTROL_MODIFY_FLAGS            = 52,
+		    CTDB_CONTROL_GET_ALL_TUNABLES        = 53,
+		    CTDB_CONTROL_KILL_TCP                = 54,
+		    CTDB_CONTROL_GET_TCP_TICKLE_LIST     = 55,
+		    CTDB_CONTROL_SET_TCP_TICKLE_LIST     = 56,
+		    CTDB_CONTROL_REGISTER_SERVER_ID	 = 57,
+		    CTDB_CONTROL_UNREGISTER_SERVER_ID	 = 58,
+		    CTDB_CONTROL_CHECK_SERVER_ID	 = 59,
+		    CTDB_CONTROL_GET_SERVER_ID_LIST	 = 60,
+		    CTDB_CONTROL_DB_ATTACH_PERSISTENT    = 61,
+		    CTDB_CONTROL_PERSISTENT_STORE        = 62,
+		    CTDB_CONTROL_UPDATE_RECORD           = 63,
+		    CTDB_CONTROL_SEND_GRATIOUS_ARP       = 64,
+		    CTDB_CONTROL_TRANSACTION_START       = 65,
+		    CTDB_CONTROL_TRANSACTION_COMMIT      = 66,
+		    CTDB_CONTROL_WIPE_DATABASE           = 67,
+		    /* #68 removed */
+		    CTDB_CONTROL_UPTIME                  = 69,
+		    CTDB_CONTROL_START_RECOVERY          = 70,
+		    CTDB_CONTROL_END_RECOVERY            = 71,
+		    CTDB_CONTROL_RELOAD_NODES_FILE       = 72,
+		    /* #73 removed */
+		    CTDB_CONTROL_TRY_DELETE_RECORDS      = 74,
+		    CTDB_CONTROL_ENABLE_MONITOR          = 75,
+		    CTDB_CONTROL_DISABLE_MONITOR         = 76,
+		    CTDB_CONTROL_ADD_PUBLIC_IP           = 77,
+		    CTDB_CONTROL_DEL_PUBLIC_IP           = 78,
+		    CTDB_CONTROL_RUN_EVENTSCRIPTS        = 79,
+		    CTDB_CONTROL_GET_CAPABILITIES	 = 80,
+		    CTDB_CONTROL_START_PERSISTENT_UPDATE = 81,
+		    CTDB_CONTROL_CANCEL_PERSISTENT_UPDATE= 82,
+		    CTDB_CONTROL_TRANS2_COMMIT           = 83,
+		    CTDB_CONTROL_TRANS2_FINISHED         = 84,
+		    CTDB_CONTROL_TRANS2_ERROR            = 85,
+		    CTDB_CONTROL_TRANS2_COMMIT_RETRY     = 86,
+		    CTDB_CONTROL_RECD_PING		 = 87,
+		    CTDB_CONTROL_RELEASE_IP              = 88,
+		    CTDB_CONTROL_TAKEOVER_IP             = 89,
+		    CTDB_CONTROL_GET_PUBLIC_IPS          = 90,
+		    CTDB_CONTROL_GET_NODEMAP             = 91,
+		    CTDB_CONTROL_GET_EVENT_SCRIPT_STATUS = 96,
+		    CTDB_CONTROL_TRAVERSE_KILL		 = 97,
+		    CTDB_CONTROL_RECD_RECLOCK_LATENCY    = 98,
+		    CTDB_CONTROL_GET_RECLOCK_FILE        = 99,
+		    CTDB_CONTROL_SET_RECLOCK_FILE        = 100,
+		    CTDB_CONTROL_STOP_NODE               = 101,
+		    CTDB_CONTROL_CONTINUE_NODE           = 102,
+		    CTDB_CONTROL_SET_NATGWSTATE          = 103,
+		    CTDB_CONTROL_SET_LMASTERROLE         = 104,
+		    CTDB_CONTROL_SET_RECMASTERROLE       = 105,
+		    CTDB_CONTROL_ENABLE_SCRIPT           = 107,
+		    CTDB_CONTROL_DISABLE_SCRIPT          = 108,
+		    CTDB_CONTROL_SET_BAN_STATE           = 109,
+		    CTDB_CONTROL_GET_BAN_STATE           = 110,
+		    CTDB_CONTROL_SET_DB_PRIORITY         = 111,
+		    CTDB_CONTROL_GET_DB_PRIORITY         = 112,
+		    CTDB_CONTROL_TRANSACTION_CANCEL      = 113,
+		    CTDB_CONTROL_REGISTER_NOTIFY         = 114,
+		    CTDB_CONTROL_DEREGISTER_NOTIFY       = 115,
+		    CTDB_CONTROL_TRANS2_ACTIVE           = 116,
+		    CTDB_CONTROL_GET_LOG		 = 117,
+		    CTDB_CONTROL_CLEAR_LOG		 = 118,
+		    CTDB_CONTROL_TRANS3_COMMIT           = 119,
+		    CTDB_CONTROL_GET_DB_SEQNUM           = 120,
+		    CTDB_CONTROL_DB_SET_HEALTHY		 = 121,
+		    CTDB_CONTROL_DB_GET_HEALTH		 = 122,
+		    CTDB_CONTROL_GET_PUBLIC_IP_INFO	 = 123,
+		    CTDB_CONTROL_GET_IFACES		 = 124,
+		    CTDB_CONTROL_SET_IFACE_LINK_STATE	 = 125,
+		    CTDB_CONTROL_TCP_ADD_DELAYED_UPDATE  = 126,
+		    CTDB_CONTROL_GET_STAT_HISTORY	 = 127,
+		    CTDB_CONTROL_SCHEDULE_FOR_DELETION   = 128,
+		    CTDB_CONTROL_SET_DB_READONLY	 = 129,
+		    CTDB_CONTROL_CHECK_SRVIDS		 = 130,
+		    CTDB_CONTROL_TRAVERSE_START_EXT	 = 131,
+		    CTDB_CONTROL_GET_DB_STATISTICS	 = 132,
+		    CTDB_CONTROL_SET_DB_STICKY		 = 133,
+		    CTDB_CONTROL_RELOAD_PUBLIC_IPS	 = 134,
+};
+
+/*
+  packet structures
+*/
+struct ctdb_req_header {
+	uint32_t length;
+	uint32_t ctdb_magic;
+	uint32_t ctdb_version;
+	uint32_t generation;
+	uint32_t operation;
+	uint32_t destnode;
+	uint32_t srcnode;
+	uint32_t reqid;
+};
+
+struct ctdb_req_call {
+	struct ctdb_req_header hdr;
+	uint32_t flags;
+	uint32_t db_id;
+	uint32_t callid;
+	uint32_t hopcount;
+	uint32_t keylen;
+	uint32_t calldatalen;
+	uint8_t data[1]; /* key[] followed by calldata[] */
+};
+
+struct ctdb_reply_call {
+	struct ctdb_req_header hdr;
+	uint32_t status;
+	uint32_t datalen;
+	uint8_t  data[1];
+};
+
+struct ctdb_reply_error {
+	struct ctdb_req_header hdr;
+	uint32_t status;
+	uint32_t msglen;
+	uint8_t  msg[1];
+};
+
+struct ctdb_req_dmaster {
+	struct ctdb_req_header hdr;
+	uint32_t db_id;
+	uint64_t rsn;
+	uint32_t dmaster;
+	uint32_t keylen;
+	uint32_t datalen;
+	uint8_t  data[1];
+};
+
+struct ctdb_reply_dmaster {
+	struct ctdb_req_header hdr;
+	uint32_t db_id;
+	uint64_t rsn;
+	uint32_t keylen;
+	uint32_t datalen;
+	uint8_t  data[1];
+};
+
+struct ctdb_req_message {
+	struct ctdb_req_header hdr;
+	uint64_t srvid;
+	uint32_t datalen;
+	uint8_t data[1];
+};
+
+struct ctdb_req_getdbpath {
+	struct ctdb_req_header hdr;
+	uint32_t db_id;
+};
+
+struct ctdb_reply_getdbpath {
+	struct ctdb_req_header hdr;
+	uint32_t datalen;
+	uint8_t data[1];
+};
+
+struct ctdb_req_control {
+	struct ctdb_req_header hdr;
+	uint32_t opcode;
+	uint32_t pad;
+	uint64_t srvid;
+	uint32_t client_id;
+#define CTDB_CTRL_FLAG_NOREPLY   1
+#define CTDB_CTRL_FLAG_OPCODE_SPECIFIC   0xFFFF0000
+	uint32_t flags;
+	uint32_t datalen;
+	uint8_t data[1];
+};
+
+struct ctdb_reply_control {
+	struct ctdb_req_header hdr;
+	int32_t  status;
+	uint32_t datalen;
+	uint32_t errorlen;
+	uint8_t data[1];
+};
+
+struct ctdb_req_keepalive {
+	struct ctdb_req_header hdr;
+};
+
+
+/* types of failures possible from TRANS2_COMMIT */
+enum ctdb_trans2_commit_error {
+	CTDB_TRANS2_COMMIT_SUCCESS=0, /* all nodes committed successfully */
+	CTDB_TRANS2_COMMIT_TIMEOUT=1, /* at least one node timed out */
+	CTDB_TRANS2_COMMIT_ALLFAIL=2, /* all nodes failed the commit */
+	CTDB_TRANS2_COMMIT_SOMEFAIL=3 /* some nodes failed the commit, some allowed it */
+};
+
+/*
+  the extended header for records in the ltdb
+*/
+struct ctdb_ltdb_header {
+	uint64_t rsn;
+	uint32_t dmaster;
+	uint16_t laccessor;
+	uint16_t lacount;
+#define CTDB_REC_FLAG_DEFAULT			0x00000000
+#define CTDB_REC_FLAG_MIGRATED_WITH_DATA	0x00010000
+#define CTDB_REC_FLAG_VACUUM_MIGRATED		0x00020000
+#define CTDB_REC_FLAG_AUTOMATIC			0x00040000
+#define CTDB_REC_RO_HAVE_DELEGATIONS		0x01000000
+#define CTDB_REC_RO_HAVE_READONLY		0x02000000
+#define CTDB_REC_RO_REVOKING_READONLY		0x04000000
+#define CTDB_REC_RO_REVOKE_COMPLETE		0x08000000
+	uint32_t flags;
+};
+
+
+/*
+  definitions for different socket structures
+ */
+typedef struct sockaddr_in ctdb_addr_in;
+typedef struct sockaddr_in6 ctdb_addr_in6;
+typedef union {
+	struct sockaddr sa;
+	ctdb_addr_in	ip;
+	ctdb_addr_in6	ip6;
+} ctdb_sock_addr;
+
+/*
+   A structure describing a single node, its flags and its address
+*/
+struct ctdb_node_and_flags {
+	uint32_t pnn;
+	uint32_t flags;
+	ctdb_sock_addr addr;
+};
+
+
+/*
+   Structure used for a nodemap. 
+   The nodemap is the structure containing a list of all nodes
+   known to the cluster and their associated flags.
+*/
+struct ctdb_node_map {
+	uint32_t num;
+	struct ctdb_node_and_flags nodes[1];
+};
+
+/*
+ * Node flags
+ */
+#define NODE_FLAGS_DISCONNECTED		0x00000001 /* node isn't connected */
+#define NODE_FLAGS_UNHEALTHY  		0x00000002 /* monitoring says node is unhealthy */
+#define NODE_FLAGS_PERMANENTLY_DISABLED	0x00000004 /* administrator has disabled node */
+#define NODE_FLAGS_BANNED		0x00000008 /* recovery daemon has banned the node */
+#define NODE_FLAGS_DELETED		0x00000010 /* this node has been deleted */
+#define NODE_FLAGS_STOPPED		0x00000020 /* this node has been stopped */
+#define NODE_FLAGS_DISABLED		(NODE_FLAGS_UNHEALTHY|NODE_FLAGS_PERMANENTLY_DISABLED)
+#define NODE_FLAGS_INACTIVE		(NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED|NODE_FLAGS_BANNED|NODE_FLAGS_STOPPED)
+
+#define NODE_FLAGS_NOIPTAKEOVER		0x01000000 /* this node can takeover any new ip addresses, this flag is ONLY valid within the recovery daemon */
+
+
+struct ctdb_public_ip {
+	uint32_t pnn;
+	ctdb_sock_addr addr;
+};
+
+struct ctdb_all_public_ips {
+	uint32_t num;
+	struct ctdb_public_ip ips[1];
+};
+
+
+struct latency_counter {
+	int num;
+	double min;
+	double max;
+	double total;
+};
+
+/*
+  structure used to pass record data between the child and parent
+ */
+struct ctdb_rec_data {
+	uint32_t length;
+	uint32_t reqid;
+	uint32_t keylen;
+	uint32_t datalen;
+	uint8_t  data[1];
+};
+
+struct ctdb_traverse_start {
+	uint32_t db_id;
+	uint32_t reqid;
+	uint64_t srvid;
+};
+
+struct ctdb_traverse_start_ext {
+	uint32_t db_id;
+	uint32_t reqid;
+	uint64_t srvid;
+	bool withemptyrecords;
+};
+
+/*
+  ctdb statistics information
+ */
+#define MAX_COUNT_BUCKETS 16
+#define MAX_HOT_KEYS      10
+
+struct ctdb_statistics {
+	uint32_t num_clients;
+	uint32_t frozen;
+	uint32_t recovering;
+	uint32_t client_packets_sent;
+	uint32_t client_packets_recv;
+	uint32_t node_packets_sent;
+	uint32_t node_packets_recv;
+	uint32_t keepalive_packets_sent;
+	uint32_t keepalive_packets_recv;
+	struct {
+		uint32_t req_call;
+		uint32_t reply_call;
+		uint32_t req_dmaster;
+		uint32_t reply_dmaster;
+		uint32_t reply_error;
+		uint32_t req_message;
+		uint32_t req_control;
+		uint32_t reply_control;
+	} node;
+	struct {
+		uint32_t req_call;
+		uint32_t req_message;
+		uint32_t req_control;
+	} client;
+	struct {
+		uint32_t call;
+		uint32_t control;
+		uint32_t traverse;
+	} timeouts;
+	struct {
+		struct latency_counter ctdbd;
+		struct latency_counter recd;
+	} reclock;
+	uint32_t total_calls;
+	uint32_t pending_calls;
+	uint32_t lockwait_calls;
+	uint32_t pending_lockwait_calls;
+	uint32_t childwrite_calls;
+	uint32_t pending_childwrite_calls;
+	uint32_t memory_used;
+	uint32_t __last_counter; /* hack for control_statistics_all */
+	uint32_t max_hop_count;
+	uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+	struct latency_counter call_latency;
+	struct latency_counter lockwait_latency;
+	struct latency_counter childwrite_latency;
+	uint32_t num_recoveries;
+	struct timeval statistics_start_time;
+	struct timeval statistics_current_time;
+	uint32_t total_ro_delegations;
+	uint32_t total_ro_revokes;
+};
+
+/*
+ * wire format for statistics history
+ */
+struct ctdb_statistics_wire {
+	uint32_t num;
+	struct ctdb_statistics stats[1];
+};
+
+/*
+ * db statistics
+ */
+struct ctdb_db_hot_key {
+	uint32_t count;
+	TDB_DATA key;
+};
+struct ctdb_db_statistics {
+	uint32_t db_ro_delegations;
+	uint32_t db_ro_revokes;
+	uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+	uint32_t num_hot_keys;
+	struct ctdb_db_hot_key hot_keys[MAX_HOT_KEYS];
+};
+struct ctdb_db_statistics_wire {
+	uint32_t db_ro_delegations;
+	uint32_t db_ro_revokes;
+	uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
+	uint32_t num_hot_keys;
+	char hot_keys[1];
+};
+
+/*
+ * wire format for interface list
+ */
+#ifdef IFNAMSIZ
+#define CTDB_IFACE_SIZE IFNAMSIZ
+#else
+#define CTDB_IFACE_SIZE 16
+#endif
+
+struct ctdb_iface_info {
+	char name[CTDB_IFACE_SIZE+2];
+	uint16_t link_state;
+	uint32_t references;
+};
+
+struct ctdb_ifaces_list {
+	uint32_t num;
+	struct ctdb_iface_info ifaces[1];
+};
+
+#define INVALID_GENERATION 1
+/* table that contains the mapping between a hash value and lmaster
+ */
+struct ctdb_vnn_map {
+	uint32_t generation;
+	uint32_t size;
+	uint32_t *map;
+};
+
+/* 
+   a wire representation of the vnn map
+ */
+struct ctdb_vnn_map_wire {
+	uint32_t generation;
+	uint32_t size;
+	uint32_t map[1];
+};
+
+#endif
diff --git a/ctdb/include/ctdb_typesafe_cb.h b/ctdb/include/ctdb_typesafe_cb.h
new file mode 100644
index 0000000000..b1f2c5f5b5
--- /dev/null
+++ b/ctdb/include/ctdb_typesafe_cb.h
@@ -0,0 +1,177 @@
+#ifndef CCAN_CAST_IF_TYPE_H
+#define CCAN_CAST_IF_TYPE_H
+
+#if (__GNUC__ >= 3)
+#define HAVE_TYPEOF 1
+#define HAVE_BUILTIN_CHOOSE_EXPR 1
+#define HAVE_BUILTIN_TYPES_COMPATIBLE_P 1
+#endif
+
+#if HAVE_TYPEOF && HAVE_BUILTIN_CHOOSE_EXPR && HAVE_BUILTIN_TYPES_COMPATIBLE_P
+/**
+ * cast_if_type - only cast an expression if test matches a given type
+ * @desttype: the type to cast to
+ * @expr: the expression to cast
+ * @test: the expression to test
+ * @oktype: the type we allow
+ *
+ * This macro is used to create functions which allow multiple types.
+ * The result of this macro is used somewhere that a @desttype type is
+ * expected: if @expr was of type @oktype, it will be cast to
+ * @desttype type.  As a result, if @expr is any type other than
+ * @oktype or @desttype, a compiler warning will be issued.
+ *
+ * This macro can be used in static initializers.
+ *
+ * This is merely useful for warnings: if the compiler does not
+ * support the primitives required for cast_if_type(), it becomes an
+ * unconditional cast, and the @test and @oktype argument is not used.  In
+ * particular, this means that @oktype can be a type which uses
+ * the "typeof": it will not be evaluated if typeof is not supported.
+ *
+ * Example:
+ *	// We can take either an unsigned long or a void *.
+ *	void _set_some_value(void *val);
+ *	#define set_some_value(e)			\
+ *		_set_some_value(cast_if_type(void *, (e), (e), unsigned long))
+ */
+#define cast_if_type(desttype, expr, test, oktype)			\
+__builtin_choose_expr(__builtin_types_compatible_p(typeof(1?(test):0), oktype), \
+			(desttype)(expr), (expr))
+#else
+#define cast_if_type(desttype, expr, test, oktype) ((desttype)(expr))
+#endif
+
+/**
+ * cast_if_any - only cast an expression if it is one of the three given types
+ * @desttype: the type to cast to
+ * @expr: the expression to cast
+ * @test: the expression to test
+ * @ok1: the first type we allow
+ * @ok2: the second type we allow
+ * @ok3: the third type we allow
+ *
+ * This is a convenient wrapper for multiple cast_if_type() calls.  You can
+ * chain them inside each other (ie. use cast_if_any() for expr) if you need
+ * more than 3 arguments.
+ *
+ * Example:
+ *	// We can take either a long, unsigned long, void * or a const void *.
+ *	void _set_some_value(void *val);
+ *	#define set_some_value(expr)					\
+ *		_set_some_value(cast_if_any(void *, (expr), (expr),	\
+ *					    long, unsigned long, const void *))
+ */
+#define cast_if_any(desttype, expr, test, ok1, ok2, ok3)		\
+	cast_if_type(desttype,						\
+		     cast_if_type(desttype,				\
+				  cast_if_type(desttype, (expr), (test), ok1), \
+				  ok2),					\
+		     ok3)
+
+/**
+ * typesafe_cb - cast a callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * If a callback function takes a single argument, this macro does
+ * appropriate casts to a function which takes a single void * argument if the
+ * callback provided matches the @arg (or a const or volatile version).
+ *
+ * It is assumed that @arg is of pointer type: usually @arg is passed
+ * or assigned to a void * elsewhere anyway.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(void *arg), void *arg);
+ *	#define register_callback(fn, arg) \
+ *		_register_callback(typesafe_cb(void, (fn), (arg)), (arg))
+ */
+#define typesafe_cb(rtype, fn, arg)			\
+	cast_if_type(rtype (*)(void *), (fn), (fn)(arg), rtype)
+
+/**
+ * typesafe_cb_const - cast a const callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * If a callback function takes a single argument, this macro does appropriate
+ * casts to a function which takes a single const void * argument if the
+ * callback provided matches the @arg.
+ *
+ * It is assumed that @arg is of pointer type: usually @arg is passed
+ * or assigned to a void * elsewhere anyway.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(const void *arg), const void *arg);
+ *	#define register_callback(fn, arg) \
+ *		_register_callback(typesafe_cb_const(void, (fn), (arg)), (arg))
+ */
+#define typesafe_cb_const(rtype, fn, arg)				\
+	sizeof((fn)((const void *)0)),					\
+		cast_if_type(rtype (*)(const void *),			\
+			     (fn), (fn)(arg), rtype (*)(typeof(arg)))
+
+/**
+ * typesafe_cb_preargs - cast a callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * This is a version of typesafe_cb() for callbacks that take other arguments
+ * before the @arg.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(int, void *arg), void *arg);
+ *	#define register_callback(fn, arg) \
+ *		_register_callback(typesafe_cb_preargs(void, (fn), (arg), int),\
+ *				   (arg))
+ */
+#define typesafe_cb_preargs(rtype, fn, arg, ...)			\
+	cast_if_type(rtype (*)(__VA_ARGS__, void *), (fn), (fn),	\
+		     rtype (*)(__VA_ARGS__, typeof(arg)))
+/**
+ * typesafe_cb_postargs - cast a callback function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument to hand to the callback function.
+ *
+ * This is a version of typesafe_cb() for callbacks that take other arguments
+ * after the @arg.
+ *
+ * Example:
+ *	void _register_callback(void (*fn)(void *arg, int), void *arg);
+ *	#define register_callback(fn, arg) \
+ *		_register_callback(typesafe_cb_postargs(void, (fn), (arg), int),\
+ *				   (arg))
+ */
+#define typesafe_cb_postargs(rtype, fn, arg, ...)			\
+	cast_if_type(rtype (*)(void *, __VA_ARGS__), (fn), (fn),	\
+		     rtype (*)(typeof(arg), __VA_ARGS__))
+/**
+ * typesafe_cb_cmp - cast a compare function if it matches the arg
+ * @rtype: the return type of the callback function
+ * @fn: the callback function to cast
+ * @arg: the (pointer) argument(s) to hand to the compare function.
+ *
+ * If a callback function takes two matching-type arguments, this macro does
+ * appropriate casts to a function which takes two const void * arguments if
+ * the callback provided takes two a const pointers to @arg.
+ *
+ * It is assumed that @arg is of pointer type: usually @arg is passed
+ * or assigned to a void * elsewhere anyway.  Note also that the type
+ * arg points to must be defined.
+ *
+ * Example:
+ *	void _my_qsort(void *base, size_t nmemb, size_t size,
+ *		       int (*cmp)(const void *, const void *));
+ *	#define my_qsort(base, nmemb, cmpfn) \
+ *		_my_qsort((base), (nmemb), sizeof(*(base)), \
+ *			  typesafe_cb_cmp(int, (cmpfn), (base)), (arg))
+ */
+#define typesafe_cb_cmp(rtype, cmpfn, arg)				\
+	cast_if_type(rtype (*)(const void *, const void *), (cmpfn),	\
+		     rtype (*)(const typeof(*arg)*, const typeof(*arg)*))
+		     
+#endif /* CCAN_CAST_IF_TYPE_H */