s3: Fix a long-standing problem with recycled PIDs

When a samba server process dies hard, it has no chance to clean up its entries
in locking.tdb, brlock.tdb, connections.tdb and sessionid.tdb.

For locking.tdb and brlock.tdb Samba is robust by checking every time we read
an entry from the database if the corresponding process still exists. If it
does not exist anymore, the entry is deleted. This is not 100% failsafe though:
On systems with a limited PID space there is a non-zero chance that between the
smbd's death and the fresh access, the PID is recycled by another long-running
process. This renders all files that had been locked by the killed smbd
potentially unusable until the new process also dies.

This patch is supposed to fix the problem the following way: Every process ID
in every database is augmented by a random 64-bit number that is stored in a
serverid.tdb. Whenever we need to check if a process still exists we know its
PID and the 64-bit number. We look up the PID in serverid.tdb and compare the
64-bit number. If it's the same, the process still is a valid smbd holding the
lock. If it is different, a new smbd has taken over.

I believe this is safe against an smbd that has died hard and the PID has been
taken over by a non-samba process. This process would not have registered
itself with a fresh 64-bit number in serverid.tdb, so the old one still exists
in serverid.tdb. We protect against this case by the parent smbd taking care of
deregistering PIDs from serverid.tdb and the fact that serverid.tdb is
CLEAR_IF_FIRST.

CLEAR_IF_FIRST does not work in a cluster, so the automatic cleanup does not
work when all smbds are restarted. For this, "net serverid wipe" has to be run
before smbd starts up. As a convenience, "net serverid wipedbs" also cleans up
sessionid.tdb and connections.tdb.

While there, this also cleans up overloading connections.tdb with all the
process entries just for messaging_send_all().

Volker

3 files changed, 309 insertions, 14 deletions

diff --git a/source3/lib/messages.c b/source3/lib/messages.c
index 5e11dd4e25..2fcdc24179 100644
--- a/source3/lib/messages.c
+++ b/source3/lib/messages.c
@@ -95,36 +95,29 @@ struct msg_all {
  Send one of the messages for the broadcast.
 ****************************************************************************/
 
-static int traverse_fn(struct db_record *rec,
-		       const struct connections_key *ckey,
-		       const struct connections_data *crec,
-		       void *state)
+static int traverse_fn(struct db_record *rec, const struct server_id *id,
+		       uint32_t msg_flags, void *state)
 {
 	struct msg_all *msg_all = (struct msg_all *)state;
 	NTSTATUS status;
 
-	if (crec->cnum != -1)
-		return 0;
-
 	/* Don't send if the receiver hasn't registered an interest. */
 
-	if(!(crec->bcast_msg_flags & msg_all->msg_flag))
+	if((msg_flags & msg_all->msg_flag) == 0) {
 		return 0;
+	}
 
 	/* If the msg send fails because the pid was not found (i.e. smbd died), 
 	 * the msg has already been deleted from the messages.tdb.*/
 
-	status = messaging_send_buf(msg_all->msg_ctx,
-				    crec->pid, msg_all->msg_type,
+	status = messaging_send_buf(msg_all->msg_ctx, *id, msg_all->msg_type,
 				    (uint8 *)msg_all->buf, msg_all->len);
 
 	if (NT_STATUS_EQUAL(status, NT_STATUS_INVALID_HANDLE)) {
 		
 		/* If the pid was not found delete the entry from connections.tdb */
 
-		DEBUG(2,("pid %s doesn't exist - deleting connections %d [%s]\n",
-			 procid_str_static(&crec->pid), crec->cnum,
-			 crec->servicename));
+		DEBUG(2, ("pid %s doesn't exist\n", procid_str_static(id)));
 
 		rec->delete_rec(rec);
 	}
@@ -172,7 +165,7 @@ bool message_send_all(struct messaging_context *msg_ctx,
 	msg_all.n_sent = 0;
 	msg_all.msg_ctx = msg_ctx;
 
-	connections_forall(traverse_fn, &msg_all);
+	serverid_traverse(traverse_fn, &msg_all);
 	if (n_sent)
 		*n_sent = msg_all.n_sent;
 	return True;
diff --git a/source3/lib/serverid.c b/source3/lib/serverid.c
new file mode 100644
index 0000000000..9842ead103
--- /dev/null
+++ b/source3/lib/serverid.c
@@ -0,0 +1,293 @@
+/*
+   Unix SMB/CIFS implementation.
+   Implementation of a reliable server_exists()
+   Copyright (C) Volker Lendecke 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "serverid.h"
+
+struct serverid_key {
+	pid_t pid;
+#ifdef CLUSTER_SUPPORT
+	uint32_t vnn;
+#endif
+};
+
+struct serverid_data {
+	uint64_t unique_id;
+	uint32_t msg_flags;
+};
+
+static struct db_context *serverid_db(void)
+{
+	static struct db_context *db;
+
+	if (db != NULL) {
+		return db;
+	}
+	db = db_open(talloc_autofree_context(), lock_path("serverid.tdb"),
+		     0, TDB_DEFAULT|TDB_CLEAR_IF_FIRST, O_RDWR|O_CREAT, 0644);
+	return db;
+}
+
+static void serverid_fill_key(const struct server_id *id,
+			      struct serverid_key *key)
+{
+	ZERO_STRUCTP(key);
+	key->pid = id->pid;
+#ifdef CLUSTER_SUPPORT
+	key->vnn = id->vnn;
+#endif
+}
+
+bool serverid_register(const struct server_id *id, uint32_t msg_flags)
+{
+	struct db_context *db;
+	struct serverid_key key;
+	struct serverid_data data;
+	struct db_record *rec;
+	TDB_DATA tdbkey, tdbdata;
+	NTSTATUS status;
+	bool ret = false;
+
+	db = serverid_db();
+	if (db == NULL) {
+		return false;
+	}
+
+	serverid_fill_key(id, &key);
+	tdbkey = make_tdb_data((uint8_t *)&key, sizeof(key));
+
+	rec = db->fetch_locked(db, talloc_tos(), tdbkey);
+	if (rec == NULL) {
+		DEBUG(1, ("Could not fetch_lock serverid.tdb record\n"));
+		return false;
+	}
+
+	ZERO_STRUCT(data);
+	data.unique_id = id->unique_id;
+	data.msg_flags = msg_flags;
+
+	tdbdata = make_tdb_data((uint8_t *)&data, sizeof(data));
+	status = rec->store(rec, tdbdata, 0);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(1, ("Storing serverid.tdb record failed: %s\n",
+			  nt_errstr(status)));
+		goto done;
+	}
+	ret = true;
+done:
+	TALLOC_FREE(rec);
+	return ret;
+}
+
+bool serverid_register_self(uint32_t msg_flags)
+{
+	struct server_id pid;
+
+	pid = procid_self();
+	return serverid_register(&pid, msg_flags);
+}
+
+bool serverid_deregister(const struct server_id *id)
+{
+	struct db_context *db;
+	struct serverid_key key;
+	struct db_record *rec;
+	TDB_DATA tdbkey;
+	NTSTATUS status;
+	bool ret = false;
+
+	db = serverid_db();
+	if (db == NULL) {
+		return false;
+	}
+
+	serverid_fill_key(id, &key);
+	tdbkey = make_tdb_data((uint8_t *)&key, sizeof(key));
+
+	rec = db->fetch_locked(db, talloc_tos(), tdbkey);
+	if (rec == NULL) {
+		DEBUG(1, ("Could not fetch_lock serverid.tdb record\n"));
+		return false;
+	}
+
+	status = rec->delete_rec(rec);
+	if (!NT_STATUS_IS_OK(status)) {
+		DEBUG(1, ("Deleting serverid.tdb record failed: %s\n",
+			  nt_errstr(status)));
+		goto done;
+	}
+	ret = true;
+done:
+	TALLOC_FREE(rec);
+	return ret;
+}
+
+bool serverid_deregister_self(void)
+{
+	struct server_id pid;
+
+	pid = procid_self();
+	return serverid_deregister(&pid);
+}
+
+struct serverid_exists_state {
+	const struct server_id *id;
+	bool exists;
+};
+
+static int server_exists_parse(TDB_DATA key, TDB_DATA data, void *priv)
+{
+	struct serverid_exists_state *state =
+		(struct serverid_exists_state *)priv;
+	uint64_t unique_id;
+
+	if (data.dsize != sizeof(struct serverid_data)) {
+		return -1;
+	}
+
+	/* memcpy, data.dptr might not be aligned */
+	memcpy(&unique_id, data.dptr, sizeof(unique_id));
+
+	state->exists = (state->id->unique_id == unique_id);
+	return 0;
+}
+
+bool serverid_exists(const struct server_id *id)
+{
+	struct db_context *db;
+	struct serverid_exists_state state;
+	struct serverid_key key;
+	TDB_DATA tdbkey;
+
+	db = serverid_db();
+	if (db == NULL) {
+		return false;
+	}
+
+	serverid_fill_key(id, &key);
+	tdbkey = make_tdb_data((uint8_t *)&key, sizeof(key));
+
+	state.id = id;
+	state.exists = false;
+
+	if (db->parse_record(db, tdbkey, server_exists_parse, &state) == -1) {
+		return false;
+	}
+	return state.exists;
+}
+
+static bool serverid_rec_parse(const struct db_record *rec,
+			       struct server_id *id, uint32_t *msg_flags)
+{
+	struct serverid_key key;
+	struct serverid_data data;
+
+	if (rec->key.dsize != sizeof(key)) {
+		DEBUG(1, ("Found invalid key length %d in serverid.tdb\n",
+			  (int)rec->key.dsize));
+		return false;
+	}
+	if (rec->value.dsize != sizeof(data)) {
+		DEBUG(1, ("Found invalid value length %d in serverid.tdb\n",
+			  (int)rec->value.dsize));
+		return false;
+	}
+
+	memcpy(&key, rec->key.dptr, sizeof(key));
+	memcpy(&data, rec->value.dptr, sizeof(data));
+
+	id->pid = key.pid;
+#ifdef CLUSTER_SUPPORT
+	id->vnn = key.vnn;
+#endif
+	id->unique_id = data.unique_id;
+	*msg_flags = data.msg_flags;
+	return true;
+}
+
+struct serverid_traverse_read_state {
+	int (*fn)(const struct server_id *id, uint32_t msg_flags,
+		  void *private_data);
+	void *private_data;
+};
+
+static int serverid_traverse_read_fn(struct db_record *rec, void *private_data)
+{
+	struct serverid_traverse_read_state *state =
+		(struct serverid_traverse_read_state *)private_data;
+	struct server_id id;
+	uint32_t msg_flags;
+
+	if (!serverid_rec_parse(rec, &id, &msg_flags)) {
+		return 0;
+	}
+	return state->fn(&id, msg_flags,state->private_data);
+}
+
+bool serverid_traverse_read(int (*fn)(const struct server_id *id,
+				      uint32_t msg_flags, void *private_data),
+			    void *private_data)
+{
+	struct db_context *db;
+	struct serverid_traverse_read_state state;
+
+	db = serverid_db();
+	if (db == NULL) {
+		return false;
+	}
+	state.fn = fn;
+	state.private_data = private_data;
+	return db->traverse_read(db, serverid_traverse_read_fn, &state);
+}
+
+struct serverid_traverse_state {
+	int (*fn)(struct db_record *rec, const struct server_id *id,
+		  uint32_t msg_flags, void *private_data);
+	void *private_data;
+};
+
+static int serverid_traverse_fn(struct db_record *rec, void *private_data)
+{
+	struct serverid_traverse_state *state =
+		(struct serverid_traverse_state *)private_data;
+	struct server_id id;
+	uint32_t msg_flags;
+
+	if (!serverid_rec_parse(rec, &id, &msg_flags)) {
+		return 0;
+	}
+	return state->fn(rec, &id, msg_flags, state->private_data);
+}
+
+bool serverid_traverse(int (*fn)(struct db_record *rec,
+				 const struct server_id *id,
+				 uint32_t msg_flags, void *private_data),
+			    void *private_data)
+{
+	struct db_context *db;
+	struct serverid_traverse_state state;
+
+	db = serverid_db();
+	if (db == NULL) {
+		return false;
+	}
+	state.fn = fn;
+	state.private_data = private_data;
+	return db->traverse(db, serverid_traverse_fn, &state);
+}
diff --git a/source3/lib/util.c b/source3/lib/util.c
index f35a55fbed..68f26b8393 100644
--- a/source3/lib/util.c
+++ b/source3/lib/util.c
@@ -2647,10 +2647,18 @@ uint32 get_my_vnn(void)
 	return my_vnn;
 }
 
+static uint64_t my_unique_id = 0;
+
+void set_my_unique_id(uint64_t unique_id)
+{
+	my_unique_id = unique_id;
+}
+
 struct server_id pid_to_procid(pid_t pid)
 {
 	struct server_id result;
 	result.pid = pid;
+	result.unique_id = my_unique_id;
 #ifdef CLUSTER_SUPPORT
 	result.vnn = my_vnn;
 #endif
@@ -2720,6 +2728,7 @@ struct server_id interpret_pid(const char *pid_string)
 	if (result.pid < 0) {
 		result.pid = -1;
 	}
+	result.unique_id = 0;
 	return result;
 }