r12434: implement database scavenging, the only missing part is the verifying of active replicas

with the owning wins server, after the verify interval passes.

metze
(This used to be commit 7d1f7ae9c65c09f8bf72e159b771f231f96e8591)

5 files changed, 429 insertions, 2 deletions

diff --git a/source4/wrepl_server/config.mk b/source4/wrepl_server/config.mk
index dc9508dc5a..74335cbf05 100644
--- a/source4/wrepl_server/config.mk
+++ b/source4/wrepl_server/config.mk
@@ -9,6 +9,7 @@ INIT_OBJ_FILES = \
 		wrepl_in_call.o \
 		wrepl_apply_records.o \
 		wrepl_periodic.o \
+		wrepl_scavenging.o \
 		wrepl_out_pull.o \
 		wrepl_out_push.o \
 		wrepl_out_helpers.o
diff --git a/source4/wrepl_server/wrepl_periodic.c b/source4/wrepl_server/wrepl_periodic.c
index 2cea3b8e7c..09554d1659 100644
--- a/source4/wrepl_server/wrepl_periodic.c
+++ b/source4/wrepl_server/wrepl_periodic.c
@@ -39,6 +39,9 @@ static NTSTATUS wreplsrv_periodic_run(struct wreplsrv_service *service)
 {
 	NTSTATUS status;
 
+	status = wreplsrv_scavenging_run(service);
+	NT_STATUS_NOT_OK_RETURN(status);
+
 	status = wreplsrv_out_pull_run(service);
 	NT_STATUS_NOT_OK_RETURN(status);
 
diff --git a/source4/wrepl_server/wrepl_scavenging.c b/source4/wrepl_server/wrepl_scavenging.c
new file mode 100644
index 0000000000..a19604d416
--- /dev/null
+++ b/source4/wrepl_server/wrepl_scavenging.c
@@ -0,0 +1,383 @@
+/* 
+   Unix SMB/CIFS implementation.
+   
+   WINS Replication server
+   
+   Copyright (C) Stefan Metzmacher	2005
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include "includes.h"
+#include "dlinklist.h"
+#include "lib/events/events.h"
+#include "lib/socket/socket.h"
+#include "smbd/service_task.h"
+#include "smbd/service_stream.h"
+#include "lib/messaging/irpc.h"
+#include "librpc/gen_ndr/ndr_winsrepl.h"
+#include "wrepl_server/wrepl_server.h"
+#include "nbt_server/wins/winsdb.h"
+#include "ldb/include/ldb.h"
+#include "ldb/include/ldb_errors.h"
+#include "libcli/composite/composite.h"
+#include "libcli/wrepl/winsrepl.h"
+#include "wrepl_server/wrepl_out_helpers.h"
+#include "system/time.h"
+
+static NTSTATUS wreplsrv_scavenging_owned_records(struct wreplsrv_service *service, TALLOC_CTX *tmp_mem)
+{
+	NTSTATUS status;
+	struct winsdb_record *rec = NULL;
+	struct ldb_result *res = NULL;
+	const char *filter;
+	uint32_t i;
+	int ret;
+	time_t now = time(NULL);
+	const char *now_timestr;
+	const char *action;
+	const char *old_state;
+	uint32_t modify_flags;
+	BOOL modify_record;
+	BOOL delete_record;
+	BOOL delete_tombstones;
+	struct timeval tombstone_extra_time;
+
+	now_timestr = ldb_timestring(tmp_mem, now);
+	NT_STATUS_HAVE_NO_MEMORY(now_timestr);
+	filter = talloc_asprintf(tmp_mem,
+				 "(&(winsOwner=%s)(objectClass=winsRecord)"
+				 "(expireTime<=%s)(!(isStatic=1)))",
+				 WINSDB_OWNER_LOCAL, now_timestr);
+	NT_STATUS_HAVE_NO_MEMORY(filter);
+	ret = ldb_search(service->wins_db, NULL, LDB_SCOPE_SUBTREE, filter, NULL, &res);
+	if (ret != LDB_SUCCESS) return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	talloc_steal(tmp_mem, res);
+
+	tombstone_extra_time = timeval_add(&service->startup_time,
+					   service->config.tombstone_extra_timeout,
+					   0);
+	delete_tombstones = timeval_expired(&tombstone_extra_time);
+
+	for (i=0; i < res->count; i++) {
+		status = winsdb_record(res->msgs[i], tmp_mem, &rec);
+		NT_STATUS_NOT_OK_RETURN(status);
+
+		if (rec->is_static) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		if (rec->expire_time > now) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		modify_flags	= 0;
+		modify_record	= False;
+		delete_record	= False;
+
+		switch (rec->state) {
+		case WREPL_STATE_ACTIVE:
+			old_state	= "active";
+			rec->state	= WREPL_STATE_RELEASED;
+			rec->expire_time= service->config.tombstone_interval + now;
+			modify_flags	= 0;
+			modify_record	= True;
+			break;
+
+		case WREPL_STATE_RELEASED:
+			old_state	= "released";
+			rec->state	= WREPL_STATE_TOMBSTONE;
+			rec->expire_time= service->config.tombstone_timeout + now;
+			modify_flags	= WINSDB_FLAG_ALLOC_VERSION | WINSDB_FLAG_TAKE_OWNERSHIP;
+			modify_record	= True;
+			break;
+
+		case WREPL_STATE_TOMBSTONE:
+			old_state	= "tombstone";
+			if (!delete_tombstones) break;
+			delete_record = True;
+			break;
+
+		case WREPL_STATE_RESERVED:
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		if (modify_record) {
+			action = "modify";
+			ret = winsdb_modify(service->wins_db, rec, modify_flags);
+		} else if (delete_record) {
+			action = "delete";
+			ret = winsdb_delete(service->wins_db, rec);
+		} else {
+			action = "skip";
+			ret = NBT_RCODE_OK;
+		}
+
+		if (ret != NBT_RCODE_OK) {
+			DEBUG(1,("WINS scavenging: failed to %s name %s (owned:%s): error:%u\n",
+				action, nbt_name_string(rec, rec->name), old_state, ret));
+		} else {
+			DEBUG(4,("WINS scavenging: %s name: %s (owned:%s)\n",
+				action, nbt_name_string(rec, rec->name), old_state));
+		}
+
+		talloc_free(rec);
+	}
+
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS wreplsrv_scavenging_replica_non_active_records(struct wreplsrv_service *service, TALLOC_CTX *tmp_mem)
+{
+	NTSTATUS status;
+	struct winsdb_record *rec = NULL;
+	struct ldb_result *res = NULL;
+	const char *filter;
+	uint32_t i;
+	int ret;
+	time_t now = time(NULL);
+	const char *now_timestr;
+	const char *action;
+	const char *old_state;
+	uint32_t modify_flags;
+	BOOL modify_record;
+	BOOL delete_record;
+	BOOL delete_tombstones;
+	struct timeval tombstone_extra_time;
+
+	now_timestr = ldb_timestring(tmp_mem, now);
+	NT_STATUS_HAVE_NO_MEMORY(now_timestr);
+	filter = talloc_asprintf(tmp_mem,
+				 "(&(!(winsOwner=%s))(objectClass=winsRecord)"
+				 "(!(recordState=%u))(expireTime<=%s)(!(isStatic=1)))",
+				 WINSDB_OWNER_LOCAL, WREPL_STATE_ACTIVE, now_timestr);
+	NT_STATUS_HAVE_NO_MEMORY(filter);
+	ret = ldb_search(service->wins_db, NULL, LDB_SCOPE_SUBTREE, filter, NULL, &res);
+	if (ret != LDB_SUCCESS) return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	talloc_steal(tmp_mem, res);
+
+	tombstone_extra_time = timeval_add(&service->startup_time,
+					   service->config.tombstone_extra_timeout,
+					   0);
+	delete_tombstones = timeval_expired(&tombstone_extra_time);
+
+	for (i=0; i < res->count; i++) {
+		status = winsdb_record(res->msgs[i], tmp_mem, &rec);
+		NT_STATUS_NOT_OK_RETURN(status);
+
+		if (rec->is_static) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		if (rec->expire_time > now) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		modify_flags	= 0;
+		modify_record	= False;
+		delete_record	= False;
+
+		switch (rec->state) {
+		case WREPL_STATE_ACTIVE:
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+
+		case WREPL_STATE_RELEASED:
+			old_state	= "released";
+			rec->state	= WREPL_STATE_TOMBSTONE;
+			rec->expire_time= service->config.tombstone_timeout + now;
+			modify_flags	= 0;
+			modify_record	= True;
+			break;
+
+		case WREPL_STATE_TOMBSTONE:
+			old_state	= "tombstone";
+			if (!delete_tombstones) break;
+			delete_record = True;
+			break;
+
+		case WREPL_STATE_RESERVED:
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		if (modify_record) {
+			action = "modify";
+			ret = winsdb_modify(service->wins_db, rec, modify_flags);
+		} else if (delete_record) {
+			action = "delete";
+			ret = winsdb_delete(service->wins_db, rec);
+		} else {
+			action = "skip";
+			ret = NBT_RCODE_OK;
+		}
+
+		if (ret != NBT_RCODE_OK) {
+			DEBUG(1,("WINS scavenging: failed to %s name %s (replica:%s): error:%u\n",
+				action, nbt_name_string(rec, rec->name), old_state, ret));
+		} else {
+			DEBUG(4,("WINS scavenging: %s name: %s (replica:%s)\n",
+				action, nbt_name_string(rec, rec->name), old_state));
+		}
+
+		talloc_free(rec);
+	}
+
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS wreplsrv_scavenging_replica_active_records(struct wreplsrv_service *service, TALLOC_CTX *tmp_mem)
+{
+	NTSTATUS status;
+	struct winsdb_record *rec = NULL;
+	struct ldb_result *res = NULL;
+	const char *filter;
+	uint32_t i;
+	int ret;
+	time_t now = time(NULL);
+	const char *now_timestr;
+	const char *action;
+	const char *old_state;
+	BOOL modify_flags;
+	BOOL modify_record;
+	BOOL delete_record;
+
+	now_timestr = ldb_timestring(tmp_mem, now);
+	NT_STATUS_HAVE_NO_MEMORY(now_timestr);
+	filter = talloc_asprintf(tmp_mem,
+				 "(&(!(winsOwner=%s))(objectClass=winsRecord)"
+				 "(recordState=%u)(expireTime<=%s)(!(isStatic=1)))",
+				 WINSDB_OWNER_LOCAL, WREPL_STATE_ACTIVE, now_timestr);
+	NT_STATUS_HAVE_NO_MEMORY(filter);
+	ret = ldb_search(service->wins_db, NULL, LDB_SCOPE_SUBTREE, filter, NULL, &res);
+	if (ret != LDB_SUCCESS) return NT_STATUS_INTERNAL_DB_CORRUPTION;
+	talloc_steal(tmp_mem, res);
+
+	for (i=0; i < res->count; i++) {
+		status = winsdb_record(res->msgs[i], tmp_mem, &rec);
+		NT_STATUS_NOT_OK_RETURN(status);
+
+		if (rec->is_static) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		if (rec->expire_time > now) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		if (rec->state != WREPL_STATE_ACTIVE) {
+			DEBUG(0,("%s: corrupted record: %s\n",
+				__location__, nbt_name_string(rec, rec->name)));
+			return NT_STATUS_INTERNAL_DB_CORRUPTION;
+		}
+
+		old_state = "active";
+
+		modify_flags	= 0;
+		modify_record	= False;
+		delete_record	= False;
+
+		/* 
+		 * TODO: ask the owning wins server if the record still exists,
+		 *       if not delete the record
+		 */
+		DEBUG(0,("TODO: ask wins server '%s' if '%s' with version_id:%llu still exists\n",
+			rec->wins_owner, nbt_name_string(rec, rec->name), rec->version));
+
+		if (modify_record) {
+			action = "modify";
+			ret = winsdb_modify(service->wins_db, rec, modify_flags);
+		} else if (delete_record) {
+			action = "delete";
+			ret = winsdb_delete(service->wins_db, rec);
+		} else {
+			action = "skip";
+			ret = NBT_RCODE_OK;
+		}
+
+		if (ret != NBT_RCODE_OK) {
+			DEBUG(1,("WINS scavenging: failed to %s name %s (replica:%s): error:%u\n",
+				action, nbt_name_string(rec, rec->name), old_state, ret));
+		} else {
+			DEBUG(4,("WINS scavenging: %s name: %s (replica:%s)\n",
+				action, nbt_name_string(rec, rec->name), old_state));
+		}
+
+		talloc_free(rec);
+	}
+
+	return NT_STATUS_OK;
+}
+
+NTSTATUS wreplsrv_scavenging_run(struct wreplsrv_service *service)
+{
+	NTSTATUS status;
+	TALLOC_CTX *tmp_mem;
+
+	if (!timeval_expired(&service->scavenging.next_run)) {
+		return NT_STATUS_OK;
+	}
+
+	service->scavenging.next_run = timeval_current_ofs(service->config.scavenging_interval, 0);
+	status = wreplsrv_periodic_schedule(service, service->config.scavenging_interval);
+	NT_STATUS_NOT_OK_RETURN(status);
+
+	if (service->scavenging.processing) {
+		return NT_STATUS_OK;
+	}
+
+	DEBUG(4,("wreplsrv_scavenging_run(): start\n"));
+
+	tmp_mem = talloc_new(service);
+	service->scavenging.processing = True;
+	status = wreplsrv_scavenging_owned_records(service,tmp_mem);
+	service->scavenging.processing = False;
+	talloc_free(tmp_mem);
+	NT_STATUS_NOT_OK_RETURN(status);
+
+	tmp_mem = talloc_new(service);	
+	service->scavenging.processing = True;
+	status = wreplsrv_scavenging_replica_non_active_records(service, tmp_mem);
+	service->scavenging.processing = False;
+	talloc_free(tmp_mem);
+	NT_STATUS_NOT_OK_RETURN(status);
+
+	tmp_mem = talloc_new(service);
+	service->scavenging.processing = True;
+	status = wreplsrv_scavenging_replica_active_records(service, tmp_mem);
+	service->scavenging.processing = False;
+	talloc_free(tmp_mem);
+	NT_STATUS_NOT_OK_RETURN(status);
+
+	DEBUG(4,("wreplsrv_scavenging_run(): end\n"));
+
+	return NT_STATUS_OK;
+}
diff --git a/source4/wrepl_server/wrepl_server.c b/source4/wrepl_server/wrepl_server.c
index 3b800333ed..ad1bca3e60 100644
--- a/source4/wrepl_server/wrepl_server.c
+++ b/source4/wrepl_server/wrepl_server.c
@@ -63,9 +63,16 @@ static NTSTATUS wreplsrv_open_winsdb(struct wreplsrv_service *service)
 	/* the default tombstone (extinction) timeout is 1 day */
 	service->config.tombstone_timeout = lp_parm_int(-1,"wreplsrv","tombstone_timeout", 1*24*60*60);
 
+	/* the default tombstone extra timeout is 3 days */
+	service->config.tombstone_extra_timeout = lp_parm_int(-1,"wreplsrv","tombstone_extra_timeout", 3*24*60*60);
+
 	/* the default verify interval is 24 days */
 	service->config.verify_interval   = lp_parm_int(-1,"wreplsrv","verify_interval", 24*24*60*60);
 
+	/* the default scavenging interval is 'renew_interval/2' */
+	service->config.scavenging_interval=lp_parm_int(-1,"wreplsrv","scavenging_interval",
+							service->config.renew_interval/2);
+
 	/* the maximun interval to the next periodic processing event */
 	service->config.periodic_interval = lp_parm_int(-1,"wreplsrv","periodic_interval", 60);
 
@@ -364,8 +371,9 @@ static void wreplsrv_task_init(struct task_server *task)
 		task_server_terminate(task, "wreplsrv_task_init: out of memory");
 		return;
 	}
-	service->task = task;
-	task->private = service;
+	service->task		= task;
+	service->startup_time	= timeval_current();
+	task->private		= service;
 
 	/*
 	 * setup up all partners, and open the winsdb
diff --git a/source4/wrepl_server/wrepl_server.h b/source4/wrepl_server/wrepl_server.h
index 56751b6f29..125e04b84c 100644
--- a/source4/wrepl_server/wrepl_server.h
+++ b/source4/wrepl_server/wrepl_server.h
@@ -209,6 +209,9 @@ struct wreplsrv_service {
 	/* the whole wrepl service is in one task */
 	struct task_server *task;
 
+	/* the time the service was started */
+	struct timeval startup_time;
+
 	/* the winsdb handle */
 	struct ldb_context *wins_db;
 
@@ -232,17 +235,32 @@ struct wreplsrv_service {
 		/* 
 		 * the interval (in secs) a record remains in TOMBSTONE state,
 		 * before it will be removed from the database.
+		 * See also 'tombstone_extra_timeout'.
 		 * (also known as extinction timeout)
 		 */
 		uint32_t tombstone_timeout;
 
 		/* 
+		 * the interval (in secs) a record remains in TOMBSTONE state,
+		 * even after 'tombstone_timeout' passes the current timestamp.
+		 * this is the minimum uptime of the wrepl service, before
+		 * we start delete tombstones. This is to prevent deletion of
+		 * tombstones, without replacte them.
+		 */
+		uint32_t tombstone_extra_timeout;
+
+		/* 
 		 * the interval (in secs) till a replica record will be verified
 		 * with the owning wins server
 		 */
 		uint32_t verify_interval;
 
 		/* 
+		 * the interval (in secs) till a do a database cleanup
+		 */
+		uint32_t scavenging_interval;
+
+		/* 
 		 * the interval (in secs) to the next periodic processing
 		 * (this is the maximun interval)
 		 */
@@ -269,4 +287,18 @@ struct wreplsrv_service {
 		/* here we have a reference to the timed event the schedules the periodic stuff */
 		struct timed_event *te;
 	} periodic;
+
+	/* some stuff for scavenging processing */
+	struct {
+		/*
+		 * the timestamp for the next scavenging run,
+		 * this is the timstamp passed to event_add_timed()
+		 */
+		struct timeval next_run;
+
+		/*
+		 * are we currently inside a scavenging run
+		 */
+		BOOL processing;	
+	} scavenging;
 };