r18781: Move the usnCreated and usnChanged handling around again.

This moves these attributes from objectguid into an optional backend
(objectguid), used by ltdb.  For OpenLDAP, the entryUUID module
converts entryCSN into usnChanged.

This also changes the sequence number API, and uses 'time based'
sequence numbers, when an LDAP or similar backend is detected.

To assist this, we also store the last modified time in the TDB,
whenever we change a value.

Andrew Bartlett
(This used to be commit 72858f859483c0c532dddb2c146d6bd7b9be5072)

7 files changed, 71 insertions, 144 deletions

diff --git a/source4/lib/ldb/common/ldb.c b/source4/lib/ldb/common/ldb.c
index a68733244d..1aae76bad6 100644
--- a/source4/lib/ldb/common/ldb.c
+++ b/source4/lib/ldb/common/ldb.c
@@ -888,7 +888,7 @@ int ldb_rename(struct ldb_context *ldb, const struct ldb_dn *olddn, const struct
 /*
   return the global sequence number
 */
-int ldb_sequence_number(struct ldb_context *ldb, uint64_t *seq_num)
+int ldb_sequence_number(struct ldb_context *ldb, enum ldb_sequence_type type, uint64_t *seq_num)
 {
 	struct ldb_request *req;
 	int ret;
@@ -905,6 +905,7 @@ int ldb_sequence_number(struct ldb_context *ldb, uint64_t *seq_num)
 	req->callback = NULL;
 	ldb_set_timeout(ldb, req, 0); /* use default timeout */
 
+	req->op.seq_num.type = type;
 	/* do request and autostart a transaction */
 	ret = ldb_request(ldb, req);
 	
diff --git a/source4/lib/ldb/include/ldb.h b/source4/lib/ldb/include/ldb.h
index 78d701689d..cf4a1f282b 100644
--- a/source4/lib/ldb/include/ldb.h
+++ b/source4/lib/ldb/include/ldb.h
@@ -740,7 +740,13 @@ struct ldb_register_partition {
 };
 
 struct ldb_sequence_number {
+	enum ldb_sequence_type {
+		LDB_SEQ_HIGHEST_SEQ,
+		LDB_SEQ_HIGHEST_TIMESTAMP,
+		LDB_SEQ_NEXT
+	} type;
 	uint64_t seq_num;
+	uint32_t flags;
 };
 
 typedef int (*ldb_request_callback_t)(struct ldb_context *, void *, struct ldb_reply *);
@@ -923,11 +929,6 @@ int ldb_rename(struct ldb_context *ldb, const struct ldb_dn *olddn, const struct
 int ldb_delete(struct ldb_context *ldb, const struct ldb_dn *dn);
 
 /**
-  Obtain current database sequence number
-*/
-int ldb_sequence_number(struct ldb_context *ldb, uint64_t *seq_num);
-
-/**
   start a transaction
 */
 int ldb_transaction_start(struct ldb_context *ldb);
diff --git a/source4/lib/ldb/include/ldb_private.h b/source4/lib/ldb/include/ldb_private.h
index d66b6f1cc7..96b71ff3b4 100644
--- a/source4/lib/ldb/include/ldb_private.h
+++ b/source4/lib/ldb/include/ldb_private.h
@@ -207,4 +207,14 @@ int check_critical_controls(struct ldb_control **controls);
 
 /* The following definitions come from lib/ldb/common/ldb_utf8.c */
 char *ldb_casefold_default(void *context, void *mem_ctx, const char *s);
+
+/**
+  Obtain current/next database sequence number
+*/
+int ldb_sequence_number(struct ldb_context *ldb, enum ldb_sequence_type type, uint64_t *seq_num);
+
+#define LDB_SEQ_GLOBAL_SEQUENCE    0x01
+#define LDB_SEQ_TIMESTAMP_SEQUENCE 0x02
+
+
 #endif
diff --git a/source4/lib/ldb/ldb_tdb/ldb_cache.c b/source4/lib/ldb/ldb_tdb/ldb_cache.c
index 5634e9ad16..d6d66dd37f 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_cache.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_cache.c
@@ -413,8 +413,10 @@ int ltdb_increase_sequence_number(struct ldb_module *module)
 {
 	struct ltdb_private *ltdb = module->private_data;
 	struct ldb_message *msg;
-	struct ldb_message_element el;
+	struct ldb_message_element el[2];
 	struct ldb_val val;
+	struct ldb_val val_time;
+	time_t t = time(NULL);
 	char *s = NULL;
 	int ret;
 
@@ -424,32 +426,50 @@ int ltdb_increase_sequence_number(struct ldb_module *module)
 		return -1;
 	}
 
-	s = talloc_asprintf(msg, "%.0f", ltdb->sequence_number+1);
+	s = talloc_asprintf(msg, "%llu", ltdb->sequence_number+1);
 	if (!s) {
 		errno = ENOMEM;
 		return -1;
 	}
 
-	msg->num_elements = 1;
-	msg->elements = ⪙
+	msg->num_elements = ARRAY_SIZE(el);
+	msg->elements = el;
 	msg->dn = ldb_dn_explode(msg, LTDB_BASEINFO);
 	if (msg->dn == NULL) {
 		talloc_free(msg);
 		errno = ENOMEM;
 		return -1;
 	}
-	el.name = talloc_strdup(msg, LTDB_SEQUENCE_NUMBER);
-	if (el.name == NULL) {
+	el[0].name = talloc_strdup(msg, LTDB_SEQUENCE_NUMBER);
+	if (el[0].name == NULL) {
 		talloc_free(msg);
 		errno = ENOMEM;
 		return -1;
 	}
-	el.values = &val;
-	el.num_values = 1;
-	el.flags = LDB_FLAG_MOD_REPLACE;
+	el[0].values = &val;
+	el[0].num_values = 1;
+	el[0].flags = LDB_FLAG_MOD_REPLACE;
 	val.data = (uint8_t *)s;
 	val.length = strlen(s);
 
+	el[1].name = talloc_strdup(msg, LTDB_MOD_TIMESTAMP);
+	if (el[1].name == NULL) {
+		talloc_free(msg);
+		errno = ENOMEM;
+		return -1;
+	}
+	el[1].values = &val_time;
+	el[1].num_values = 1;
+	el[1].flags = LDB_FLAG_MOD_REPLACE;
+
+	s = ldb_timestring(msg, t);
+	if (s == NULL) {
+		return -1;
+	}
+
+	val_time.data = (uint8_t *)s;
+	val_time.length = strlen(s);
+
 	ret = ltdb_modify_internal(module, msg);
 
 	talloc_free(msg);
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
index 5a19dd96fc..8f676654a6 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c
@@ -944,6 +944,8 @@ static int ltdb_sequence_number(struct ldb_module *module, struct ldb_request *r
 		return LDB_ERR_OPERATIONS_ERROR;
 	}
 
+	req->op.seq_num.flags = 0;
+
 	tret = ltdb_search_dn1(module, dn, msg);
 	if (tret != 1) {
 		talloc_free(tmp_ctx);
@@ -952,7 +954,26 @@ static int ltdb_sequence_number(struct ldb_module *module, struct ldb_request *r
 		return LDB_SUCCESS;
 	}
 
-	req->op.seq_num.seq_num = ldb_msg_find_attr_as_uint64(msg, LTDB_SEQUENCE_NUMBER, 0);
+	switch (req->op.seq_num.type) {
+	case LDB_SEQ_HIGHEST_SEQ:
+		req->op.seq_num.seq_num = ldb_msg_find_attr_as_uint64(msg, LTDB_SEQUENCE_NUMBER, 0);
+		break;
+	case LDB_SEQ_NEXT:
+		req->op.seq_num.seq_num = ldb_msg_find_attr_as_uint64(msg, LTDB_SEQUENCE_NUMBER, 0);
+		req->op.seq_num.seq_num++;
+		break;
+	case LDB_SEQ_HIGHEST_TIMESTAMP:
+	{
+		const char *date = ldb_msg_find_attr_as_string(msg, LTDB_MOD_TIMESTAMP, NULL);
+		if (date) {
+			req->op.seq_num.seq_num = ldb_string_to_time(date);
+		} else {
+			req->op.seq_num.seq_num = 0;
+			/* zero is as good as anything when we don't know */
+		}
+		break;
+	}
+	}
 	talloc_free(tmp_ctx);
 	return LDB_SUCCESS;
 }
diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index fb28d00847..7b98b9ddee 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -14,7 +14,7 @@ struct ltdb_private {
 	
 	/* a double is used for portability and ease of string
 	   handling. It has plenty of digits of precision */
-	double sequence_number;
+	unsigned long long sequence_number;
 
 	struct ltdb_cache {
 		struct ldb_message *baseinfo;
@@ -58,6 +58,7 @@ struct ltdb_context {
 
 /* special attribute types */
 #define LTDB_SEQUENCE_NUMBER "sequenceNumber"
+#define LTDB_MOD_TIMESTAMP "whenChanged"
 #define LTDB_OBJECTCLASS "objectClass"
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_cache.c  */
diff --git a/source4/lib/ldb/modules/operational.c b/source4/lib/ldb/modules/operational.c
index 1e405355c4..c327a96f90 100644
--- a/source4/lib/ldb/modules/operational.c
+++ b/source4/lib/ldb/modules/operational.c
@@ -169,29 +169,6 @@ failed:
 	return -1;
 }
 
-/*
-  add a uint64_t element to a record
-*/
-static int add_uint64_element(struct ldb_message *msg, const char *attr, uint64_t v)
-{
-	struct ldb_message_element *el;
-
-	if (ldb_msg_find_element(msg, attr) != NULL) {
-		return 0;
-	}
-
-	if (ldb_msg_add_fmt(msg, attr, "%llu", (unsigned long long)v) != 0) {
-		return -1;
-	}
-
-	el = ldb_msg_find_element(msg, attr);
-	/* always set as replace. This works because on add ops, the flag
-	   is ignored */
-	el->flags = LDB_FLAG_MOD_REPLACE;
-
-	return 0;
-}
-
 
 /*
   hook search operations
@@ -312,108 +289,6 @@ static int operational_search(struct ldb_module *module, struct ldb_request *req
 	return ret;
 }
 
-/*
-  hook add record ops
-*/
-static int operational_add(struct ldb_module *module, struct ldb_request *req)
-{
-	struct ldb_request *down_req;
-	struct ldb_message *msg;
-	uint64_t seq_num;
-	int ret;
-
-	if (ldb_dn_is_special(req->op.add.message->dn)) {
-		return ldb_next_request(module, req);
-	}
-
-	down_req = talloc(req, struct ldb_request);
-	if (down_req == NULL) {
-		return LDB_ERR_OPERATIONS_ERROR;
-	}
-
-	*down_req = *req;
-
-	/* we have to copy the message as the caller might have it as a const */
-	down_req->op.mod.message = msg = ldb_msg_copy_shallow(down_req, req->op.mod.message);
-	if (msg == NULL) {
-		return LDB_ERR_OPERATIONS_ERROR;
-	}
-
-	/* Get a sequence number from the backend */
-	ret = ldb_sequence_number(module->ldb, &seq_num);
-	if (ret == LDB_SUCCESS) {
-		if (add_uint64_element(msg, "uSNCreated", seq_num) != 0 ||
-		    add_uint64_element(msg, "uSNChanged", seq_num) != 0) {
-			talloc_free(down_req);
-			return LDB_ERR_OPERATIONS_ERROR;
-		}
-	}
-
-	ldb_set_timeout_from_prev_req(module->ldb, req, down_req);
-
-	/* go on with the call chain */
-	ret = ldb_next_request(module, down_req);
-
-	/* do not free down_req as the call results may be linked to it,
-	 * it will be freed when the upper level request get freed */
-	if (ret == LDB_SUCCESS) {
-		req->handle = down_req->handle;
-	}
-
-	return ret;
-}
-
-/*
-  hook modify record ops
-*/
-static int operational_modify(struct ldb_module *module, struct ldb_request *req)
-{
-	struct ldb_request *down_req;
-	struct ldb_message *msg;
-	uint64_t seq_num;
-	int ret;
-
-	if (ldb_dn_is_special(req->op.mod.message->dn)) {
-		return ldb_next_request(module, req);
-	}
-
-	down_req = talloc(req, struct ldb_request);
-	if (down_req == NULL) {
-		return LDB_ERR_OPERATIONS_ERROR;
-	}
-
-	*down_req = *req;
-
-	/* we have to copy the message as the caller might have it as a const */
-	down_req->op.mod.message = msg = ldb_msg_copy_shallow(down_req, req->op.mod.message);
-	if (msg == NULL) {
-		return LDB_ERR_OPERATIONS_ERROR;
-	}
-	/* Get a sequence number from the backend */
-	ret = ldb_sequence_number(module->ldb, &seq_num);
-	if (ret == LDB_SUCCESS) {
-		/* update the records USN if possible */
-		if (add_uint64_element(msg, "uSNChanged", 
-				       seq_num) != 0) {
-			talloc_free(down_req);
-			return -1;
-		}
-	}
-	
-	ldb_set_timeout_from_prev_req(module->ldb, req, down_req);
-
-	/* go on with the call chain */
-	ret = ldb_next_request(module, down_req);
-
-	/* do not free down_req as the call results may be linked to it,
-	 * it will be freed when the upper level request get freed */
-	if (ret == LDB_SUCCESS) {
-		req->handle = down_req->handle;
-	}
-
-	return ret;
-}
-
 static int operational_init(struct ldb_module *ctx)
 {
 	/* setup some standard attribute handlers */
@@ -428,8 +303,6 @@ static int operational_init(struct ldb_module *ctx)
 static const struct ldb_module_ops operational_ops = {
 	.name              = "operational",
 	.search            = operational_search,
-	.add               = operational_add,
-	.modify            = operational_modify,
 	.init_context	   = operational_init
 };