s3: Add dbwrap_try_fetch_locked

This is designed to spread the load on individual ctdb records to allow upper
layers to do backoff mechanisms. In the ctdb case, do not get the record if a
local lock is already taken. If we are not dmaster, do at most one migrate
attempt.

For the tdb case, this is a nonblocking fetch_locked. If someone else has the
lock, give up.

8 files changed, 97 insertions, 16 deletions

diff --git a/source3/lib/dbwrap/dbwrap.c b/source3/lib/dbwrap/dbwrap.c
index 11ab5d91f0..cfd15a976b 100644
--- a/source3/lib/dbwrap/dbwrap.c
+++ b/source3/lib/dbwrap/dbwrap.c
@@ -141,9 +141,10 @@ static struct dbwrap_lock_order_state *dbwrap_check_lock_order(
 	return state;
 }
 
-struct db_record *dbwrap_fetch_locked(struct db_context *db,
-				      TALLOC_CTX *mem_ctx,
-				      TDB_DATA key)
+static struct db_record *dbwrap_fetch_locked_internal(
+	struct db_context *db, TALLOC_CTX *mem_ctx, TDB_DATA key,
+	struct db_record *(*db_fn)(struct db_context *db, TALLOC_CTX *mem_ctx,
+				   TDB_DATA key))
 {
 	struct db_record *rec;
 	struct dbwrap_lock_order_state *lock_order;
@@ -152,7 +153,7 @@ struct db_record *dbwrap_fetch_locked(struct db_context *db,
 	if (lock_order == NULL) {
 		return NULL;
 	}
-	rec = db->fetch_locked(db, mem_ctx, key);
+	rec = db_fn(db, mem_ctx, key);
 	if (rec == NULL) {
 		TALLOC_FREE(lock_order);
 		return NULL;
@@ -161,6 +162,24 @@ struct db_record *dbwrap_fetch_locked(struct db_context *db,
 	return rec;
 }
 
+struct db_record *dbwrap_fetch_locked(struct db_context *db,
+				      TALLOC_CTX *mem_ctx,
+				      TDB_DATA key)
+{
+	return dbwrap_fetch_locked_internal(db, mem_ctx, key,
+					    db->fetch_locked);
+}
+
+struct db_record *dbwrap_try_fetch_locked(struct db_context *db,
+				      TALLOC_CTX *mem_ctx,
+				      TDB_DATA key)
+{
+	return dbwrap_fetch_locked_internal(
+		db, mem_ctx, key,
+		db->try_fetch_locked
+		? db->try_fetch_locked : db->fetch_locked);
+}
+
 struct dbwrap_fetch_state {
 	TALLOC_CTX *mem_ctx;
 	TDB_DATA data;
diff --git a/source3/lib/dbwrap/dbwrap.h b/source3/lib/dbwrap/dbwrap.h
index 386a9fa2a4..9981b6dfd9 100644
--- a/source3/lib/dbwrap/dbwrap.h
+++ b/source3/lib/dbwrap/dbwrap.h
@@ -34,6 +34,9 @@ NTSTATUS dbwrap_record_delete(struct db_record *rec);
 struct db_record *dbwrap_fetch_locked(struct db_context *db,
 				      TALLOC_CTX *mem_ctx,
 				      TDB_DATA key);
+struct db_record *dbwrap_try_fetch_locked(struct db_context *db,
+					  TALLOC_CTX *mem_ctx,
+					  TDB_DATA key);
 
 NTSTATUS dbwrap_delete(struct db_context *db, TDB_DATA key);
 NTSTATUS dbwrap_store(struct db_context *db, TDB_DATA key,
diff --git a/source3/lib/dbwrap/dbwrap_cache.c b/source3/lib/dbwrap/dbwrap_cache.c
index 43c85f7b09..ded85258a3 100644
--- a/source3/lib/dbwrap/dbwrap_cache.c
+++ b/source3/lib/dbwrap/dbwrap_cache.c
@@ -194,6 +194,7 @@ struct db_context *db_open_cache(TALLOC_CTX *mem_ctx,
 	dbwrap_cache_validate(ctx);
 
 	db->fetch_locked = dbwrap_cache_fetch_locked;
+	db->try_fetch_locked = NULL;
 	db->traverse = dbwrap_cache_traverse;
 	db->traverse_read = dbwrap_cache_traverse_read;
 	db->get_seqnum = dbwrap_cache_get_seqnum;
diff --git a/source3/lib/dbwrap/dbwrap_ctdb.c b/source3/lib/dbwrap/dbwrap_ctdb.c
index 47d5dc6929..fe2af3609c 100644
--- a/source3/lib/dbwrap/dbwrap_ctdb.c
+++ b/source3/lib/dbwrap/dbwrap_ctdb.c
@@ -1028,13 +1028,15 @@ static bool db_ctdb_own_record(TDB_DATA ctdb_data, bool read_only)
 
 static struct db_record *fetch_locked_internal(struct db_ctdb_ctx *ctx,
 					       TALLOC_CTX *mem_ctx,
-					       TDB_DATA key)
+					       TDB_DATA key,
+					       bool tryonly)
 {
 	struct db_record *result;
 	struct db_ctdb_rec *crec;
 	NTSTATUS status;
 	TDB_DATA ctdb_data;
 	int migrate_attempts = 0;
+	int lockret;
 
 	if (!(result = talloc(mem_ctx, struct db_record))) {
 		DEBUG(0, ("talloc failed\n"));
@@ -1072,7 +1074,10 @@ again:
 		TALLOC_FREE(keystr);
 	}
 
-	if (tdb_chainlock(ctx->wtdb->tdb, key) != 0) {
+	lockret = tryonly
+		? tdb_chainlock_nonblock(ctx->wtdb->tdb, key)
+		: tdb_chainlock(ctx->wtdb->tdb, key);
+	if (lockret != 0) {
 		DEBUG(3, ("tdb_chainlock failed\n"));
 		TALLOC_FREE(result);
 		return NULL;
@@ -1098,6 +1103,12 @@ again:
 		tdb_chainunlock(ctx->wtdb->tdb, key);
 		talloc_set_destructor(result, NULL);
 
+		if (tryonly && (migrate_attempts != 0)) {
+			DEBUG(5, ("record migrated away again\n"));
+			TALLOC_FREE(result);
+			return NULL;
+		}
+
 		migrate_attempts += 1;
 
 		DEBUG(10, ("ctdb_data.dptr = %p, dmaster = %u (%u) %u\n",
@@ -1163,7 +1174,25 @@ static struct db_record *db_ctdb_fetch_locked(struct db_context *db,
 		return db_ctdb_fetch_locked_persistent(ctx, mem_ctx, key);
 	}
 
-	return fetch_locked_internal(ctx, mem_ctx, key);
+	return fetch_locked_internal(ctx, mem_ctx, key, false);
+}
+
+static struct db_record *db_ctdb_try_fetch_locked(struct db_context *db,
+						  TALLOC_CTX *mem_ctx,
+						  TDB_DATA key)
+{
+	struct db_ctdb_ctx *ctx = talloc_get_type_abort(db->private_data,
+							struct db_ctdb_ctx);
+
+	if (ctx->transaction != NULL) {
+		return db_ctdb_fetch_locked_transaction(ctx, mem_ctx, key);
+	}
+
+	if (db->persistent) {
+		return db_ctdb_fetch_locked_persistent(ctx, mem_ctx, key);
+	}
+
+	return fetch_locked_internal(ctx, mem_ctx, key, true);
 }
 
 /*
@@ -1559,6 +1588,7 @@ struct db_context *db_open_ctdb(TALLOC_CTX *mem_ctx,
 
 	result->private_data = (void *)db_ctdb;
 	result->fetch_locked = db_ctdb_fetch_locked;
+	result->try_fetch_locked = db_ctdb_try_fetch_locked;
 	result->parse_record = db_ctdb_parse_record;
 	result->traverse = db_ctdb_traverse;
 	result->traverse_read = db_ctdb_traverse_read;
diff --git a/source3/lib/dbwrap/dbwrap_file.c b/source3/lib/dbwrap/dbwrap_file.c
index a8a31e3750..ebe768528b 100644
--- a/source3/lib/dbwrap/dbwrap_file.c
+++ b/source3/lib/dbwrap/dbwrap_file.c
@@ -367,6 +367,7 @@ struct db_context *db_open_file(TALLOC_CTX *mem_ctx,
 
 	result->private_data = ctx;
 	result->fetch_locked = db_file_fetch_locked;
+	result->try_fetch_locked = NULL;
 	result->traverse = db_file_traverse;
 	result->traverse_read = db_file_traverse;
 	result->persistent = ((tdb_flags & TDB_CLEAR_IF_FIRST) == 0);
diff --git a/source3/lib/dbwrap/dbwrap_private.h b/source3/lib/dbwrap/dbwrap_private.h
index 111f02dc6b..f95e305a00 100644
--- a/source3/lib/dbwrap/dbwrap_private.h
+++ b/source3/lib/dbwrap/dbwrap_private.h
@@ -36,6 +36,9 @@ struct db_context {
 	struct db_record *(*fetch_locked)(struct db_context *db,
 					  TALLOC_CTX *mem_ctx,
 					  TDB_DATA key);
+	struct db_record *(*try_fetch_locked)(struct db_context *db,
+					      TALLOC_CTX *mem_ctx,
+					      TDB_DATA key);
 	int (*traverse)(struct db_context *db,
 			int (*f)(struct db_record *rec,
 				 void *private_data),
diff --git a/source3/lib/dbwrap/dbwrap_rbt.c b/source3/lib/dbwrap/dbwrap_rbt.c
index 4fbb0bc1a0..95cd3e82e8 100644
--- a/source3/lib/dbwrap/dbwrap_rbt.c
+++ b/source3/lib/dbwrap/dbwrap_rbt.c
@@ -426,6 +426,7 @@ struct db_context *db_open_rbt(TALLOC_CTX *mem_ctx)
 	}
 
 	result->fetch_locked = db_rbt_fetch_locked;
+	result->try_fetch_locked = NULL;
 	result->traverse = db_rbt_traverse;
 	result->traverse_read = db_rbt_traverse;
 	result->get_seqnum = db_rbt_get_seqnum;
diff --git a/source3/lib/dbwrap/dbwrap_tdb.c b/source3/lib/dbwrap/dbwrap_tdb.c
index 46d6cdb8d1..ffad39bed8 100644
--- a/source3/lib/dbwrap/dbwrap_tdb.c
+++ b/source3/lib/dbwrap/dbwrap_tdb.c
@@ -101,20 +101,13 @@ static int db_tdb_fetchlock_parse(TDB_DATA key, TDB_DATA data,
 	return 0;
 }
 
-static struct db_record *db_tdb_fetch_locked(struct db_context *db,
-				     TALLOC_CTX *mem_ctx, TDB_DATA key)
+static struct db_record *db_tdb_fetch_locked_internal(
+	struct db_context *db, TALLOC_CTX *mem_ctx, TDB_DATA key)
 {
 	struct db_tdb_ctx *ctx = talloc_get_type_abort(db->private_data,
 						       struct db_tdb_ctx);
 	struct tdb_fetch_locked_state state;
 
-	db_tdb_log_key("Locking", key);
-
-	if (tdb_chainlock(ctx->wtdb->tdb, key) != 0) {
-		DEBUG(3, ("tdb_chainlock failed\n"));
-		return NULL;
-	}
-
 	state.mem_ctx = mem_ctx;
 	state.result = NULL;
 
@@ -140,6 +133,35 @@ static struct db_record *db_tdb_fetch_locked(struct db_context *db,
 	return state.result;
 }
 
+static struct db_record *db_tdb_fetch_locked(
+	struct db_context *db, TALLOC_CTX *mem_ctx, TDB_DATA key)
+{
+	struct db_tdb_ctx *ctx = talloc_get_type_abort(db->private_data,
+						       struct db_tdb_ctx);
+
+	db_tdb_log_key("Locking", key);
+	if (tdb_chainlock(ctx->wtdb->tdb, key) != 0) {
+		DEBUG(3, ("tdb_chainlock failed\n"));
+		return NULL;
+	}
+	return db_tdb_fetch_locked_internal(db, mem_ctx, key);
+}
+
+static struct db_record *db_tdb_try_fetch_locked(
+	struct db_context *db, TALLOC_CTX *mem_ctx, TDB_DATA key)
+{
+	struct db_tdb_ctx *ctx = talloc_get_type_abort(db->private_data,
+						       struct db_tdb_ctx);
+
+	db_tdb_log_key("Trying to lock", key);
+	if (tdb_chainlock_nonblock(ctx->wtdb->tdb, key) != 0) {
+		DEBUG(3, ("tdb_chainlock_nonblock failed\n"));
+		return NULL;
+	}
+	return db_tdb_fetch_locked_internal(db, mem_ctx, key);
+}
+
+
 static int db_tdb_exists(struct db_context *db, TDB_DATA key)
 {
 	struct db_tdb_ctx *ctx = talloc_get_type_abort(
@@ -373,6 +395,7 @@ struct db_context *db_open_tdb(TALLOC_CTX *mem_ctx,
 	}
 
 	result->fetch_locked = db_tdb_fetch_locked;
+	result->try_fetch_locked = db_tdb_try_fetch_locked;
 	result->traverse = db_tdb_traverse;
 	result->traverse_read = db_tdb_traverse_read;
 	result->parse_record = db_tdb_parse;