Merge commit 'origin/v4-0-test' into 4-0-local

(This used to be commit 51422414bed28f425197519298c6b04a9f4e2c4e)

9 files changed, 67 insertions, 112 deletions

diff --git a/source4/lib/tdb/common/freelist.c b/source4/lib/tdb/common/freelist.c
index c086c151fa..2f2a4c379b 100644
--- a/source4/lib/tdb/common/freelist.c
+++ b/source4/lib/tdb/common/freelist.c
@@ -208,62 +208,61 @@ update:
 }
 
 
+
 /* 
    the core of tdb_allocate - called when we have decided which
    free list entry to use
+
+   Note that we try to allocate by grabbing data from the end of an existing record,
+   not the beginning. This is so the left merge in a free is more likely to be
+   able to free up the record without fragmentation
  */
-static tdb_off_t tdb_allocate_ofs(struct tdb_context *tdb, tdb_len_t length, tdb_off_t rec_ptr,
-				struct list_struct *rec, tdb_off_t last_ptr)
+static tdb_off_t tdb_allocate_ofs(struct tdb_context *tdb, 
+				  tdb_len_t length, tdb_off_t rec_ptr,
+				  struct list_struct *rec, tdb_off_t last_ptr)
 {
-	struct list_struct newrec;
-	tdb_off_t newrec_ptr;
+#define MIN_REC_SIZE (sizeof(struct list_struct) + sizeof(tdb_off_t) + 8)
 
-	memset(&newrec, '\0', sizeof(newrec));
+	if (rec->rec_len < length + MIN_REC_SIZE) {
+		/* we have to grab the whole record */
 
-	/* found it - now possibly split it up  */
-	if (rec->rec_len > length + MIN_REC_SIZE) {
-		/* Length of left piece */
-		length = TDB_ALIGN(length, TDB_ALIGNMENT);
-		
-		/* Right piece to go on free list */
-		newrec.rec_len = rec->rec_len - (sizeof(*rec) + length);
-		newrec_ptr = rec_ptr + sizeof(*rec) + length;
-		
-		/* And left record is shortened */
-		rec->rec_len = length;
-	} else {
-		newrec_ptr = 0;
+		/* unlink it from the previous record */
+		if (tdb_ofs_write(tdb, last_ptr, &rec->next) == -1) {
+			return 0;
+		}
+
+		/* mark it not free */
+		rec->magic = TDB_MAGIC;
+		if (tdb_rec_write(tdb, rec_ptr, rec) == -1) {
+			return 0;
+		}
+		return rec_ptr;
+	}
+
+	/* we're going to just shorten the existing record */
+	rec->rec_len -= (length + sizeof(*rec));
+	if (tdb_rec_write(tdb, rec_ptr, rec) == -1) {
+		return 0;
 	}
-	
-	/* Remove allocated record from the free list */
-	if (tdb_ofs_write(tdb, last_ptr, &rec->next) == -1) {
+	if (update_tailer(tdb, rec_ptr, rec) == -1) {
 		return 0;
 	}
-	
-	/* Update header: do this before we drop alloc
-	   lock, otherwise tdb_free() might try to
-	   merge with us, thinking we're free.
-	   (Thanks Jeremy Allison). */
+
+	/* and setup the new record */
+	rec_ptr += sizeof(*rec) + rec->rec_len;	
+
+	memset(rec, '\0', sizeof(*rec));
+	rec->rec_len = length;
 	rec->magic = TDB_MAGIC;
+
 	if (tdb_rec_write(tdb, rec_ptr, rec) == -1) {
 		return 0;
 	}
-	
-	/* Did we create new block? */
-	if (newrec_ptr) {
-		/* Update allocated record tailer (we
-		   shortened it). */
-		if (update_tailer(tdb, rec_ptr, rec) == -1) {
-			return 0;
-		}
-		
-		/* Free new record */
-		if (tdb_free(tdb, newrec_ptr, &newrec) == -1) {
-			return 0;
-		}
+
+	if (update_tailer(tdb, rec_ptr, rec) == -1) {
+		return 0;
 	}
-	
-	/* all done - return the new record offset */
+
 	return rec_ptr;
 }
 
@@ -287,6 +286,7 @@ tdb_off_t tdb_allocate(struct tdb_context *tdb, tdb_len_t length, struct list_st
 
 	/* Extra bytes required for tailer */
 	length += sizeof(tdb_off_t);
+	length = TDB_ALIGN(length, TDB_ALIGNMENT);
 
  again:
 	last_ptr = FREELIST_TOP;
@@ -343,7 +343,8 @@ tdb_off_t tdb_allocate(struct tdb_context *tdb, tdb_len_t length, struct list_st
 			goto fail;
 		}
 
-		newrec_ptr = tdb_allocate_ofs(tdb, length, bestfit.rec_ptr, rec, bestfit.last_ptr);
+		newrec_ptr = tdb_allocate_ofs(tdb, length, bestfit.rec_ptr, 
+					      rec, bestfit.last_ptr);
 		tdb_unlock(tdb, -1, F_WRLCK);
 		return newrec_ptr;
 	}
diff --git a/source4/lib/tdb/common/open.c b/source4/lib/tdb/common/open.c
index 6bd8fda2bf..b19e4cea29 100644
--- a/source4/lib/tdb/common/open.c
+++ b/source4/lib/tdb/common/open.c
@@ -179,9 +179,7 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 		tdb->page_size = 0x2000;
 	}
 
-	if (open_flags & TDB_VOLATILE) {
-		tdb->max_dead_records = 5;
-	}
+	tdb->max_dead_records = (tdb_flags & TDB_VOLATILE) ? 5 : 0;
 
 	if ((open_flags & O_ACCMODE) == O_WRONLY) {
 		TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_open_ex: can't open tdb %s write-only\n",
@@ -229,6 +227,7 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 
 	/* we need to zero database if we are the only one with it open */
 	if ((tdb_flags & TDB_CLEAR_IF_FIRST) &&
+	    (!tdb->read_only) &&
 	    (locked = (tdb->methods->tdb_brlock(tdb, ACTIVE_LOCK, F_WRLCK, F_SETLK, 0, 1) == 0))) {
 		open_flags |= O_CREAT;
 		if (ftruncate(tdb->fd, 0) == -1) {
@@ -288,7 +287,6 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 	tdb->map_size = st.st_size;
 	tdb->device = st.st_dev;
 	tdb->inode = st.st_ino;
-	tdb->max_dead_records = 0;
 	tdb_mmap(tdb);
 	if (locked) {
 		if (tdb->methods->tdb_brlock(tdb, ACTIVE_LOCK, F_UNLCK, F_SETLK, 0, 1) == -1) {
diff --git a/source4/lib/tdb/common/tdb.c b/source4/lib/tdb/common/tdb.c
index fd4e1cc8af..ea5d9ccc60 100644
--- a/source4/lib/tdb/common/tdb.c
+++ b/source4/lib/tdb/common/tdb.c
@@ -743,45 +743,3 @@ failed:
 	tdb_unlockall(tdb);
 	return -1;
 }
-
-
-/* 
-   validate the integrity of all tdb hash chains. Useful when debugging
- */
-int tdb_validate(struct tdb_context *tdb)
-{
-	int h;
-	for (h=-1;h<(int)tdb->header.hash_size;h++) {
-		tdb_off_t rec_ptr;
-		uint32_t count = 0;
-		if (tdb_ofs_read(tdb, TDB_HASH_TOP(h), &rec_ptr) == -1) {
-			TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_validate: failed ofs_read at top of hash %d\n", h));
-			return -1;
-		}
-		while (rec_ptr) {
-			struct list_struct r;
-			tdb_off_t size;
-
-			if (tdb_rec_read(tdb, rec_ptr, &r) == -1) {
-				TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_validate: failed rec_read h=%d rec_ptr=%u count=%u\n",
-					 h, rec_ptr, count));
-				return -1;
-			}
-			if (tdb_ofs_read(tdb, rec_ptr + sizeof(r) + r.rec_len - sizeof(tdb_off_t), &size) == -1) {
-				TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_validate: failed ofs_read h=%d rec_ptr=%u count=%u\n",
-					 h, rec_ptr, count));
-				return -1;
-			}
-			if (size != r.rec_len + sizeof(r)) {
-				TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_validate: failed size check size=%u h=%d rec_ptr=%u count=%u\n",
-					 size, h, rec_ptr, count));
-				return -1;
-			}
-			rec_ptr = r.next;
-			count++;
-		}		
-	}
-	return 0;
-}
-
-
diff --git a/source4/lib/tdb/common/tdb_private.h b/source4/lib/tdb/common/tdb_private.h
index 63a6d04e72..ffac89ff0e 100644
--- a/source4/lib/tdb/common/tdb_private.h
+++ b/source4/lib/tdb/common/tdb_private.h
@@ -49,7 +49,6 @@ typedef uint32_t tdb_off_t;
 #define TDB_DEAD_MAGIC (0xFEE1DEAD)
 #define TDB_RECOVERY_MAGIC (0xf53bc0e7U)
 #define TDB_ALIGNMENT 4
-#define MIN_REC_SIZE (2*sizeof(struct list_struct) + TDB_ALIGNMENT)
 #define DEFAULT_HASH_SIZE 131
 #define FREELIST_TOP (sizeof(struct tdb_header))
 #define TDB_ALIGN(x,a) (((x) + (a)-1) & ~((a)-1))
@@ -178,6 +177,7 @@ struct tdb_context {
 int tdb_munmap(struct tdb_context *tdb);
 void tdb_mmap(struct tdb_context *tdb);
 int tdb_lock(struct tdb_context *tdb, int list, int ltype);
+int tdb_lock_nonblock(struct tdb_context *tdb, int list, int ltype);
 int tdb_unlock(struct tdb_context *tdb, int list, int ltype);
 int tdb_brlock(struct tdb_context *tdb, tdb_off_t offset, int rw_type, int lck_type, int probe, size_t len);
 int tdb_transaction_lock(struct tdb_context *tdb, int ltype);
diff --git a/source4/lib/tdb/common/transaction.c b/source4/lib/tdb/common/transaction.c
index 0ecfb9b7ff..c3e7a4e2c0 100644
--- a/source4/lib/tdb/common/transaction.c
+++ b/source4/lib/tdb/common/transaction.c
@@ -219,9 +219,12 @@ static int transaction_write(struct tdb_context *tdb, tdb_off_t off,
 		uint8_t **new_blocks;
 		/* expand the blocks array */
 		if (tdb->transaction->blocks == NULL) {
-			new_blocks = malloc((blk+1)*sizeof(uint8_t *));
+			new_blocks = (uint8_t **)malloc(
+				(blk+1)*sizeof(uint8_t *));
 		} else {
-			new_blocks = realloc(tdb->transaction->blocks, (blk+1)*sizeof(uint8_t *));
+			new_blocks = (uint8_t **)realloc(
+				tdb->transaction->blocks,
+				(blk+1)*sizeof(uint8_t *));
 		}
 		if (new_blocks == NULL) {
 			tdb->ecode = TDB_ERR_OOM;
@@ -316,25 +319,15 @@ static int transaction_write_existing(struct tdb_context *tdb, tdb_off_t off,
 		return 0;
 	}
 
-	/* overwrite part of an existing block */
-	if (buf == NULL) {
-		memset(tdb->transaction->blocks[blk] + off, 0, len);
-	} else {
-		memcpy(tdb->transaction->blocks[blk] + off, buf, len);
-	}
-	if (blk == tdb->transaction->num_blocks-1) {
-		if (len + off > tdb->transaction->last_block_size) {
-			tdb->transaction->last_block_size = len + off;
-		}
+	if (blk == tdb->transaction->num_blocks-1 &&
+	    off + len > tdb->transaction->last_block_size) {
+		len = tdb->transaction->last_block_size - off;
 	}
 
-	return 0;
+	/* overwrite part of an existing block */
+	memcpy(tdb->transaction->blocks[blk] + off, buf, len);
 
-fail:
-	TDB_LOG((tdb, TDB_DEBUG_FATAL, "transaction_write: failed at off=%d len=%d\n", 
-		 (blk*tdb->transaction->block_size) + off, len));
-	tdb->transaction->transaction_error = 1;
-	return -1;
+	return 0;
 }
 
 
diff --git a/source4/lib/tdb/common/traverse.c b/source4/lib/tdb/common/traverse.c
index 2bde1270a0..07b0c23858 100644
--- a/source4/lib/tdb/common/traverse.c
+++ b/source4/lib/tdb/common/traverse.c
@@ -223,6 +223,9 @@ int tdb_traverse_read(struct tdb_context *tdb,
 /*
   a write style traverse - needs to get the transaction lock to
   prevent deadlocks
+
+  WARNING: The data buffer given to the callback fn does NOT meet the
+  alignment restrictions malloc gives you.
 */
 int tdb_traverse(struct tdb_context *tdb, 
 		 tdb_traverse_func fn, void *private_data)
diff --git a/source4/lib/tdb/docs/README b/source4/lib/tdb/docs/README
index b31ce36ab1..63fcf5e049 100644
--- a/source4/lib/tdb/docs/README
+++ b/source4/lib/tdb/docs/README
@@ -130,6 +130,9 @@ int tdb_traverse(TDB_CONTEXT *tdb, int (*fn)(TDB_CONTEXT *tdb,
    a non-zero return value from fn() indicates that the traversal
    should stop. Traversal callbacks may not start transactions.
 
+   WARNING: The data buffer given to the callback fn does NOT meet the
+   alignment restrictions malloc gives you.
+
 ----------------------------------------------------------------------
 int tdb_traverse_read(TDB_CONTEXT *tdb, int (*fn)(TDB_CONTEXT *tdb,
                      TDB_DATA key, TDB_DATA dbuf, void *state), void *state);
diff --git a/source4/lib/tdb/include/tdb.h b/source4/lib/tdb/include/tdb.h
index 0058d55793..0008085de5 100644
--- a/source4/lib/tdb/include/tdb.h
+++ b/source4/lib/tdb/include/tdb.h
@@ -32,9 +32,9 @@ extern "C" {
 
 
 /* flags to tdb_store() */
-#define TDB_REPLACE 1
-#define TDB_INSERT 2
-#define TDB_MODIFY 3
+#define TDB_REPLACE 1		/* Unused */
+#define TDB_INSERT 2 		/* Don't overwrite an existing entry */
+#define TDB_MODIFY 3		/* Don't create an existing entry    */
 
 /* flags for tdb_open() */
 #define TDB_DEFAULT 0 /* just a readability place holder */
@@ -157,7 +157,6 @@ int tdb_printfreelist(struct tdb_context *tdb);
 int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries);
 int tdb_wipe_all(struct tdb_context *tdb);
 int tdb_freelist_size(struct tdb_context *tdb);
-int tdb_validate(struct tdb_context *tdb);
 
 extern TDB_DATA tdb_null;
 
diff --git a/source4/lib/tdb/tools/tdbtool.c b/source4/lib/tdb/tools/tdbtool.c
index 79435a3571..d104ccd7c4 100644
--- a/source4/lib/tdb/tools/tdbtool.c
+++ b/source4/lib/tdb/tools/tdbtool.c
@@ -135,7 +135,7 @@ static void print_data(const char *buf,int len)
 	if (len<=0) return;
 	printf("[%03X] ",i);
 	for (i=0;i<len;) {
-		printf("%02X ",(int)buf[i]);
+		printf("%02X ",(int)((unsigned char)buf[i]));
 		i++;
 		if (i%8 == 0) printf(" ");
 		if (i%16 == 0) {