Merge Herb's idmap endian fix.

Jeremy.
(This used to be commit 7ddad4061a1b7ed25e4d6471c7a1f8f97a98ed37)

3 files changed, 16 insertions, 12 deletions

diff --git a/source3/nsswitch/winbindd_idmap.c b/source3/nsswitch/winbindd_idmap.c
index 60ea188b1d..0594f61680 100644
--- a/source3/nsswitch/winbindd_idmap.c
+++ b/source3/nsswitch/winbindd_idmap.c
@@ -364,6 +364,7 @@ fail:
 static BOOL idmap_convert(const char *idmap_name)
 {
 	int32 vers = tdb_fetch_int32(idmap_tdb, "IDMAP_VERSION");
+	BOOL bigendianheader = (idmap_tdb->flags & TDB_BIGENDIAN) ? True : False;
 
 	if (vers == IDMAP_VERSION)
 		return True;
@@ -374,23 +375,20 @@ static BOOL idmap_convert(const char *idmap_name)
 		return False;
 #endif
 
-	if ((vers == -1) || (IREV(vers) == IDMAP_VERSION)) {
-		/* Arrggghh ! Bytereversed or missing - make order independent ! */
+	if (((vers == -1) && bigendianheader) || (IREV(vers) == IDMAP_VERSION)) {
+		/* Arrggghh ! Bytereversed or old big-endian - make order independent ! */
 		/*
-		 * If the header needed to be converted then the
-		 * high and low records may have been created on a
-		 * foreign endian machine and will need byte-reversing.
+		 * high and low records were created on a
+		 * big endian machine and will need byte-reversing.
 		 */
 
-		BOOL bytereverse_needed = (idmap_tdb->flags & TDB_CONVERT);
 		int32 wm;
 
 		wm = tdb_fetch_int32(idmap_tdb, HWM_USER);
 
-		if (wm != -1 && bytereverse_needed) {
-			/* A record existed and it was from a foreign endian machine. */
+		if (wm != -1) {
 			wm = IREV(wm);
-		} else if (wm == -1)
+		}  else
 			wm = server_state.uid_low;
 
 		if (tdb_store_int32(idmap_tdb, HWM_USER, wm) == -1) {
@@ -399,10 +397,9 @@ static BOOL idmap_convert(const char *idmap_name)
 		}
 
 		wm = tdb_fetch_int32(idmap_tdb, HWM_GROUP);
-		if (wm != -1 && bytereverse_needed) {
-			/* A record existed and it was from a foreign endian machine. */
+		if (wm != -1) {
 			wm = IREV(wm);
-		} else if (wm == -1)
+		} else
 			wm = server_state.gid_low;
 
 		if (tdb_store_int32(idmap_tdb, HWM_GROUP, wm) == -1) {
diff --git a/source3/tdb/tdb.c b/source3/tdb/tdb.c
index 5c0fd436e3..c937090de4 100644
--- a/source3/tdb/tdb.c
+++ b/source3/tdb/tdb.c
@@ -1471,6 +1471,8 @@ TDB_CONTEXT *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 	TDB_CONTEXT *tdb;
 	struct stat st;
 	int rev = 0, locked;
+	unsigned char *vp;
+	u32 vertest;
 
 	if (!(tdb = calloc(1, sizeof *tdb))) {
 		/* Can't log this */
@@ -1548,6 +1550,10 @@ TDB_CONTEXT *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 		}
 		rev = (tdb->flags & TDB_CONVERT);
 	}
+	vp = (unsigned char *)&tdb->header.version;
+	vertest = (((u32)vp[0]) << 24) | (((u32)vp[1]) << 16) |
+		  (((u32)vp[2]) << 8) | (u32)vp[3];
+	tdb->flags |= (vertest==TDB_VERSION) ? TDB_BIGENDIAN : 0;
 	if (!rev)
 		tdb->flags &= ~TDB_CONVERT;
 	else {
diff --git a/source3/tdb/tdb.h b/source3/tdb/tdb.h
index 54cde10d95..8cc908703f 100644
--- a/source3/tdb/tdb.h
+++ b/source3/tdb/tdb.h
@@ -38,6 +38,7 @@ extern "C" {
 #define TDB_NOLOCK   4 /* don't do any locking */
 #define TDB_NOMMAP   8 /* don't use mmap */
 #define TDB_CONVERT 16 /* convert endian (internal use) */
+#define TDB_BIGENDIAN 32 /* header is big-endian (internal use) */
 
 #define TDB_ERRCODE(code, ret) ((tdb->ecode = (code)), ret)