clean up the hash entry code a bit. Got rid of lp_shmem_hash_size()

and made it private to the 2 shmem implementations. Added new
shmops->hash_size() function.

Added code to handle the IPC system limits by looping decreasing the
size of the resources (semaphores and shared memory) that we request
until we get under the system limits, which can be quite low on some
systems!

Added checks that the creator of the IPC objects is root. Otherwise we
would be open to a security hole where someone pre-creates the shared
memory segment and attaches.
(This used to be commit 6b6f624b63137d4750200e8cb4961b1402513632)

3 files changed, 88 insertions, 81 deletions

diff --git a/source3/locking/locking_shm.c b/source3/locking/locking_shm.c
index 3abd6b25a7..d299e34caa 100644
--- a/source3/locking/locking_shm.c
+++ b/source3/locking/locking_shm.c
@@ -64,7 +64,7 @@ static int read_only;
 
 
 /* Conversion to hash entry index from device and inode numbers. */
-#define HASH_ENTRY(dev,ino) ((( (uint32)(dev) )* ( (uint32)(ino) )) % lp_shmem_hash_size())
+#define HASH_ENTRY(dev,ino) ((((uint32)(dev)) * ((uint32)(ino))) % shmops->hash_size())
 
 
 /*******************************************************************
@@ -110,15 +110,6 @@ static int shm_get_share_modes(int cnum, int token, uint32 dev, uint32 inode,
 
   *old_shares = 0;
 
-  if(hash_entry > lp_shmem_hash_size() )
-  {
-    DEBUG(0, 
-      ("PANIC ERROR : get_share_modes (FAST_SHARE_MODES): hash_entry %d too large \
-(max = %d)\n",
-      hash_entry, lp_shmem_hash_size() ));
-    return 0;
-  }
-
   mode_array = (int *)shmops->offset2addr(shmops->get_userdef_off());
   
   if(mode_array[hash_entry] == NULL_OFFSET)
@@ -289,15 +280,6 @@ static void shm_del_share_mode(int token, int fnum)
 
   hash_entry = HASH_ENTRY(dev, inode);
 
-  if(hash_entry > lp_shmem_hash_size() )
-  {
-    DEBUG(0,
-      ("PANIC ERROR:del_share_mode (FAST_SHARE_MODES): hash_entry %d too large \
-(max = %d)\n",
-      hash_entry, lp_shmem_hash_size() ));
-    return;
-  }
-
   mode_array = (int *)shmops->offset2addr(shmops->get_userdef_off());
  
   if(mode_array[hash_entry] == NULL_OFFSET)
@@ -428,14 +410,6 @@ static BOOL shm_set_share_mode(int token, int fnum, uint16 port, uint16 op_type)
   inode = fs_p->fd_ptr->inode;
 
   hash_entry = HASH_ENTRY(dev, inode);
-  if(hash_entry > lp_shmem_hash_size() )
-  {
-    DEBUG(0,
-      ("PANIC ERROR:set_share_mode (FAST_SHARE_MODES): hash_entry %d too large \
-(max = %d)\n",
-      hash_entry, lp_shmem_hash_size() ));
-    return False;
-  }
 
   mode_array = (int *)shmops->offset2addr(shmops->get_userdef_off());
 
@@ -550,15 +524,6 @@ static BOOL shm_remove_share_oplock(int fnum, int token)
 
   hash_entry = HASH_ENTRY(dev, inode);
 
-  if(hash_entry > lp_shmem_hash_size() )
-  {
-    DEBUG(0,
-      ("PANIC ERROR:remove_share_oplock (FAST_SHARE_MODES): hash_entry %d too large \
-(max = %d)\n",
-      hash_entry, lp_shmem_hash_size() ));
-    return False;
-  }
-
   mode_array = (int *)shmops->offset2addr(shmops->get_userdef_off());
 
   if(mode_array[hash_entry] == NULL_OFFSET)
@@ -654,7 +619,7 @@ static int shm_share_forall(void (*fn)(share_mode_entry *, char *))
 
 	mode_array = (int *)shmops->offset2addr(shmops->get_userdef_off());
 
-	for( i = 0; i < lp_shmem_hash_size(); i++) {
+	for( i = 0; i < shmops->hash_size(); i++) {
 		shmops->lock_hash_entry(i);
 		if(mode_array[i] == NULL_OFFSET)  {
 			shmops->unlock_hash_entry(i);
@@ -730,19 +695,11 @@ struct share_ops *locking_shm_init(int ronly)
 	read_only = ronly;
 
 #ifdef USE_SYSV_IPC
-	shmops = sysv_shm_open(lp_shmem_size(), read_only);
+	shmops = sysv_shm_open(read_only);
 	if (shmops) return &share_ops;
 #endif
 
-	pstrcpy(shmem_file_name,lp_lockdir());
-	if (!directory_exist(shmem_file_name,NULL)) {
-		if (read_only) return NULL;
-		mkdir(shmem_file_name,0755);
-	}
-	trim_string(shmem_file_name,"","/");
-	if (!*shmem_file_name) return(False);
-	strcat(shmem_file_name, "/SHARE_MEM_FILE");
-	shmops = smb_shm_open(shmem_file_name, lp_shmem_size(), read_only);
+	shmops = smb_shm_open(read_only);
 	if (shmops) return &share_ops;
 
 	return NULL;
diff --git a/source3/locking/shmem.c b/source3/locking/shmem.c
index 72d7c07751..909e8dd435 100644
--- a/source3/locking/shmem.c
+++ b/source3/locking/shmem.c
@@ -41,6 +41,9 @@ extern int DEBUGLEVEL;
 #define SHM_FILE_MODE 0644
 #endif
 
+#define SHMEM_HASH_SIZE 113
+
+
 /* WARNING : offsets are used because mmap() does not guarantee that all processes have the 
    shared memory mapped to the same address */
 
@@ -653,12 +656,6 @@ static BOOL smb_shm_lock_hash_entry( unsigned int entry)
       return False;
     }
 
-  if(entry >= lp_shmem_hash_size())
-    {
-      DEBUG(0,("ERROR smb_shm_lock_hash_entry : hash entry size too big (%d)\n", entry));
-      return False;
-    }
-  
   /* Do an exclusive wait lock on the 4 byte region mapping into this entry  */
   if (fcntl_lock(smb_shm_fd, F_SETLKW, start, sizeof(int), F_WRLCK) == False)
     {
@@ -683,12 +680,6 @@ static BOOL smb_shm_unlock_hash_entry( unsigned int entry )
       return False;
     }
    
-  if(entry >= lp_shmem_hash_size())
-    {
-      DEBUG(0,("ERROR smb_shm_unlock_hash_entry : hash entry size too big (%d)\n", entry));
-      return False;
-    }
-
   /* Do a wait lock on the 4 byte region mapping into this entry  */
   if (fcntl_lock(smb_shm_fd, F_SETLKW, start, sizeof(int), F_UNLCK) == False)
     {
@@ -720,6 +711,13 @@ static BOOL smb_shm_get_usage(int *bytes_free,
    return True;
 }
 
+/*******************************************************************
+hash a number into a hash_entry
+  ******************************************************************/
+static unsigned smb_shm_hash_size(void)
+{
+	return SHMEM_HASH_SIZE;
+}
 
 static struct shmem_ops shmops = {
 	smb_shm_close,
@@ -731,18 +729,30 @@ static struct shmem_ops shmops = {
 	smb_shm_lock_hash_entry,
 	smb_shm_unlock_hash_entry,
 	smb_shm_get_usage,
+	smb_shm_hash_size,
 };
 
 /*******************************************************************
   open the shared memory
   ******************************************************************/
-struct shmem_ops *smb_shm_open(char *file_name, int size, int ronly)
+struct shmem_ops *smb_shm_open(int ronly)
 {
-   int filesize;
-   BOOL created_new = False;
-   BOOL other_processes = True;
-
-   read_only = ronly;
+	pstring file_name;
+	int filesize;
+	BOOL created_new = False;
+	BOOL other_processes = True;
+	int size = lp_shmem_size();
+
+	read_only = ronly;
+
+	pstrcpy(file_name,lp_lockdir());
+	if (!directory_exist(file_name,NULL)) {
+		if (read_only) return NULL;
+		mkdir(file_name,0755);
+	}
+	trim_string(file_name,"","/");
+	if (!*file_name) return(False);
+	strcat(file_name, "/SHARE_MEM_FILE");
    
    DEBUG(5,("smb_shm_open : using shmem file %s to be of size %d\n",file_name,size));
 
@@ -840,7 +850,7 @@ struct shmem_ops *smb_shm_open(char *file_name, int size, int ronly)
    {
       smb_shm_initialize(size);
       /* Create the hash buckets for the share file entries. */
-      smb_shm_create_hash_table( lp_shmem_hash_size() );
+      smb_shm_create_hash_table(SHMEM_HASH_SIZE);
    }
    else if (!smb_shm_validate_header(size) )
    {
diff --git a/source3/locking/shmem_sysv.c b/source3/locking/shmem_sysv.c
index 36d0cca27f..e3f40418d9 100644
--- a/source3/locking/shmem_sysv.c
+++ b/source3/locking/shmem_sysv.c
@@ -39,6 +39,14 @@ extern int DEBUGLEVEL;
 #define IPC_PERMS 0644
 #endif
 
+#ifdef SEMMSL
+#define SHMEM_HASH_SIZE (SEMMSL-1)
+#else
+#define SHMEM_HASH_SIZE 63
+#endif
+
+#define MIN_SHM_SIZE 10240
+
 static int shm_id;
 static int sem_id;
 static int shm_size;
@@ -487,6 +495,16 @@ static BOOL shm_get_usage(int *bytes_free,
 	return True;
 }
 
+
+/*******************************************************************
+hash a number into a hash_entry
+  ******************************************************************/
+static unsigned shm_hash_size(void)
+{
+	return hash_size;
+}
+
+
 static struct shmem_ops shmops = {
 	shm_close,
 	shm_alloc,
@@ -497,12 +515,13 @@ static struct shmem_ops shmops = {
 	shm_lock_hash_entry,
 	shm_unlock_hash_entry,
 	shm_get_usage,
+	shm_hash_size,
 };
 
 /*******************************************************************
   open the shared memory
   ******************************************************************/
-struct shmem_ops *sysv_shm_open(int size, int ronly)
+struct shmem_ops *sysv_shm_open(int ronly)
 {
 	BOOL created_new = False;
 	BOOL other_processes;
@@ -513,17 +532,23 @@ struct shmem_ops *sysv_shm_open(int size, int ronly)
 
 	read_only = ronly;
 
-	shm_size = size;
+	shm_size = lp_shmem_size();
 
-	DEBUG(4,("Trying sysv shmem open of size %d\n", size));
+	DEBUG(4,("Trying sysv shmem open of size %d\n", shm_size));
 
 	/* first the semaphore */
 	sem_id = semget(SEMAPHORE_KEY, 0, 0);
 	if (sem_id == -1) {
 		if (read_only) return NULL;
 
-		sem_id = semget(SEMAPHORE_KEY, lp_shmem_hash_size()+1, 
-				IPC_CREAT | IPC_EXCL | IPC_PERMS);
+		hash_size = SHMEM_HASH_SIZE;
+
+		while (hash_size > 1) {
+			sem_id = semget(SEMAPHORE_KEY, hash_size+1, 
+					IPC_CREAT | IPC_EXCL | IPC_PERMS);
+			if (sem_id != -1 || errno != EINVAL) break;
+			hash_size--;
+		}
 
 		if (sem_id == -1) {
 			DEBUG(0,("Can't create or use semaphore %s\n", 
@@ -532,7 +557,7 @@ struct shmem_ops *sysv_shm_open(int size, int ronly)
 
 		if (sem_id != -1) {
 			su.val = 1;
-			for (i=0;i<lp_shmem_hash_size()+1;i++) {
+			for (i=0;i<hash_size+1;i++) {
 				if (semctl(sem_id, i, SETVAL, su) != 0) {
 					DEBUG(1,("Failed to init semaphore %d\n", i));
 				}
@@ -552,10 +577,16 @@ struct shmem_ops *sysv_shm_open(int size, int ronly)
 	if (semctl(sem_id, 0, IPC_STAT, su) != 0) {
 		DEBUG(0,("ERROR shm_open : can't IPC_STAT\n"));
 	}
-	hash_size = sem_ds.sem_nsems;
-	if (hash_size != lp_shmem_hash_size()+1) {
-		DEBUG(0,("WARNING: nsems=%d\n", hash_size));
+	hash_size = sem_ds.sem_nsems-1;
+
+	if (!read_only) {
+		if (sem_ds.sem_perm.cuid != 0 || sem_ds.sem_perm.cgid != 0) {
+			DEBUG(0,("ERROR: root did not create the semaphore\n"));
+			return NULL;
+		}
 	}
+
+	
 	
 	if (!global_lock())
 		return NULL;
@@ -566,7 +597,12 @@ struct shmem_ops *sysv_shm_open(int size, int ronly)
 	/* if that failed then create one */
 	if (shm_id == -1) {
 		if (read_only) return NULL;
-		shm_id = shmget(SHMEM_KEY, shm_size, IPC_CREAT | IPC_EXCL);
+		while (shm_size > MIN_SHM_SIZE) {
+			shm_id = shmget(SHMEM_KEY, shm_size, 
+					IPC_CREAT | IPC_EXCL | IPC_PERMS);
+			if (shm_id != -1 || errno != EINVAL) break;
+			shm_size *= 0.9;
+		}
 		created_new = (shm_id != -1);
 	}
 	
@@ -592,19 +628,23 @@ struct shmem_ops *sysv_shm_open(int size, int ronly)
 		DEBUG(0,("ERROR shm_open : can't IPC_STAT\n"));
 	}
 
-	/* set the permissions */
 	if (!read_only) {
-		shm_ds.shm_perm.mode = IPC_PERMS;
-		shmctl(shm_id, IPC_SET, &shm_ds);
+		if (shm_ds.shm_perm.cuid != 0 || shm_ds.shm_perm.cgid != 0) {
+			DEBUG(0,("ERROR: root did not create the shmem\n"));
+			global_unlock();
+			return NULL;
+		}
 	}
 
+	shm_size = shm_ds.shm_segsz;
+
 	other_processes = (shm_ds.shm_nattch > 1);
 
 	if (!read_only && !other_processes) {
 		memset((char *)shm_header_p, 0, shm_size);
 		shm_initialize(shm_size);
-		shm_create_hash_table(lp_shmem_hash_size());
-		DEBUG(1,("Initialised IPC area of size %d\n", shm_size));
+		shm_create_hash_table(hash_size);
+		DEBUG(3,("Initialised IPC area of size %d\n", shm_size));
 	} else if (!shm_validate_header(shm_size)) {
 		/* existing file is corrupt, samba admin should remove
                    it by hand */