next_token() was supposed to be a reentrant replacement for strtok(),

but the code suffered from bitrot and is not now reentrant. That means
we can get bizarre behaviour
i've fixed this by making next_token() reentrant and creating a
next_token_nr() that is a small non-reentrant wrapper for those lumps
of code (mostly smbclient) that have come to rely on the non-reentrant
behaviour
(This used to be commit 674ee2f1d12b0afc164a9e9072758fd1c5e54df7)

4 files changed, 75 insertions, 210 deletions

diff --git a/source3/lib/cmd_interp.c b/source3/lib/cmd_interp.c
index 7934dd0ce9..20812d3f9a 100644
--- a/source3/lib/cmd_interp.c
+++ b/source3/lib/cmd_interp.c
@@ -292,14 +292,14 @@ static BOOL get_cmd_args(char *line)
 	cmd_argv = NULL;
 
 	/* get the first part of the command */
-	if (!next_token(&ptr, tok, NULL, sizeof(tok)))
+	if (!next_token_nr(&ptr, tok, NULL, sizeof(tok)))
 		return False;
 
 	do
 	{
 		add_chars_to_array(&cmd_argc, &cmd_argv, tok);
 	}
-	while (next_token(NULL, tok, NULL, sizeof(tok)));
+	while (next_token_nr(NULL, tok, NULL, sizeof(tok)));
 
 	add_chars_to_array(&cmd_argc, &cmd_argv, NULL);
 
diff --git a/source3/lib/util.c b/source3/lib/util.c
index 0c0d3f70a7..37bb1693d8 100644
--- a/source3/lib/util.c
+++ b/source3/lib/util.c
@@ -1658,7 +1658,7 @@ BOOL reg_split_key(char *full_keyname, uint32 *reg_type, char *key_name)
 		return False;
 	}
 	
-	if (next_token(NULL, tmp, "\n\r", sizeof(tmp)))
+	if (next_token(&full_keyname, tmp, "\n\r", sizeof(tmp)))
 	{
 		fstrcpy(key_name, tmp);
 	}
diff --git a/source3/lib/util_str.c b/source3/lib/util_str.c
index 0f32fda63c..f1376fe0f7 100644
--- a/source3/lib/util_str.c
+++ b/source3/lib/util_str.c
@@ -23,13 +23,6 @@
 
 extern int DEBUGLEVEL;
 
-static char *last_ptr=NULL;
-
-void set_first_token(char *ptr)
-{
-	last_ptr = ptr;
-}
-
 /****************************************************************************
   Get the next token from a string, return False if none found
   handles double-quotes. 
@@ -38,77 +31,99 @@ Extensively modified by Andrew.Tridgell@anu.edu.au
 ****************************************************************************/
 BOOL next_token(char **ptr,char *buff,char *sep, size_t bufsize)
 {
-  char *s;
-  BOOL quoted;
-  size_t len=1;
+	char *s;
+	BOOL quoted;
+	size_t len=1;
 
-  if (!ptr) ptr = &last_ptr;
-  if (!ptr) return(False);
+	if (!ptr) return(False);
 
-  s = *ptr;
+	s = *ptr;
 
-  /* default to simple separators */
-  if (!sep) sep = " \t\n\r";
+	/* default to simple separators */
+	if (!sep) sep = " \t\n\r";
 
-  /* find the first non sep char */
-  while(*s && strchr(sep,*s)) s++;
+	/* find the first non sep char */
+	while (*s && strchr(sep,*s)) s++;
+	
+	/* nothing left? */
+	if (! *s) return(False);
+	
+	/* copy over the token */
+	for (quoted = False; len < bufsize && *s && (quoted || !strchr(sep,*s)); s++) {
+		if (*s == '\"') {
+			quoted = !quoted;
+		} else {
+			len++;
+			*buff++ = *s;
+		}
+	}
+	
+	*ptr = (*s) ? s+1 : s;  
+	*buff = 0;
+	
+	return(True);
+}
 
-  /* nothing left? */
-  if (! *s) return(False);
 
-  /* copy over the token */
-  for (quoted = False; len < bufsize && *s && (quoted || !strchr(sep,*s)); s++)
-    {
-	    if (*s == '\"') {
-		    quoted = !quoted;
-	    } else {
-		    len++;
-		    *buff++ = *s;
-	    }
-    }
 
-  *ptr = (*s) ? s+1 : s;  
-  *buff = 0;
-  last_ptr = *ptr;
+/****************************************************************************
+This is like next_token but is not re-entrant and "remembers" the first 
+parameter so you can pass NULL. This is useful for user interface code
+but beware the fact that it is not re-entrant!
+****************************************************************************/
+static char *last_ptr=NULL;
 
-  return(True);
+BOOL next_token_nr(char **ptr,char *buff,char *sep, size_t bufsize)
+{
+	BOOL ret;
+	if (!ptr) ptr = &last_ptr;
+
+	ret = next_token(ptr, buff, sep, bufsize);
+	last_ptr = *ptr;
+	return ret;	
 }
 
+void set_first_token(char *ptr)
+{
+	last_ptr = ptr;
+}
+
+
 /****************************************************************************
 Convert list of tokens to array; dependent on above routine.
 Uses last_ptr from above - bit of a hack.
 ****************************************************************************/
 char **toktocliplist(int *ctok, char *sep)
 {
-  char *s=last_ptr;
-  int ictok=0;
-  char **ret, **iret;
+	char *s=last_ptr;
+	int ictok=0;
+	char **ret, **iret;
 
-  if (!sep) sep = " \t\n\r";
+	if (!sep) sep = " \t\n\r";
 
-  while(*s && strchr(sep,*s)) s++;
+	while(*s && strchr(sep,*s)) s++;
 
-  /* nothing left? */
-  if (!*s) return(NULL);
+	/* nothing left? */
+	if (!*s) return(NULL);
 
-  do {
-    ictok++;
-    while(*s && (!strchr(sep,*s))) s++;
-    while(*s && strchr(sep,*s)) *s++=0;
-  } while(*s);
-
-  *ctok=ictok;
-  s=last_ptr;
-
-  if (!(ret=iret=malloc(ictok*sizeof(char *)))) return NULL;
-  
-  while(ictok--) {    
-    *iret++=s;
-    while(*s++);
-    while(!*s) s++;
-  }
+	do {
+		ictok++;
+		while(*s && (!strchr(sep,*s))) s++;
+		while(*s && strchr(sep,*s)) *s++=0;
+	} while(*s);
+	
+	*ctok=ictok;
+	s=last_ptr;
+	
+	if (!(ret=iret=malloc(ictok*sizeof(char *)))) return NULL;
+	
+	while(ictok--) {    
+		*iret++=s;
+		while(*s++);
+		while(!*s) s++;
+	}
 
-  return ret;
+	return ret;
 }
 
 
diff --git a/source3/lib/util_unistr.c b/source3/lib/util_unistr.c
index f6bb7e8068..7058665145 100644
--- a/source3/lib/util_unistr.c
+++ b/source3/lib/util_unistr.c
@@ -1169,133 +1169,6 @@ smb_ucs2_t tolower_w( smb_ucs2_t val )
 	return map_table_lower(val);
 }
 
-static smb_ucs2_t *last_ptr = NULL;
-
-void set_first_token_w(smb_ucs2_t *ptr)
-{
-	last_ptr = ptr;
-}
-
-/****************************************************************************
- Get the next token from a string, return False if none found
- handles double-quotes. 
- Based on a routine by GJC@VILLAGE.COM. 
- Extensively modified by Andrew.Tridgell@anu.edu.au
- bufsize is in bytes.
-****************************************************************************/
-
-static smb_ucs2_t sep_list[] = { (smb_ucs2_t)' ', (smb_ucs2_t)'\t',  (smb_ucs2_t)'\n',  (smb_ucs2_t)'\r', 0};
-static smb_ucs2_t quotechar = (smb_ucs2_t)'\"';
-
-BOOL next_token_w(smb_ucs2_t **ptr, smb_ucs2_t *buff, smb_ucs2_t *sep, size_t bufsize)
-{
-	smb_ucs2_t *s;
-	BOOL quoted;
-	size_t len=1;
-
-	/*
-	 * Convert bufsize to smb_ucs2_t units.
-	 */
-
-	bufsize /= sizeof(smb_ucs2_t);
-
-	if (!ptr)
-		ptr = &last_ptr;
-	if (!ptr)
-		return(False);
-
-	s = *ptr;
-
-	/*
-	 * Default to simple separators.
-	 */
-
-	if (!sep)
-		sep = sep_list;
-
-	/*
-	 * Find the first non sep char.
-	 */
-
-	while(*s && strchr_w(sep,*s))
-		s++;
-
-	/*
-	 * Nothing left ?
-	 */
-
-	if (!*s)
-		return(False);
-
-	/*
-	 * Copy over the token.
-	 */
-
-	for (quoted = False; len < bufsize && *s && (quoted || !strchr_w(sep,*s)); s++) {
-		if (*s == quotechar) {
-			quoted = !quoted;
-		} else {
-			len++;
-			*buff++ = *s;
-		}
-	}
-
-	*ptr = (*s) ? s+1 : s;  
-	*buff = 0;
-	last_ptr = *ptr;
-
-	return(True);
-}
-
-/****************************************************************************
- Convert list of tokens to array; dependent on above routine.
- Uses last_ptr from above - bit of a hack.
-****************************************************************************/
-
-smb_ucs2_t **toktocliplist_w(int *ctok, smb_ucs2_t *sep)
-{
-	smb_ucs2_t *s=last_ptr;
-	int ictok=0;
-	smb_ucs2_t **ret, **iret;
-
-	if (!sep)
-		sep = sep_list;
-
-	while(*s && strchr_w(sep,*s))
-		s++;
-
-	/*
-	 * Nothing left ?
-	 */
-
-	if (!*s)
-		return(NULL);
-
-	do {
-		ictok++;
-		while(*s && (!strchr_w(sep,*s)))
-			s++;
-		while(*s && strchr_w(sep,*s))
-			*s++=0;
-	} while(*s);
-
-	*ctok = ictok;
-	s = last_ptr;
-
-	if (!(ret=iret=malloc(ictok*sizeof(smb_ucs2_t *))))
-		return NULL;
-  
-	while(ictok--) {
-		*iret++=s;
-		while(*s++)
-			;
-		while(!*s)
-			s++;
-	}
-
-	return ret;
-}
-
 /*******************************************************************
  Case insensitive string compararison.
 ********************************************************************/
@@ -1715,29 +1588,6 @@ size_t strhex_to_str_w(char *p, size_t len, const smb_ucs2_t *strhex)
 	return num_chars;
 }
 
-/****************************************************************************
- Check if a string is part of a list.
-****************************************************************************/
-
-BOOL in_list_w(smb_ucs2_t *s,smb_ucs2_t *list,BOOL casesensitive)
-{
-	wpstring tok;
-	smb_ucs2_t *p=list;
-
-	if (!list)
-		return(False);
-
-	while (next_token_w(&p,tok,LIST_SEP_W,sizeof(tok))) {
-		if (casesensitive) {
-			if (strcmp_w(tok,s) == 0)
-				return(True);
-		} else {
-			if (StrCaseCmp_w(tok,s) == 0)
-				return(True);
-		}
-	}
-	return(False);
-}
 
 /* This is used to prevent lots of mallocs of size 2 */
 static smb_ucs2_t *null_string = NULL;