r2857: this commit gets rid of smb_ucs2_t, wpstring and fpstring, plus lots of associated functions.

The motivation for this change was to avoid having to convert to/from
ucs2 strings for so many operations. Doing that was slow, used many
static buffers, and was also incorrect as it didn't cope properly with
unicode codepoints above 65536 (which could not be represented
correctly as smb_ucs2_t chars)

The two core functions that allowed this change are next_codepoint()
and push_codepoint(). These functions allow you to correctly walk a
arbitrary multi-byte string a character at a time without converting
the whole string to ucs2.

While doing this cleanup I also fixed several ucs2 string handling
bugs. See the commit for details.

The following code (which counts the number of occuraces of 'c' in a
string) shows how to use the new interface:

size_t count_chars(const char *s, char c)
{
	size_t count = 0;

	while (*s) {
		size_t size;
		codepoint_t c2 = next_codepoint(s, &size);
		if (c2 == c) count++;
		s += size;
	}

	return count;
}
(This used to be commit 814881f0e50019196b3aa9fbe4aeadbb98172040)

4 files changed, 65 insertions, 19 deletions

diff --git a/source4/torture/basic/utable.c b/source4/torture/basic/utable.c
index 6faf020ef9..8047e5a468 100644
--- a/source4/torture/basic/utable.c
+++ b/source4/torture/basic/utable.c
@@ -26,7 +26,7 @@ BOOL torture_utable(int dummy)
 	fstring fname;
 	const char *alt_name;
 	int fnum;
-	smb_ucs2_t c2;
+	char c2[4];
 	int c, len, fd;
 	int chars_allowed=0, alt_allowed=0;
 	uint8_t valid[0x10000];
@@ -47,11 +47,11 @@ BOOL torture_utable(int dummy)
 	for (c=1; c < 0x10000; c++) {
 		char *p;
 
-		SSVAL(&c2, 0, c);
+		SSVAL(c2, 0, c);
 		fstrcpy(fname, "\\utable\\x");
 		p = fname+strlen(fname);
 		len = convert_string(CH_UTF16, CH_UNIX, 
-				     &c2, 2, 
+				     c2, 2, 
 				     p, sizeof(fname)-strlen(fname));
 		p[len] = 0;
 		fstrcat(fname,"_a_long_extension");
@@ -99,16 +99,16 @@ BOOL torture_utable(int dummy)
 static char *form_name(int c)
 {
 	static fstring fname;
-	smb_ucs2_t c2;
+	char c2[4];
 	char *p;
 	int len;
 
 	fstrcpy(fname, "\\utable\\");
 	p = fname+strlen(fname);
-	SSVAL(&c2, 0, c);
+	SSVAL(c2, 0, c);
 
 	len = convert_string(CH_UTF16, CH_UNIX, 
-			     &c2, 2, 
+			     c2, 2, 
 			     p, sizeof(fname)-strlen(fname));
 	p[len] = 0;
 	return fname;
@@ -121,7 +121,7 @@ BOOL torture_casetable(int dummy)
 	int fnum;
 	int c, i;
 #define MAX_EQUIVALENCE 8
-	smb_ucs2_t equiv[0x10000][MAX_EQUIVALENCE];
+	codepoint_t equiv[0x10000][MAX_EQUIVALENCE];
 	printf("starting casetable\n");
 
 	if (!torture_open_connection(&cli)) {
diff --git a/source4/torture/local/iconv.c b/source4/torture/local/iconv.c
index 84bca802e9..606ac3da5d 100644
--- a/source4/torture/local/iconv.c
+++ b/source4/torture/local/iconv.c
@@ -27,8 +27,8 @@
 /*
   generate a UTF-16LE buffer for a given unicode codepoint
 */
-static int gen_codepoint(unsigned int codepoint,
-			  char *buf, size_t *size)
+static int gen_codepoint_utf16(unsigned int codepoint,
+			       char *buf, size_t *size)
 {
 	static iconv_t cd;
 	uint8_t in[4];
@@ -254,6 +254,45 @@ static int test_buffer(uint8_t *inbuf, size_t size, const char *charset)
 	return ok;
 }
 
+
+/*
+  test the push_codepoint() and next_codepoint() functions for a given
+  codepoint
+*/
+static int test_codepoint(unsigned int codepoint)
+{
+	uint8_t buf[10];
+	size_t size, size2;
+	codepoint_t c;
+
+	size = push_codepoint(buf, codepoint);
+	if (size == -1) {
+		if (codepoint < 0xd800 || codepoint > 0x10000) {
+			return 0;
+		}
+		return 1;
+	}
+	buf[size] = random();
+	buf[size+1] = random();
+	buf[size+2] = random();
+	buf[size+3] = random();
+
+	c = next_codepoint(buf, &size2);
+
+	if (c != codepoint) {
+		printf("next_codepoint(%u) failed - gave %u\n", codepoint, c);
+		return 0;
+	}
+
+	if (size2 != size) {
+		printf("next_codepoint(%u) gave wrong size %d (should be %d)\n", 
+		       codepoint, size2, size);
+		return 0;
+	}
+
+	return 1;
+}
+
 BOOL torture_local_iconv(int dummy) 
 {
 	size_t size;
@@ -263,13 +302,18 @@ BOOL torture_local_iconv(int dummy)
 
 	srandom(time(NULL));
 
+	printf("Testing next_codepoint()\n");
+	for (codepoint=0;ok && codepoint<(1<<20);codepoint++) {
+		ok = test_codepoint(codepoint);
+	}
+
 	printf("Testing first 1M codepoints\n");
 	for (codepoint=0;ok && codepoint<(1<<20);codepoint++) {
-		if (gen_codepoint(codepoint, inbuf, &size) != 0) {
+		if (gen_codepoint_utf16(codepoint, inbuf, &size) != 0) {
 			continue;
 		}
 
-		if (codepoint % 100 == 0) {
+		if (codepoint % 1000 == 0) {
 			printf("codepoint=%u   \r", codepoint);
 		}
 
@@ -279,7 +323,7 @@ BOOL torture_local_iconv(int dummy)
 
 	printf("Testing 5M random UTF-16LE sequences\n");
 	for (i=0;ok && i<500000;i++) {
-		if (i % 100 == 0) {
+		if (i % 1000 == 0) {
 			printf("i=%u              \r", i);
 		}
 
diff --git a/source4/torture/masktest.c b/source4/torture/masktest.c
index dbd7c2f74b..82b15722c5 100644
--- a/source4/torture/masktest.c
+++ b/source4/torture/masktest.c
@@ -161,9 +161,10 @@ static void testpair(struct smbcli_state *cli, char *mask, char *file)
 	fstrcpy(short_name, "");
 	get_real_name(cli, long_name, short_name);
 	fstrcpy(res1, "---");
-	smbcli_list(cli->tree, mask, 
-		 FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY, 
-		 listfn, NULL);
+	smbcli_list_new(cli->tree, mask, 
+			FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_DIRECTORY, 
+			RAW_SEARCH_BOTH_DIRECTORY_INFO,
+			listfn, NULL);
 
 	res2 = reg_test(cli, mask, long_name, short_name);
 
@@ -301,6 +302,8 @@ static void usage(void)
 
 	seed = time(NULL);
 
+	init_iconv();
+
 	while ((opt = getopt(argc, argv, "n:d:U:s:hm:f:aoW:M:vEl:")) != EOF) {
 		switch (opt) {
 		case 'n':
diff --git a/source4/torture/rpc/netlogon.c b/source4/torture/rpc/netlogon.c
index 16ba679be9..b0509659e7 100644
--- a/source4/torture/rpc/netlogon.c
+++ b/source4/torture/rpc/netlogon.c
@@ -778,7 +778,7 @@ static BOOL test_plaintext(struct samlogon_state *samlogon_state, enum ntlm_brea
 	DATA_BLOB lm_response = data_blob(NULL, 0);
 	char *password;
 	char *dospw;
-	smb_ucs2_t *unicodepw;
+	void *unicodepw;
 
 	uint8_t user_session_key[16];
 	uint8_t lm_key[16];
@@ -787,14 +787,13 @@ static BOOL test_plaintext(struct samlogon_state *samlogon_state, enum ntlm_brea
 
 	ZERO_STRUCT(user_session_key);
 	
-	if ((push_ucs2_talloc(samlogon_state->mem_ctx, (smb_ucs2_t **)&unicodepw, 
+	if ((push_ucs2_talloc(samlogon_state->mem_ctx, &unicodepw, 
 			      samlogon_state->password)) == -1) {
 		DEBUG(0, ("push_ucs2_allocate failed!\n"));
 		exit(1);
 	}
 
-	nt_response = data_blob_talloc(samlogon_state->mem_ctx, unicodepw, 
-				       strlen_w(((void *)unicodepw))*sizeof(smb_ucs2_t));
+	nt_response = data_blob_talloc(samlogon_state->mem_ctx, unicodepw, utf16_len(unicodepw));
 
 	password = strupper_talloc(samlogon_state->mem_ctx, samlogon_state->password);