a fairly major upgrade to the dcerpc system

 * added a NDR validator. The way it works is that when the
   DCERPC_DEBUG_VALIDATE_* flags are set the dcerpc system will
   perform NDR buffer validation. On sending a request the packet is
   first marshalled, then unmarahslled, then marshalled again, and it is
   confirmed that the two marshalling results are idential. This
   ensures that our pull and push routines are absolutely in sync, so
   that we can be very confident that if a routine works in the client
   then the corresponding routine must work on the server side. A
   similar validation is performed on all replies.

 * a result of this change is that pidl is fussier about the [ref]
   tag. You can only use it on pointers (which is the only place it
   makes sense)

 * fixed a basic alignment bug in the push side of the NDR code

 * added server side pull/push support. Our dcerpc system is now fully
   ready to be used on the server side.

 * fixed the relative offset pointer list. It must be traversed in
   reverse order on push

 * added automatic value setting for the size parameter in outgoing
   SdBuf structures.

 * expanded the ndr debugging code to always give a message on any
   failure

 * fixed the subcontext push code

 * fixed some memory leaks in smbtorture RPC tests
(This used to be commit 8ecf720206a2eef3f8ea7cbdb1f460664a5dba9a)

4 files changed, 244 insertions, 28 deletions

diff --git a/source4/librpc/ndr/libndr.h b/source4/librpc/ndr/libndr.h
index 5c771ff2d2..3c77646239 100644
--- a/source4/librpc/ndr/libndr.h
+++ b/source4/librpc/ndr/libndr.h
@@ -73,7 +73,7 @@ struct ndr_push {
 	struct ndr_ofs_list *ofs_list;
 
 	/* this list is used by the [relative] code to find the offsets */
-	struct ndr_ofs_list *relative_list;
+	struct ndr_ofs_list *relative_list, *relative_list_end;
 };
 
 struct ndr_push_save {
@@ -101,6 +101,8 @@ struct ndr_print {
 #define LIBNDR_FLAG_STR_NULLTERM (1<<6)
 #define LIBNDR_STRING_FLAGS      (0x7C)
 
+#define LIBNDR_FLAG_REF_ALLOC    (1<<10)
+
 /* useful macro for debugging */
 #define NDR_PRINT_DEBUG(type, p) ndr_print_debug((ndr_print_fn_t)ndr_print_ ##type, #p, p)
 #define NDR_PRINT_UNION_DEBUG(type, level, p) ndr_print_union_debug((ndr_print_union_fn_t)ndr_print_ ##type, #p, level, p)
@@ -119,7 +121,10 @@ enum ndr_err_code {
 	NDR_ERR_CHARCNV,
 	NDR_ERR_LENGTH,
 	NDR_ERR_SUBCONTEXT,
-	NDR_ERR_STRING
+	NDR_ERR_STRING,
+	NDR_ERR_VALIDATE,
+	NDR_ERR_BUFSIZE,
+	NDR_ERR_ALLOC
 };
 
 /*
@@ -137,7 +142,7 @@ enum ndr_err_code {
 
 #define NDR_PULL_NEED_BYTES(ndr, n) do { \
 	if ((n) > ndr->data_size || ndr->offset + (n) > ndr->data_size) { \
-		return NT_STATUS_BUFFER_TOO_SMALL; \
+		return ndr_pull_error(ndr, NDR_ERR_BUFSIZE, "Pull bytes %u", n); \
 	} \
 } while(0)
 
@@ -146,7 +151,7 @@ enum ndr_err_code {
 		ndr->offset = (ndr->offset + (n-1)) & ~(n-1); \
 	} \
 	if (ndr->offset >= ndr->data_size) { \
-		return NT_STATUS_BUFFER_TOO_SMALL; \
+		return ndr_pull_error(ndr, NDR_ERR_BUFSIZE, "Pull align %u", n); \
 	} \
 } while(0)
 
@@ -154,7 +159,7 @@ enum ndr_err_code {
 
 #define NDR_PUSH_ALIGN(ndr, n) do { \
 	if (!(ndr->flags & LIBNDR_FLAG_NOALIGN)) { \
-		uint32 _pad = (ndr->offset & (n-1)); \
+		uint32 _pad = ((ndr->offset + (n-1)) & ~(n-1)) - ndr->offset; \
 		while (_pad--) NDR_CHECK(ndr_push_uint8(ndr, 0)); \
 	} \
 } while(0)
@@ -171,7 +176,9 @@ enum ndr_err_code {
 
 #define NDR_ALLOC_SIZE(ndr, s, size) do { \
 	                       (s) = talloc(ndr->mem_ctx, size); \
-                               if (!(s)) return NT_STATUS_NO_MEMORY; \
+                               if (!(s)) return ndr_pull_error(ndr, NDR_ERR_ALLOC, \
+							       "Alloc %u failed\n", \
+							       size); \
                            } while (0)
 
 #define NDR_ALLOC(ndr, s) NDR_ALLOC_SIZE(ndr, s, sizeof(*(s)))
@@ -182,12 +189,26 @@ enum ndr_err_code {
 					(s) = NULL; \
 				} else { \
 					(s) = talloc(ndr->mem_ctx, (n) * elsize); \
-					if (!(s)) return NT_STATUS_NO_MEMORY; \
+                               		if (!(s)) return ndr_pull_error(ndr, \
+									NDR_ERR_ALLOC, \
+									"Alloc %u * %u failed\n", \
+									n, elsize); \
 				} \
                            } while (0)
 
 #define NDR_ALLOC_N(ndr, s, n) NDR_ALLOC_N_SIZE(ndr, s, n, sizeof(*(s)))
 
+
+#define NDR_PUSH_ALLOC_SIZE(ndr, s, size) do { \
+	                       (s) = talloc(ndr->mem_ctx, size); \
+                               if (!(s)) return ndr_push_error(ndr, NDR_ERR_ALLOC, \
+							       "push alloc %u failed\n",\
+							       size); \
+                           } while (0)
+
+#define NDR_PUSH_ALLOC(ndr, s) NDR_PUSH_ALLOC_SIZE(ndr, s, sizeof(*(s)))
+
+
 /* these are used when generic fn pointers are needed for ndr push/pull fns */
 typedef NTSTATUS (*ndr_push_fn_t)(struct ndr_push *, void *);
 typedef NTSTATUS (*ndr_pull_fn_t)(struct ndr_pull *, void *);
diff --git a/source4/librpc/ndr/ndr.c b/source4/librpc/ndr/ndr.c
index b67fb25b1e..7deee6be3b 100644
--- a/source4/librpc/ndr/ndr.c
+++ b/source4/librpc/ndr/ndr.c
@@ -81,7 +81,9 @@ NTSTATUS ndr_pull_limit_size(struct ndr_pull *ndr, uint32 size, uint32 ofs)
 	new_size = ndr->offset + size - ofs;
 
 	if (new_size > ndr->data_size) {
-		return NT_STATUS_BUFFER_TOO_SMALL;
+		return ndr_pull_error(ndr, NDR_ERR_BUFSIZE, 
+				      "ndr_pull_limit_size %s %u failed",
+				      size, ofs);
 	}
 	ndr->data_size = new_size;
 
@@ -96,7 +98,9 @@ NTSTATUS ndr_pull_advance(struct ndr_pull *ndr, uint32 size)
 {
 	ndr->offset += size;
 	if (ndr->offset > ndr->data_size) {
-		return NT_STATUS_BUFFER_TOO_SMALL;
+		return ndr_pull_error(ndr, NDR_ERR_BUFSIZE, 
+				      "ndr_pull_advance by %u failed",
+				      size);
 	}
 	return NT_STATUS_OK;
 }
@@ -108,7 +112,9 @@ NTSTATUS ndr_pull_set_offset(struct ndr_pull *ndr, uint32 ofs)
 {
 	ndr->offset = ofs;
 	if (ndr->offset > ndr->data_size) {
-		return NT_STATUS_BUFFER_TOO_SMALL;
+		return ndr_pull_error(ndr, NDR_ERR_BUFSIZE, 
+				      "ndr_pull_set_offset %u failed",
+				      ofs);
 	}
 	return NT_STATUS_OK;
 }
@@ -128,18 +134,13 @@ void ndr_pull_restore(struct ndr_pull *ndr, struct ndr_pull_save *save)
 }
 
 
-
-
 /* create a ndr_push structure, ready for some marshalling */
-struct ndr_push *ndr_push_init(void)
+struct ndr_push *ndr_push_init_ctx(TALLOC_CTX *mem_ctx)
 {
 	struct ndr_push *ndr;
-	TALLOC_CTX *mem_ctx = talloc_init("ndr_push_init");
-	if (!mem_ctx) return NULL;
 
 	ndr = talloc(mem_ctx, sizeof(*ndr));
 	if (!ndr) {
-		talloc_destroy(mem_ctx);
 		return NULL;
 	}
 
@@ -148,15 +149,30 @@ struct ndr_push *ndr_push_init(void)
 	ndr->alloc_size = NDR_BASE_MARSHALL_SIZE;
 	ndr->data = talloc(ndr->mem_ctx, ndr->alloc_size);
 	if (!ndr->data) {
-		ndr_push_free(ndr);
 		return NULL;
 	}
 	ndr->offset = 0;
 	ndr->ptr_count = 0;
+	ndr->relative_list = NULL;
+	ndr->relative_list_end = NULL;
 	
 	return ndr;
 }
 
+
+/* create a ndr_push structure, ready for some marshalling */
+struct ndr_push *ndr_push_init(void)
+{
+	struct ndr_push *ndr;
+	TALLOC_CTX *mem_ctx = talloc_init("ndr_push_init");
+	if (!mem_ctx) return NULL;
+	ndr = ndr_push_init_ctx(mem_ctx);
+	if (!ndr) {
+		talloc_destroy(mem_ctx);
+	}
+	return ndr;
+}
+
 /* free a ndr_push structure */
 void ndr_push_free(struct ndr_push *ndr)
 {
@@ -189,7 +205,8 @@ NTSTATUS ndr_push_expand(struct ndr_push *ndr, uint32 size)
 	}
 	ndr->data = talloc_realloc(ndr->mem_ctx, ndr->data, ndr->alloc_size);
 	if (!ndr->data) {
-		return NT_STATUS_NO_MEMORY;
+		return ndr_push_error(ndr, NDR_ERR_ALLOC, "Failed to push_expand to %u",
+				      ndr->alloc_size);
 	}
 
 	return NT_STATUS_OK;
@@ -358,6 +375,20 @@ void ndr_print_function_debug(void (*fn)(struct ndr_print *, const char *, int ,
 	talloc_destroy(ndr.mem_ctx);
 }
 
+
+static NTSTATUS ndr_map_error(enum ndr_err_code err)
+{
+	switch (err) {
+	case NDR_ERR_BUFSIZE:
+		return NT_STATUS_BUFFER_TOO_SMALL;
+	case NDR_ERR_ALLOC:
+		return NT_STATUS_NO_MEMORY;
+	}
+
+	/* we should all error codes to different status codes */
+	return NT_STATUS_INVALID_PARAMETER;
+}
+
 /*
   return and possibly log an NDR error
 */
@@ -373,8 +404,8 @@ NTSTATUS ndr_pull_error(struct ndr_pull *ndr, enum ndr_err_code err, const char
 	DEBUG(3,("ndr_pull_error(%u): %s\n", err, s));
 
 	free(s);
-	/* we should map to different status codes */
-	return NT_STATUS_INVALID_PARAMETER;
+
+	return ndr_map_error(err);
 }
 
 /*
@@ -392,8 +423,8 @@ NTSTATUS ndr_push_error(struct ndr_push *ndr, enum ndr_err_code err, const char
 	DEBUG(3,("ndr_push_error(%u): %s\n", err, s));
 
 	free(s);
-	/* we should map to different status codes */
-	return NT_STATUS_INVALID_PARAMETER;
+
+	return ndr_map_error(err);
 }
 
 
@@ -494,6 +525,92 @@ NTSTATUS ndr_pull_subcontext_union_fn(struct ndr_pull *ndr,
 
 
 /*
+  push a subcontext header 
+*/
+static NTSTATUS ndr_push_subcontext_header(struct ndr_push *ndr, 
+					   size_t sub_size,
+					   struct ndr_push *ndr2)
+{
+	switch (sub_size) {
+	case 0: 
+		break;
+
+	case 2: 
+		NDR_CHECK(ndr_push_uint16(ndr, ndr2->offset));
+		break;
+
+	case 4: 
+		NDR_CHECK(ndr_push_uint32(ndr, ndr2->offset));
+		break;
+
+	default:
+		return ndr_push_error(ndr, NDR_ERR_SUBCONTEXT, "Bad subcontext size %d", 
+				      sub_size);
+	}
+	return NT_STATUS_OK;
+}
+
+/*
+  handle subcontext buffers, which in midl land are user-marshalled, but
+  we use magic in pidl to make them easier to cope with
+*/
+NTSTATUS ndr_push_subcontext_fn(struct ndr_push *ndr, 
+				size_t sub_size,
+				void *base,
+				NTSTATUS (*fn)(struct ndr_push *, void *))
+{
+	struct ndr_push *ndr2;
+
+	ndr2 = ndr_push_init_ctx(ndr->mem_ctx);
+	if (!ndr2) return NT_STATUS_NO_MEMORY;
+
+	NDR_CHECK(fn(ndr2, base));
+	NDR_CHECK(ndr_push_subcontext_header(ndr, sub_size, ndr2));
+	NDR_CHECK(ndr_push_bytes(ndr, ndr2->data, ndr2->offset));
+	return NT_STATUS_OK;
+}
+
+/*
+  handle subcontext buffers for function that take a flags arg
+*/
+NTSTATUS ndr_push_subcontext_flags_fn(struct ndr_push *ndr, 
+				      size_t sub_size,
+				      void *base,
+				      NTSTATUS (*fn)(struct ndr_push *, int, void *))
+{
+	struct ndr_push *ndr2;
+
+	ndr2 = ndr_push_init_ctx(ndr->mem_ctx);
+	if (!ndr2) return NT_STATUS_NO_MEMORY;
+
+	NDR_CHECK(fn(ndr2, NDR_SCALARS|NDR_BUFFERS, base));
+	NDR_CHECK(ndr_push_subcontext_header(ndr, sub_size, ndr2));
+	NDR_CHECK(ndr_push_bytes(ndr, ndr2->data, ndr2->offset));
+	return NT_STATUS_OK;
+}
+
+/*
+  handle subcontext buffers for function that take a union
+*/
+NTSTATUS ndr_push_subcontext_union_fn(struct ndr_push *ndr, 
+				      size_t sub_size,
+				      uint32 level,
+				      void *base,
+				      NTSTATUS (*fn)(struct ndr_push *, int, uint32, void *))
+{
+	struct ndr_push *ndr2;
+
+	ndr2 = ndr_push_init_ctx(ndr->mem_ctx);
+	if (!ndr2) return NT_STATUS_NO_MEMORY;
+
+	NDR_CHECK(fn(ndr2, NDR_SCALARS|NDR_BUFFERS, level, base));
+	NDR_CHECK(ndr_push_subcontext_header(ndr, sub_size, ndr2));
+	NDR_CHECK(ndr_push_bytes(ndr, ndr2->data, ndr2->offset));
+	return NT_STATUS_OK;
+}
+
+
+/*
   mark the start of a structure
 */
 NTSTATUS ndr_pull_struct_start(struct ndr_pull *ndr)
@@ -520,7 +637,7 @@ void ndr_pull_struct_end(struct ndr_pull *ndr)
 NTSTATUS ndr_push_struct_start(struct ndr_push *ndr)
 {
 	struct ndr_ofs_list *ofs;
-	NDR_ALLOC(ndr, ofs);
+	NDR_PUSH_ALLOC(ndr, ofs);
 	ofs->offset = ndr->offset;
 	ofs->next = ndr->ofs_list;
 	ndr->ofs_list = ofs;
@@ -579,12 +696,17 @@ NTSTATUS ndr_push_relative(struct ndr_push *ndr, int ndr_flags, const void *p,
 			NDR_CHECK(ndr_push_uint32(ndr, 0));
 			return NT_STATUS_OK;
 		}
-		NDR_ALLOC(ndr, ofs);
+		NDR_PUSH_ALLOC(ndr, ofs);
 		NDR_CHECK(ndr_push_align(ndr, 4));
 		ofs->offset = ndr->offset;
 		NDR_CHECK(ndr_push_uint32(ndr, 0xFFFFFFFF));
-		ofs->next = ndr->relative_list;
-		ndr->relative_list = ofs;
+		ofs->next = NULL;
+		if (ndr->relative_list_end) {
+			ndr->relative_list_end->next = ofs;
+		} else {
+			ndr->relative_list = ofs;
+		}
+		ndr->relative_list_end = ofs;
 	}
 	if (ndr_flags & NDR_BUFFERS) {
 		struct ndr_push_save save;
@@ -596,6 +718,9 @@ NTSTATUS ndr_push_relative(struct ndr_push *ndr, int ndr_flags, const void *p,
 			return ndr_push_error(ndr, NDR_ERR_RELATIVE, "Empty relative stack");
 		}
 		ndr->relative_list = ndr->relative_list->next;
+		if (ndr->relative_list == NULL) {
+			ndr->relative_list_end = NULL;
+		}
 		NDR_CHECK(ndr_push_align(ndr, 8));
 		ndr_push_save(ndr, &save);
 		ndr->offset = ofs->offset;
diff --git a/source4/librpc/ndr/ndr_basic.c b/source4/librpc/ndr/ndr_basic.c
index 56c0ab231e..a3c4bc0aec 100644
--- a/source4/librpc/ndr/ndr_basic.c
+++ b/source4/librpc/ndr/ndr_basic.c
@@ -83,6 +83,14 @@ NTSTATUS ndr_pull_NTSTATUS(struct ndr_pull *ndr, NTSTATUS *status)
 	return NT_STATUS_OK;
 }
 
+/*
+  push a NTSTATUS
+*/
+NTSTATUS ndr_push_NTSTATUS(struct ndr_push *ndr, NTSTATUS status)
+{
+	return ndr_push_uint32(ndr, NT_STATUS_V(status));
+}
+
 void ndr_print_NTSTATUS(struct ndr_print *ndr, const char *name, NTSTATUS *r)
 {
 	ndr->print(ndr, "%-25s: %s", name, nt_errstr(*r));
@@ -99,6 +107,14 @@ NTSTATUS ndr_pull_WERROR(struct ndr_pull *ndr, WERROR *status)
 	return NT_STATUS_OK;
 }
 
+/*
+  push a WERROR
+*/
+NTSTATUS ndr_push_WERROR(struct ndr_push *ndr, WERROR status)
+{
+	return ndr_push_uint32(ndr, W_ERROR_V(status));
+}
+
 void ndr_print_WERROR(struct ndr_print *ndr, const char *name, WERROR *r)
 {
 	ndr->print(ndr, "%-25s: %s", name, win_errstr(*r));
@@ -340,7 +356,9 @@ NTSTATUS ndr_pull_string(struct ndr_pull *ndr, int ndr_flags, const char **s)
 		NDR_CHECK(ndr_pull_uint32(ndr, &ofs));
 		NDR_CHECK(ndr_pull_uint32(ndr, &len2));
 		if (len2 > len1) {
-			return NT_STATUS_INVALID_PARAMETER;
+			return ndr_pull_error(ndr, NDR_ERR_STRING, 
+					      "Bad string lengths len1=%u ofs=%u len2=%u\n", 
+					      len1, ofs, len2);
 		}
 		if (len2 == 0) {
 			*s = talloc_strdup(ndr->mem_ctx, "");
@@ -395,7 +413,9 @@ NTSTATUS ndr_pull_string(struct ndr_pull *ndr, int ndr_flags, const char **s)
 		NDR_CHECK(ndr_pull_uint32(ndr, &ofs));
 		NDR_CHECK(ndr_pull_uint32(ndr, &len2));
 		if (len2 > len1) {
-			return NT_STATUS_INVALID_PARAMETER;
+			return ndr_pull_error(ndr, NDR_ERR_STRING, 
+					      "Bad ascii string lengths len1=%u ofs=%u len2=%u\n", 
+					      len1, ofs, len2);
 		}
 		NDR_ALLOC_N(ndr, as, (len2+1));
 		NDR_CHECK(ndr_pull_bytes(ndr, as, len2));
diff --git a/source4/librpc/ndr/ndr_sec.c b/source4/librpc/ndr/ndr_sec.c
index 5a959b9b47..1a8d355149 100644
--- a/source4/librpc/ndr/ndr_sec.c
+++ b/source4/librpc/ndr/ndr_sec.c
@@ -88,3 +88,53 @@ void ndr_print_dom_sid2(struct ndr_print *ndr, const char *name, struct dom_sid2
 	ndr_print_dom_sid(ndr, name, sid);
 }
 
+
+/*
+  return the wire size of a security_ace
+*/
+size_t ndr_size_security_ace(struct security_ace *ace)
+{
+	if (!ace) return 0;
+	return 8 + ndr_size_dom_sid(&ace->trustee);
+}
+
+
+/*
+  return the wire size of a security_acl
+*/
+size_t ndr_size_security_acl(struct security_acl *acl)
+{
+	size_t ret;
+	int i;
+	if (!acl) return 0;
+	ret = 8;
+	for (i=0;i<acl->num_aces;i++) {
+		ret += ndr_size_security_ace(&acl->aces[i]);
+	}
+	return ret;
+}
+
+/*
+  return the wire size of a dom_sid
+*/
+size_t ndr_size_dom_sid(struct dom_sid *sid)
+{
+	if (!sid) return 0;
+	return 8 + 4*sid->num_auths;
+}
+
+/*
+  return the wire size of a security descriptor
+*/
+size_t ndr_size_security_descriptor(struct security_descriptor *sd)
+{
+	size_t ret;
+	if (!sd) return 0;
+	
+	ret = 20;
+	ret += ndr_size_dom_sid(sd->owner_sid);
+	ret += ndr_size_dom_sid(sd->group_sid);
+	ret += ndr_size_security_acl(sd->dacl);
+	ret += ndr_size_security_acl(sd->sacl);
+	return ret;
+}