r11473: Based on work by Jelmer, implement the [async] flag for rpc requests. If it's

not there (it's not yet on *any* call... :-)), the rpc client strictly
sequences calls to an rpc pipe. Might need some more work on the exact
sequencing semantics when a pipe with both sync and async calls is actually
deployed, but I want it in for winbind simplification.

Volker
(This used to be commit b8f324e4f000971b7dafc263c16dd4af958ee7f9)

8 files changed, 190 insertions, 40 deletions

diff --git a/source4/librpc/rpc/dcerpc.c b/source4/librpc/rpc/dcerpc.c
index 2c0dbcd5f3..ea1ae76b4e 100644
--- a/source4/librpc/rpc/dcerpc.c
+++ b/source4/librpc/rpc/dcerpc.c
@@ -28,6 +28,8 @@
 #include "librpc/gen_ndr/ndr_dcerpc.h"
 #include "librpc/gen_ndr/ndr_misc.h"
 
+static void dcerpc_ship_next_request(struct dcerpc_connection *c);
+
 static struct dcerpc_interface_list *dcerpc_pipes = NULL;
 
 /*
@@ -857,6 +859,10 @@ req_done:
 	if (req->async.callback) {
 		req->async.callback(req);
 	}
+
+	if (c->request_queue != NULL) {
+		dcerpc_ship_next_request(c);
+	}
 }
 
 /*
@@ -893,16 +899,13 @@ static int dcerpc_req_destructor(void *ptr)
 /*
   perform the send side of a async dcerpc request
 */
-struct rpc_request *dcerpc_request_send(struct dcerpc_pipe *p, 
-					const struct GUID *object,
-					uint16_t opnum,
-					DATA_BLOB *stub_data)
+static struct rpc_request *dcerpc_request_send(struct dcerpc_pipe *p, 
+					       const struct GUID *object,
+					       uint16_t opnum,
+					       BOOL async,
+					       DATA_BLOB *stub_data)
 {
 	struct rpc_request *req;
-	struct ncacn_packet pkt;
-	DATA_BLOB blob;
-	uint32_t remaining, chunk_size;
-	BOOL first_packet = True;
 
 	p->conn->transport.recv_data = dcerpc_request_recv_data;
 
@@ -918,8 +921,69 @@ struct rpc_request *dcerpc_request_send(struct dcerpc_pipe *p,
 	req->payload = data_blob(NULL, 0);
 	req->flags = 0;
 	req->fault_code = 0;
+	req->async_call = async;
 	req->async.callback = NULL;
 
+	if (object != NULL) {
+		req->object = talloc_memdup(req, object, sizeof(*object));
+		if (req->object == NULL) {
+			talloc_free(req);
+			return NULL;
+		}
+	} else {
+		req->object = NULL;
+	}
+
+	req->opnum = opnum;
+	req->request_data.length = stub_data->length;
+	req->request_data.data = talloc_reference(req, stub_data->data);
+	if (req->request_data.data == NULL) {
+		return NULL;
+	}
+
+	DLIST_ADD_END(p->conn->request_queue, req, struct rpc_request *);
+
+	dcerpc_ship_next_request(p->conn);
+
+	if (p->request_timeout) {
+		event_add_timed(dcerpc_event_context(p), req, 
+				timeval_current_ofs(p->request_timeout, 0), 
+				dcerpc_timeout_handler, req);
+	}
+
+	talloc_set_destructor(req, dcerpc_req_destructor);
+	return req;
+}
+
+/*
+  Send a request using the transport
+*/
+
+static void dcerpc_ship_next_request(struct dcerpc_connection *c)
+{
+	struct rpc_request *req;
+	struct dcerpc_pipe *p;
+	DATA_BLOB *stub_data;
+	struct ncacn_packet pkt;
+	DATA_BLOB blob;
+	uint32_t remaining, chunk_size;
+	BOOL first_packet = True;
+
+	req = c->request_queue;
+	if (req == NULL) {
+		return;
+	}
+
+	p = req->p;
+	stub_data = &req->request_data;
+
+	if (!req->async_call && (c->pending != NULL)) {
+		return;
+	}
+
+	DLIST_REMOVE(c->request_queue, req);
+	DLIST_ADD(c->pending, req);
+
 	init_ncacn_hdr(p->conn, &pkt);
 
 	remaining = stub_data->length;
@@ -934,16 +998,14 @@ struct rpc_request *dcerpc_request_send(struct dcerpc_pipe *p,
 	pkt.pfc_flags = 0;
 	pkt.u.request.alloc_hint = remaining;
 	pkt.u.request.context_id = p->context_id;
-	pkt.u.request.opnum = opnum;
+	pkt.u.request.opnum = req->opnum;
 
-	if (object) {
-		pkt.u.request.object.object = *object;
+	if (req->object) {
+		pkt.u.request.object.object = *req->object;
 		pkt.pfc_flags |= DCERPC_PFC_FLAG_ORPC;
-		chunk_size -= ndr_size_GUID(object,0);
+		chunk_size -= ndr_size_GUID(req->object,0);
 	}
 
-	DLIST_ADD(p->conn->pending, req);
-
 	/* we send a series of pdus without waiting for a reply */
 	while (remaining > 0 || first_packet) {
 		uint32_t chunk = MIN(chunk_size, remaining);
@@ -968,28 +1030,18 @@ struct rpc_request *dcerpc_request_send(struct dcerpc_pipe *p,
 		if (!NT_STATUS_IS_OK(req->status)) {
 			req->state = RPC_REQUEST_DONE;
 			DLIST_REMOVE(p->conn->pending, req);
-			return req;
+			return;
 		}
 		
 		req->status = p->conn->transport.send_request(p->conn, &blob, last_frag);
 		if (!NT_STATUS_IS_OK(req->status)) {
 			req->state = RPC_REQUEST_DONE;
 			DLIST_REMOVE(p->conn->pending, req);
-			return req;
+			return;
 		}		
 
 		remaining -= chunk;
 	}
-
-	if (p->request_timeout) {
-		event_add_timed(dcerpc_event_context(p), req, 
-				timeval_current_ofs(p->request_timeout, 0), 
-				dcerpc_timeout_handler, req);
-	}
-
-	talloc_set_destructor(req, dcerpc_req_destructor);
-
-	return req;
 }
 
 /*
@@ -1036,13 +1088,14 @@ NTSTATUS dcerpc_request_recv(struct rpc_request *req,
 NTSTATUS dcerpc_request(struct dcerpc_pipe *p, 
 			struct GUID *object,
 			uint16_t opnum,
+			BOOL async,
 			TALLOC_CTX *mem_ctx,
 			DATA_BLOB *stub_data_in,
 			DATA_BLOB *stub_data_out)
 {
 	struct rpc_request *req;
 
-	req = dcerpc_request_send(p, object, opnum, stub_data_in);
+	req = dcerpc_request_send(p, object, opnum, async, stub_data_in);
 	if (req == NULL) {
 		return NT_STATUS_NO_MEMORY;
 	}
@@ -1251,7 +1304,8 @@ struct rpc_request *dcerpc_ndr_request_send(struct dcerpc_pipe *p,
 	dump_data(10, request.data, request.length);
 
 	/* make the actual dcerpc request */
-	req = dcerpc_request_send(p, object, opnum, &request);
+	req = dcerpc_request_send(p, object, opnum, table->calls[opnum].async,
+				  &request);
 
 	if (req != NULL) {
 		req->ndr.table = table;
diff --git a/source4/librpc/rpc/dcerpc.h b/source4/librpc/rpc/dcerpc.h
index c5b0ad159b..85fd36f066 100644
--- a/source4/librpc/rpc/dcerpc.h
+++ b/source4/librpc/rpc/dcerpc.h
@@ -69,9 +69,12 @@ struct dcerpc_connection {
 		void (*recv_data)(struct dcerpc_connection *, DATA_BLOB *, NTSTATUS status);
 	} transport;
 
-	/* pending requests */
+	/* Requests that have been sent, waiting for a reply */
 	struct rpc_request *pending;
 
+	/* Sync requests waiting to be shipped */
+	struct rpc_request *request_queue;
+
 	/* private pointer for pending full requests */
 	void *full_request_private;
 
@@ -145,6 +148,7 @@ struct dcerpc_interface_call {
 	ndr_push_flags_fn_t ndr_push;
 	ndr_pull_flags_fn_t ndr_pull;
 	ndr_print_function_t ndr_print;
+	BOOL async;
 };
 
 struct dcerpc_endpoint_list {
@@ -204,6 +208,11 @@ struct rpc_request {
 	uint_t flags;
 	uint32_t fault_code;
 
+	const struct GUID *object;
+	uint16_t opnum;
+	DATA_BLOB request_data;
+	BOOL async_call;
+
 	/* use by the ndr level async recv call */
 	struct {
 		const struct dcerpc_interface_table *table;
diff --git a/source4/pidl/lib/Parse/Pidl/NDR.pm b/source4/pidl/lib/Parse/Pidl/NDR.pm
index d0b6708bf6..8405873589 100644
--- a/source4/pidl/lib/Parse/Pidl/NDR.pm
+++ b/source4/pidl/lib/Parse/Pidl/NDR.pm
@@ -529,10 +529,14 @@ sub ParseFunction($$$)
 		$rettype = $d->{RETURN_TYPE};
 	}
 	
+	my $async = 0;
+	if (has_property($d, "async")) { $async = 1; }
+	
 	return {
 			NAME => $d->{NAME},
 			TYPE => "FUNCTION",
 			OPNUM => $thisopnum,
+			ASYNC => $async,
 			RETURN_TYPE => $rettype,
 			PROPERTIES => $d->{PROPERTIES},
 			ELEMENTS => \@elements,
@@ -744,6 +748,7 @@ my %property_list = (
 	"noopnum"		=> ["FUNCTION"],
 	"in"			=> ["ELEMENT"],
 	"out"			=> ["ELEMENT"],
+	"async"			=> ["FUNCTION"],
 
 	# pointer
 	"ref"			=> ["ELEMENT"],
diff --git a/source4/pidl/lib/Parse/Pidl/Samba/NDR/Parser.pm b/source4/pidl/lib/Parse/Pidl/Samba/NDR/Parser.pm
index 0454f90c9a..05946b44f5 100644
--- a/source4/pidl/lib/Parse/Pidl/Samba/NDR/Parser.pm
+++ b/source4/pidl/lib/Parse/Pidl/Samba/NDR/Parser.pm
@@ -2160,11 +2160,12 @@ sub FunctionTable($)
 		pidl "\t\tsizeof(struct $d->{NAME}),";
 		pidl "\t\t(ndr_push_flags_fn_t) ndr_push_$d->{NAME},";
 		pidl "\t\t(ndr_pull_flags_fn_t) ndr_pull_$d->{NAME},";
-		pidl "\t\t(ndr_print_function_t) ndr_print_$d->{NAME}";
+		pidl "\t\t(ndr_print_function_t) ndr_print_$d->{NAME},";
+		pidl "\t\t".($d->{ASYNC}?"True":"False").",";
 		pidl "\t},";
 		$count++;
 	}
-	pidl "\t{ NULL, 0, NULL, NULL, NULL }";
+	pidl "\t{ NULL, 0, NULL, NULL, NULL, False }";
 	pidl "};";
 	pidl "";
 
diff --git a/source4/torture/rpc/autoidl.c b/source4/torture/rpc/autoidl.c
index 22768c964b..4cfee1a379 100644
--- a/source4/torture/rpc/autoidl.c
+++ b/source4/torture/rpc/autoidl.c
@@ -130,7 +130,7 @@ static void try_expand(TALLOC_CTX *mem_ctx, const struct dcerpc_interface_table
 		memcpy(stub_in.data, base_in->data, insert_ofs);
 		memcpy(stub_in.data+insert_ofs+n, base_in->data+insert_ofs, base_in->length-insert_ofs);
 
-		status = dcerpc_request(p, NULL, opnum, mem_ctx, &stub_in, &stub_out);
+		status = dcerpc_request(p, NULL, opnum, False, mem_ctx, &stub_in, &stub_out);
 
 		if (!NT_STATUS_EQUAL(status, NT_STATUS_NET_WRITE_FAULT)) {
 			print_depth(depth);
@@ -171,7 +171,7 @@ static void test_ptr_scan(TALLOC_CTX *mem_ctx, const struct dcerpc_interface_tab
 	/* work out which elements are pointers */
 	for (ofs=min_ofs;ofs<=max_ofs-4;ofs+=4) {
 		SIVAL(stub_in.data, ofs, 1);
-		status = dcerpc_request(p, NULL, opnum, mem_ctx, &stub_in, &stub_out);
+		status = dcerpc_request(p, NULL, opnum, False, mem_ctx, &stub_in, &stub_out);
 
 		if (NT_STATUS_EQUAL(status, NT_STATUS_NET_WRITE_FAULT)) {
 			print_depth(depth);
@@ -213,7 +213,7 @@ static void test_scan_call(TALLOC_CTX *mem_ctx, const struct dcerpc_interface_ta
 		data_blob_clear(&stub_in);
 
 
-		status = dcerpc_request(p, NULL, opnum, mem_ctx, &stub_in, &stub_out);
+		status = dcerpc_request(p, NULL, opnum, False, mem_ctx, &stub_in, &stub_out);
 
 		if (NT_STATUS_IS_OK(status)) {
 			printf("opnum %d   min_input %d - output %d\n", 
@@ -226,7 +226,7 @@ static void test_scan_call(TALLOC_CTX *mem_ctx, const struct dcerpc_interface_ta
 
 		fill_blob_handle(&stub_in, mem_ctx, &handle);
 
-		status = dcerpc_request(p, NULL, opnum, mem_ctx, &stub_in, &stub_out);
+		status = dcerpc_request(p, NULL, opnum, False, mem_ctx, &stub_in, &stub_out);
 
 		if (NT_STATUS_IS_OK(status)) {
 			printf("opnum %d   min_input %d - output %d (with handle)\n", 
diff --git a/source4/torture/rpc/countcalls.c b/source4/torture/rpc/countcalls.c
index ac4788f349..cd78e2eba4 100644
--- a/source4/torture/rpc/countcalls.c
+++ b/source4/torture/rpc/countcalls.c
@@ -56,7 +56,7 @@ BOOL torture_rpc_countcalls(void)
 	printf("\nScanning pipe '%s'\n", iface->name);
 
 	for (i=0;i<5000;i++) {
-		status = dcerpc_request(p, NULL, i, p, &stub_in, &stub_out);
+		status = dcerpc_request(p, NULL, i, False, p, &stub_in, &stub_out);
 		if (NT_STATUS_EQUAL(status, NT_STATUS_NET_WRITE_FAULT) &&
 		    p->last_fault_code == DCERPC_FAULT_OP_RNG_ERROR) break;
 		if (NT_STATUS_EQUAL(status, NT_STATUS_CONNECTION_DISCONNECTED)) {
diff --git a/source4/torture/rpc/lsa.c b/source4/torture/rpc/lsa.c
index 7180986a65..9e5129bf5a 100644
--- a/source4/torture/rpc/lsa.c
+++ b/source4/torture/rpc/lsa.c
@@ -22,6 +22,7 @@
 
 #include "includes.h"
 #include "librpc/gen_ndr/ndr_lsa.h"
+#include "lib/events/events.h"
 
 static void init_lsa_String(struct lsa_String *name, const char *s)
 {
@@ -457,7 +458,8 @@ BOOL test_many_LookupSids(struct dcerpc_pipe *p,
 		r.out.names = &names;
 		
 		status = dcerpc_lsa_LookupSids(p, mem_ctx, &r);
-		if (!NT_STATUS_IS_OK(status) && !NT_STATUS_EQUAL(status, STATUS_SOME_UNMAPPED)) {
+		if (!NT_STATUS_IS_OK(status) &&
+		    !NT_STATUS_EQUAL(status, STATUS_SOME_UNMAPPED)) {
 			printf("LookupSids failed - %s\n", nt_errstr(status));
 			return False;
 		}
@@ -508,6 +510,83 @@ BOOL test_many_LookupSids(struct dcerpc_pipe *p,
 	return True;
 }
 
+#define NUM_ASYNC_REQUESTS 1000
+
+static void lookupsids_cb(struct rpc_request *req)
+{
+	int *replies = (int *)req->async.private;
+	NTSTATUS status;
+
+	status = dcerpc_ndr_request_recv(req);
+	DEBUG(3, ("lookupsids returned %s\n", nt_errstr(status)));
+	if (!NT_STATUS_IS_OK(status)) {
+		*replies = -1;
+	}
+
+	*replies += 1;
+}
+
+static BOOL test_LookupSids_async(struct dcerpc_pipe *p, 
+				  TALLOC_CTX *mem_ctx, 
+				  struct policy_handle *handle)
+{
+	struct lsa_SidArray sids;
+	struct lsa_SidPtr sidptr;
+
+	uint32_t count[NUM_ASYNC_REQUESTS];
+	struct lsa_TransNameArray names[NUM_ASYNC_REQUESTS];
+	struct lsa_LookupSids r[NUM_ASYNC_REQUESTS];
+	struct rpc_request **req;
+
+	int i, replies;
+	BOOL ret = True;
+
+	printf("\nTesting %d async lookupsids request\n", 100);
+
+	req = talloc_array(mem_ctx, struct rpc_request *, NUM_ASYNC_REQUESTS);
+
+	sids.num_sids = 1;
+	sids.sids = &sidptr;
+	sidptr.sid = dom_sid_parse_talloc(mem_ctx, "S-1-5-32-545");
+
+	replies = 0;
+
+	for (i=0; i<NUM_ASYNC_REQUESTS; i++) {
+		count[i] = 0;
+		names[i].count = 0;
+		names[i].names = NULL;
+
+		r[i].in.handle = handle;
+		r[i].in.sids = &sids;
+		r[i].in.names = &names[i];
+		r[i].in.level = 1;
+		r[i].in.count = &names[i].count;
+		r[i].out.count = &count[i];
+		r[i].out.names = &names[i];
+		
+		req[i] = dcerpc_lsa_LookupSids_send(p, req, &r[i]);
+		if (req[i] == NULL) {
+			ret = False;
+			break;
+		}
+
+		req[i]->async.callback = lookupsids_cb;
+		req[i]->async.private = &replies;
+	}
+
+	while (replies < NUM_ASYNC_REQUESTS) {
+		event_loop_once(p->conn->event_ctx);
+		if (replies < 0) {
+			ret = False;
+			break;
+		}
+	}
+
+	talloc_free(req);
+
+	return ret;
+}
+
 static BOOL test_LookupPrivValue(struct dcerpc_pipe *p, 
 				 TALLOC_CTX *mem_ctx, 
 				 struct policy_handle *handle,
@@ -1758,6 +1837,10 @@ BOOL torture_rpc_lsa(void)
 	}
 
 	if (handle) {
+		if (!test_LookupSids_async(p, mem_ctx, handle)) {
+			ret = False;
+		}
+
 		if (!test_QueryDomainInfoPolicy(p, mem_ctx, handle)) {
 			ret = False;
 		}
@@ -1813,8 +1896,6 @@ BOOL torture_rpc_lsa(void)
 		}
 	}
 		
-
-	
 	talloc_free(mem_ctx);
 
 	return ret;
diff --git a/source4/torture/rpc/scanner.c b/source4/torture/rpc/scanner.c
index 9741273c64..a0adcea0ac 100644
--- a/source4/torture/rpc/scanner.c
+++ b/source4/torture/rpc/scanner.c
@@ -53,7 +53,7 @@ static BOOL test_num_calls(const struct dcerpc_interface_table *iface,
 	memset(stub_in.data, 0xFF, stub_in.length);
 
 	for (i=0;i<200;i++) {
-		status = dcerpc_request(p, NULL, i, mem_ctx, &stub_in, &stub_out);
+		status = dcerpc_request(p, NULL, False, i, mem_ctx, &stub_in, &stub_out);
 		if (!NT_STATUS_IS_OK(status) &&
 		    p->last_fault_code == DCERPC_FAULT_OP_RNG_ERROR) {
 			break;