r21278: The main goal of this was to get rid of the NetInBuffer / set_InBuffer. But it

turns out that this patch actually speeds up the async writes considerably.

I tested writing 100.000 times 65535 bytes with the allowed 10 ops in
parallel. Without this patch it took about 32 seconds on my dual-core 1.6GHz
laptop. With this patch it dropped to about 26 seconds. I can only explain it
by better cache locality, NewInBuffer allocates more than 128k, so we jump
around in memory more.

Jeremy, please check!

Volker
(This used to be commit 452d51bc6fd41771b9c41ba6391664513d7cf2cd)

2 files changed, 15 insertions, 79 deletions

diff --git a/source3/smbd/aio.c b/source3/smbd/aio.c
index 9425348474..8a9fabf228 100644
--- a/source3/smbd/aio.c
+++ b/source3/smbd/aio.c
@@ -79,7 +79,9 @@ static struct aio_extra *create_aio_ex_read(files_struct *fsp, size_t buflen,
 *****************************************************************************/
 
 static struct aio_extra *create_aio_ex_write(files_struct *fsp,
-					     size_t outbuflen, uint16 mid)
+					     size_t inbuflen,
+					     size_t outbuflen,
+					     uint16 mid)
 {
 	struct aio_extra *aio_ex = SMB_MALLOC_P(struct aio_extra);
 
@@ -94,18 +96,13 @@ static struct aio_extra *create_aio_ex_write(files_struct *fsp,
 		SAFE_FREE(aio_ex);
 		return NULL;
 	}
-	/* Steal the input buffer containing the write data from the main SMB
-	 * call. */
-	/* We must re-allocate a new one here. */
-	if (NewInBuffer(&aio_ex->inbuf) == NULL) {
+
+	if (!(aio_ex->inbuf = SMB_MALLOC_ARRAY(char, inbuflen))) {
 		SAFE_FREE(aio_ex->outbuf);
 		SAFE_FREE(aio_ex);
 		return NULL;
 	}
 
-	/* aio_ex->inbuf now contains the stolen old InBuf containing the data
-	 * to write. */
-
 	DLIST_ADD(aio_list_head, aio_ex);
 	aio_ex->fsp = fsp;
 	aio_ex->read_req = False;
@@ -120,9 +117,7 @@ static struct aio_extra *create_aio_ex_write(files_struct *fsp,
 static void delete_aio_ex(struct aio_extra *aio_ex)
 {
 	DLIST_REMOVE(aio_list_head, aio_ex);
-	/* Safe to do as we've removed ourselves from the in use list first. */
-	free_InBuffer(aio_ex->inbuf);
-
+	SAFE_FREE(aio_ex->inbuf);
 	SAFE_FREE(aio_ex->outbuf);
 	SAFE_FREE(aio_ex);
 }
@@ -288,7 +283,7 @@ BOOL schedule_aio_write_and_X(connection_struct *conn,
 {
 	struct aio_extra *aio_ex;
 	SMB_STRUCT_AIOCB *a;
-	size_t outbufsize;
+	size_t inbufsize, outbufsize;
 	BOOL write_through = BITSETW(inbuf+smb_vwv7,0);
 	size_t min_aio_write_size = lp_aio_write_size(SNUM(conn));
 
@@ -321,15 +316,16 @@ BOOL schedule_aio_write_and_X(connection_struct *conn,
 		return False;
 	}
 
+	inbufsize =  smb_len(inbuf) + 4;
 	outbufsize = smb_len(outbuf) + 4;
-	if ((aio_ex = create_aio_ex_write(fsp, outbufsize,
-					  SVAL(inbuf,smb_mid))) == NULL) {
+	if (!(aio_ex = create_aio_ex_write(fsp, inbufsize, outbufsize,
+					   SVAL(inbuf,smb_mid)))) {
 		DEBUG(0,("schedule_aio_write_and_X: malloc fail.\n"));
 		return False;
 	}
 
-	/* Paranioa.... */
-	SMB_ASSERT(aio_ex->inbuf == inbuf);
+	/* Copy the SMB header already setup in outbuf. */
+	memcpy(aio_ex->inbuf, inbuf, inbufsize);
 
 	/* Copy the SMB header already setup in outbuf. */
 	memcpy(aio_ex->outbuf, outbuf, outbufsize);
@@ -340,8 +336,7 @@ BOOL schedule_aio_write_and_X(connection_struct *conn,
 	/* Now set up the aio record for the write call. */
 	
 	a->aio_fildes = fsp->fh->fd;
-	a->aio_buf = data; /* As we've stolen inbuf this points within
-			    * inbuf. */
+	a->aio_buf = aio_ex->inbuf + (PTR_DIFF(data, inbuf));
 	a->aio_nbytes = numtowrite;
 	a->aio_offset = startpos;
 	a->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
@@ -351,9 +346,6 @@ BOOL schedule_aio_write_and_X(connection_struct *conn,
 	if (SMB_VFS_AIO_WRITE(fsp,a) == -1) {
 		DEBUG(3,("schedule_aio_wrote_and_X: aio_write failed. "
 			 "Error %s\n", strerror(errno) ));
-		/* Replace global InBuf as we're going to do a normal write. */
-		set_InBuffer(aio_ex->inbuf);
-		aio_ex->inbuf = NULL;
 		delete_aio_ex(aio_ex);
 		return False;
 	}
@@ -748,21 +740,6 @@ void cancel_aio_by_fsp(files_struct *fsp)
 	}
 }
 
-/****************************************************************************
- Check if a buffer was stolen for aio use.
-*****************************************************************************/
-
-BOOL aio_inbuffer_in_use(char *inbuf)
-{
-	struct aio_extra *aio_ex;
-
-	for( aio_ex = aio_list_head; aio_ex; aio_ex = aio_ex->next) {
-		if (aio_ex->inbuf == inbuf) {
-			return True;
-		}
-	}
-	return False;
-}
 #else
 BOOL aio_finished(void)
 {
@@ -805,9 +782,4 @@ BOOL wait_for_aio_completion(files_struct *fsp)
 {
 	return True;
 }
-
-BOOL aio_inbuffer_in_use(char *ptr)
-{
-	return False;
-}
 #endif
diff --git a/source3/smbd/process.c b/source3/smbd/process.c
index 6a6da8715f..a0e14d8445 100644
--- a/source3/smbd/process.c
+++ b/source3/smbd/process.c
@@ -1463,54 +1463,18 @@ char *get_InBuffer(void)
 	return InBuffer;
 }
 
-void set_InBuffer(char *new_inbuf)
-{
-	InBuffer = new_inbuf;
-	current_inbuf = InBuffer;
-}
-
 char *get_OutBuffer(void)
 {
 	return OutBuffer;
 }
 
-void set_OutBuffer(char *new_outbuf)
-{
-	OutBuffer = new_outbuf;
-}
-
-/****************************************************************************
- Free an InBuffer. Checks if not in use by aio system.
- Must have been allocated by NewInBuffer.
-****************************************************************************/
-
-void free_InBuffer(char *inbuf)
-{
-	if (!aio_inbuffer_in_use(inbuf)) {
-		if (current_inbuf == inbuf) {
-			current_inbuf = NULL;
-		}
-		SAFE_FREE(inbuf);
-	}
-}
-
-/****************************************************************************
- Free an OutBuffer. No outbuffers currently stolen by aio system.
- Must have been allocated by NewInBuffer.
-****************************************************************************/
-
-void free_OutBuffer(char *outbuf)
-{
-	SAFE_FREE(outbuf);
-}
-
 const int total_buffer_size = (BUFFER_SIZE + LARGE_WRITEX_HDR_SIZE + SAFETY_MARGIN);
 
 /****************************************************************************
  Allocate a new InBuffer. Returns the new and old ones.
 ****************************************************************************/
 
-char *NewInBuffer(char **old_inbuf)
+static char *NewInBuffer(char **old_inbuf)
 {
 	char *new_inbuf = (char *)SMB_MALLOC(total_buffer_size);
 	if (!new_inbuf) {
@@ -1530,7 +1494,7 @@ char *NewInBuffer(char **old_inbuf)
  Allocate a new OutBuffer. Returns the new and old ones.
 ****************************************************************************/
 
-char *NewOutBuffer(char **old_outbuf)
+static char *NewOutBuffer(char **old_outbuf)
 {
 	char *new_outbuf = (char *)SMB_MALLOC(total_buffer_size);
 	if (!new_outbuf) {