From f088353d37b433af7b979a17871233cccddf7aca Mon Sep 17 00:00:00 2001 From: Simo Sorce Date: Mon, 9 Mar 2009 18:04:38 -0400 Subject: Fix potential segfaults using freed memory. In some code paths ltdb_context was still referenced even after we were returned an error by one of the callbacks. Because the interface assumes that once an error is returned the ldb_request may be freed, and because the ltdb_context was allocated as a child of the request, this might cause access to freed memory. Allocate the ltdb_context on ldb, and keep track of what's going on with the request by adding a spy children on it. This way even if the request is freed before the ltdb_callback is called, we will safely free the ctx and just quietly return. --- source4/lib/ldb/ldb_tdb/ldb_index.c | 2 +- source4/lib/ldb/ldb_tdb/ldb_search.c | 6 ++--- source4/lib/ldb/ldb_tdb/ldb_tdb.c | 49 ++++++++++++++++++++++++++++++++---- source4/lib/ldb/ldb_tdb/ldb_tdb.h | 7 +++++- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c index ad27c9a9a9..c99c2936d8 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_index.c +++ b/source4/lib/ldb/ldb_tdb/ldb_index.c @@ -1055,7 +1055,7 @@ static int ltdb_index_filter(const struct dn_list *dn_list, ret = ldb_module_send_entry(ac->req, msg, NULL); if (ret != LDB_SUCCESS) { - ac->callback_failed = true; + ac->request_terminated = true; return ret; } } diff --git a/source4/lib/ldb/ldb_tdb/ldb_search.c b/source4/lib/ldb/ldb_tdb/ldb_search.c index 0f595267fc..d395c28f28 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_search.c +++ b/source4/lib/ldb/ldb_tdb/ldb_search.c @@ -424,10 +424,10 @@ static int search_func(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, voi ret = ldb_module_send_entry(ac->req, msg, NULL); if (ret != LDB_SUCCESS) { - ac->callback_failed = true; + ac->request_terminated = true; /* the callback failed, abort the operation */ return -1; - } + } return 0; } @@ -544,7 +544,7 @@ int ltdb_search(struct ltdb_context *ctx) /* Check if we got just a normal error. * In that case proceed to a full search unless we got a * callback error */ - if ( ! ctx->callback_failed && ret != LDB_SUCCESS) { + if ( ! ctx->request_terminated && ret != LDB_SUCCESS) { /* Not indexed, so we need to do a full scan */ ret = ltdb_search_full(ctx); if (ret != LDB_SUCCESS) { diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c index 24ec06ea32..d38cb828bb 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c @@ -1019,7 +1019,15 @@ static void ltdb_timeout(struct tevent_context *ev, struct ltdb_context *ctx; ctx = talloc_get_type(private_data, struct ltdb_context); - ltdb_request_done(ctx, LDB_ERR_TIME_LIMIT_EXCEEDED); + if (!ctx->request_terminated) { + /* neutralize the spy */ + ctx->spy->ctx = NULL; + + /* request is done now */ + ltdb_request_done(ctx, LDB_ERR_TIME_LIMIT_EXCEEDED); + } + + talloc_free(ctx); } static void ltdb_request_extended_done(struct ltdb_context *ctx, @@ -1078,6 +1086,11 @@ static void ltdb_callback(struct tevent_context *ev, ctx = talloc_get_type(private_data, struct ltdb_context); + if (!ctx->request_terminated) { + /* neutralize the spy */ + ctx->spy->ctx = NULL; + } else goto done; + switch (ctx->req->operation) { case LDB_SEARCH: ret = ltdb_search(ctx); @@ -1102,11 +1115,24 @@ static void ltdb_callback(struct tevent_context *ev, ret = LDB_ERR_UNWILLING_TO_PERFORM; } - if (!ctx->callback_failed) { - /* Once we are done, we do not need timeout events */ - talloc_free(ctx->timeout_event); + if (!ctx->request_terminated) { + /* request is done now */ ltdb_request_done(ctx, ret); } + +done: + talloc_free(ctx); +} + +static int ltdb_request_destructor(void *ptr) +{ + struct ltdb_req_spy *spy = talloc_get_type(ptr, struct ltdb_req_spy); + + if (spy->ctx != NULL) { + spy->ctx->request_terminated = true; + } + + return 0; } static int ltdb_handle_request(struct ldb_module *module, @@ -1131,7 +1157,7 @@ static int ltdb_handle_request(struct ldb_module *module, ev = ldb_get_event_context(ldb); - ac = talloc_zero(req, struct ltdb_context); + ac = talloc_zero(ldb, struct ltdb_context); if (ac == NULL) { ldb_set_errstring(ldb, "Out of Memory"); return LDB_ERR_OPERATIONS_ERROR; @@ -1144,15 +1170,28 @@ static int ltdb_handle_request(struct ldb_module *module, tv.tv_usec = 0; te = tevent_add_timer(ev, ac, tv, ltdb_callback, ac); if (NULL == te) { + talloc_free(ac); return LDB_ERR_OPERATIONS_ERROR; } tv.tv_sec = req->starttime + req->timeout; ac->timeout_event = tevent_add_timer(ev, ac, tv, ltdb_timeout, ac); if (NULL == ac->timeout_event) { + talloc_free(ac); return LDB_ERR_OPERATIONS_ERROR; } + /* set a spy so that we do not try to use the request context + * if it is freed before ltdb_callback fires */ + ac->spy = talloc(req, struct ltdb_req_spy); + if (NULL == ac->spy) { + talloc_free(ac); + return LDB_ERR_OPERATIONS_ERROR; + } + ac->spy->ctx = ac; + + talloc_set_destructor((TALLOC_CTX *)ac->spy, ltdb_request_destructor); + return LDB_SUCCESS; } diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h index 0a06cdb1b0..5a1c8fee2d 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h +++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h @@ -36,11 +36,16 @@ struct ltdb_private { the async local context holds also internal search state during a full db search */ +struct ltdb_req_spy { + struct ltdb_context *ctx; +}; + struct ltdb_context { struct ldb_module *module; struct ldb_request *req; - bool callback_failed; + bool request_terminated; + struct ltdb_req_spy *spy; /* search stuff */ const struct ldb_parse_tree *tree; -- cgit From a67bc41d6bc25974e6aa6d864d5265d07c890413 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 10 Mar 2009 15:12:48 +1100 Subject: added a test that triggers a bug in the Samba3 notify code The Samba3 internal notify code doesn't work correctly when there is more than one tree connect to the same smbd process. This change to the RAW-NOTIFY test triggers the bug. --- source4/torture/raw/notify.c | 169 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/source4/torture/raw/notify.c b/source4/torture/raw/notify.c index 3ffc58dbe6..c92170cf61 100644 --- a/source4/torture/raw/notify.c +++ b/source4/torture/raw/notify.c @@ -1429,6 +1429,174 @@ done: return ret; } + +/* + create a secondary tree connect - used to test for a bug in Samba3 messaging + with change notify +*/ +static struct smbcli_tree *secondary_tcon(struct smbcli_state *cli, + struct torture_context *tctx) +{ + NTSTATUS status; + const char *share, *host; + struct smbcli_tree *tree; + union smb_tcon tcon; + + share = torture_setting_string(tctx, "share", NULL); + host = torture_setting_string(tctx, "host", NULL); + + printf("create a second tree context on the same session\n"); + tree = smbcli_tree_init(cli->session, tctx, false); + + tcon.generic.level = RAW_TCON_TCONX; + tcon.tconx.in.flags = 0; + tcon.tconx.in.password = data_blob(NULL, 0); + tcon.tconx.in.path = talloc_asprintf(tctx, "\\\\%s\\%s", host, share); + tcon.tconx.in.device = "A:"; + status = smb_raw_tcon(tree, tctx, &tcon); + if (!NT_STATUS_IS_OK(status)) { + talloc_free(tree); + printf("Failed to create secondary tree\n"); + return NULL; + } + + tree->tid = tcon.tconx.out.tid; + printf("tid1=%d tid2=%d\n", cli->tree->tid, tree->tid); + + return tree; +} + + +/* + very simple change notify test +*/ +static bool test_notify_tcon(struct smbcli_state *cli, struct torture_context *torture) +{ + bool ret = true; + NTSTATUS status; + union smb_notify notify; + union smb_open io; + int fnum, fnum2; + struct smbcli_request *req; + extern int torture_numops; + struct smbcli_tree *tree = NULL; + + printf("TESTING SIMPLE CHANGE NOTIFY\n"); + + /* + get a handle on the directory + */ + io.generic.level = RAW_OPEN_NTCREATEX; + io.ntcreatex.in.root_fid = 0; + io.ntcreatex.in.flags = 0; + io.ntcreatex.in.access_mask = SEC_FILE_ALL; + io.ntcreatex.in.create_options = NTCREATEX_OPTIONS_DIRECTORY; + io.ntcreatex.in.file_attr = FILE_ATTRIBUTE_NORMAL; + io.ntcreatex.in.share_access = NTCREATEX_SHARE_ACCESS_READ | NTCREATEX_SHARE_ACCESS_WRITE; + io.ntcreatex.in.alloc_size = 0; + io.ntcreatex.in.open_disposition = NTCREATEX_DISP_OPEN; + io.ntcreatex.in.impersonation = NTCREATEX_IMPERSONATION_ANONYMOUS; + io.ntcreatex.in.security_flags = 0; + io.ntcreatex.in.fname = BASEDIR; + + status = smb_raw_open(cli->tree, torture, &io); + CHECK_STATUS(status, NT_STATUS_OK); + fnum = io.ntcreatex.out.file.fnum; + + status = smb_raw_open(cli->tree, torture, &io); + CHECK_STATUS(status, NT_STATUS_OK); + fnum2 = io.ntcreatex.out.file.fnum; + + /* ask for a change notify, + on file or directory name changes */ + notify.nttrans.level = RAW_NOTIFY_NTTRANS; + notify.nttrans.in.buffer_size = 1000; + notify.nttrans.in.completion_filter = FILE_NOTIFY_CHANGE_NAME; + notify.nttrans.in.file.fnum = fnum; + notify.nttrans.in.recursive = true; + + printf("testing notify mkdir\n"); + req = smb_raw_changenotify_send(cli->tree, ¬ify); + smbcli_mkdir(cli->tree, BASEDIR "\\subdir-name"); + + status = smb_raw_changenotify_recv(req, torture, ¬ify); + CHECK_STATUS(status, NT_STATUS_OK); + + CHECK_VAL(notify.nttrans.out.num_changes, 1); + CHECK_VAL(notify.nttrans.out.changes[0].action, NOTIFY_ACTION_ADDED); + CHECK_WSTR(notify.nttrans.out.changes[0].name, "subdir-name", STR_UNICODE); + + printf("testing notify rmdir\n"); + req = smb_raw_changenotify_send(cli->tree, ¬ify); + smbcli_rmdir(cli->tree, BASEDIR "\\subdir-name"); + + status = smb_raw_changenotify_recv(req, torture, ¬ify); + CHECK_STATUS(status, NT_STATUS_OK); + CHECK_VAL(notify.nttrans.out.num_changes, 1); + CHECK_VAL(notify.nttrans.out.changes[0].action, NOTIFY_ACTION_REMOVED); + CHECK_WSTR(notify.nttrans.out.changes[0].name, "subdir-name", STR_UNICODE); + + printf("SIMPLE CHANGE NOTIFY OK\n"); + + printf("TESTING WITH SECONDARY TCON\n"); + tree = secondary_tcon(cli, torture); + + printf("testing notify mkdir\n"); + req = smb_raw_changenotify_send(cli->tree, ¬ify); + smbcli_mkdir(cli->tree, BASEDIR "\\subdir-name"); + + status = smb_raw_changenotify_recv(req, torture, ¬ify); + CHECK_STATUS(status, NT_STATUS_OK); + + CHECK_VAL(notify.nttrans.out.num_changes, 1); + CHECK_VAL(notify.nttrans.out.changes[0].action, NOTIFY_ACTION_ADDED); + CHECK_WSTR(notify.nttrans.out.changes[0].name, "subdir-name", STR_UNICODE); + + printf("testing notify rmdir\n"); + req = smb_raw_changenotify_send(cli->tree, ¬ify); + smbcli_rmdir(cli->tree, BASEDIR "\\subdir-name"); + + status = smb_raw_changenotify_recv(req, torture, ¬ify); + CHECK_STATUS(status, NT_STATUS_OK); + CHECK_VAL(notify.nttrans.out.num_changes, 1); + CHECK_VAL(notify.nttrans.out.changes[0].action, NOTIFY_ACTION_REMOVED); + CHECK_WSTR(notify.nttrans.out.changes[0].name, "subdir-name", STR_UNICODE); + + printf("CHANGE NOTIFY WITH TCON OK\n"); + + printf("Disconnecting secondary tree\n"); + status = smb_tree_disconnect(tree); + CHECK_STATUS(status, NT_STATUS_OK); + talloc_free(tree); + + printf("testing notify mkdir\n"); + req = smb_raw_changenotify_send(cli->tree, ¬ify); + smbcli_mkdir(cli->tree, BASEDIR "\\subdir-name"); + + status = smb_raw_changenotify_recv(req, torture, ¬ify); + CHECK_STATUS(status, NT_STATUS_OK); + + CHECK_VAL(notify.nttrans.out.num_changes, 1); + CHECK_VAL(notify.nttrans.out.changes[0].action, NOTIFY_ACTION_ADDED); + CHECK_WSTR(notify.nttrans.out.changes[0].name, "subdir-name", STR_UNICODE); + + printf("testing notify rmdir\n"); + req = smb_raw_changenotify_send(cli->tree, ¬ify); + smbcli_rmdir(cli->tree, BASEDIR "\\subdir-name"); + + status = smb_raw_changenotify_recv(req, torture, ¬ify); + CHECK_STATUS(status, NT_STATUS_OK); + CHECK_VAL(notify.nttrans.out.num_changes, 1); + CHECK_VAL(notify.nttrans.out.changes[0].action, NOTIFY_ACTION_REMOVED); + CHECK_WSTR(notify.nttrans.out.changes[0].name, "subdir-name", STR_UNICODE); + + printf("CHANGE NOTIFY WITH TDIS OK\n"); +done: + smb_raw_exit(cli->session); + return ret; +} + + /* basic testing of change notify */ @@ -1442,6 +1610,7 @@ bool torture_raw_notify(struct torture_context *torture, return false; } + ret &= test_notify_tcon(cli, torture); ret &= test_notify_dir(cli, cli2, torture); ret &= test_notify_mask(cli, torture); ret &= test_notify_recursive(cli, torture); -- cgit From 6f1f9f6d8d3d25ab6ed5467b81f1eed70e962d27 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 10 Mar 2009 16:45:45 +1100 Subject: fixed a bug in message handling for code the change notify code The change notify code registered a separate message handler for each tree connect. This registration uses the global messaging context. The messaging code would consider a 2nd registration for the same messaging type as being an 'update' of the handler, rather than a new handler. It also would only call the first handler in the linked list for a given message type when dispatching messages. This patch changes the messaging code to allow for multiple registrations of the same message type, and allow for multiple calls to different messaging handler for one incoming message. This fixes the problem with the test_notify_tcon() test that I recently committed to the S4 smbtorture --- source3/lib/messages.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/source3/lib/messages.c b/source3/lib/messages.c index e4b20c7493..5e11dd4e25 100644 --- a/source3/lib/messages.c +++ b/source3/lib/messages.c @@ -286,7 +286,15 @@ NTSTATUS messaging_register(struct messaging_context *msg_ctx, */ for (cb = msg_ctx->callbacks; cb != NULL; cb = cb->next) { - if (cb->msg_type == msg_type) { + /* we allow a second registration of the same message + type if it has a different private pointer. This is + needed in, for example, the internal notify code, + which creates a new notify context for each tree + connect, and expects to receive messages to each of + them. */ + if (cb->msg_type == msg_type && private_data == cb->private_data) { + DEBUG(5,("Overriding messaging pointer for type %u - private_data=%p\n", + (unsigned)msg_type, private_data)); cb->fn = fn; cb->private_data = private_data; return NT_STATUS_OK; @@ -317,6 +325,8 @@ void messaging_deregister(struct messaging_context *ctx, uint32_t msg_type, next = cb->next; if ((cb->msg_type == msg_type) && (cb->private_data == private_data)) { + DEBUG(5,("Deregistering messaging pointer for type %u - private_data=%p\n", + (unsigned)msg_type, private_data)); DLIST_REMOVE(ctx->callbacks, cb); TALLOC_FREE(cb); } @@ -362,7 +372,11 @@ void messaging_dispatch_rec(struct messaging_context *msg_ctx, if (cb->msg_type == rec->msg_type) { cb->fn(msg_ctx, cb->private_data, rec->msg_type, rec->src, &rec->buf); - return; + /* we continue looking for matching messages + after finding one. This matters for + subsystems like the internal notify code + which register more than one handler for + the same message type */ } } return; -- cgit From 143b222546f5b6bd6af000426465aa7466ae2ba3 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 9 Mar 2009 17:48:08 +0100 Subject: socket_wrapper: wrap readv() and writev() metze --- lib/socket_wrapper/socket_wrapper.c | 132 ++++++++++++++++++++++++++++++++++-- lib/socket_wrapper/socket_wrapper.h | 12 ++++ 2 files changed, 137 insertions(+), 7 deletions(-) diff --git a/lib/socket_wrapper/socket_wrapper.c b/lib/socket_wrapper/socket_wrapper.c index 97e60468c4..44082e78a1 100644 --- a/lib/socket_wrapper/socket_wrapper.c +++ b/lib/socket_wrapper/socket_wrapper.c @@ -1,6 +1,6 @@ /* * Copyright (C) Jelmer Vernooij 2005,2008 - * Copyright (C) Stefan Metzmacher 2006 + * Copyright (C) Stefan Metzmacher 2006-2009 * * All rights reserved. * @@ -121,6 +121,8 @@ #define real_ioctl ioctl #define real_recv recv #define real_send send +#define real_readv readv +#define real_writev writev #define real_socket socket #define real_close close #endif @@ -1698,13 +1700,7 @@ _PUBLIC_ int swrap_connect(int s, const struct sockaddr *serv_addr, socklen_t ad si->peername_len = addrlen; si->peername = sockaddr_dup(serv_addr, addrlen); si->connected = 1; - } - - if (si->type != SOCK_STREAM) { - return ret; - } - if (ret == 0) { swrap_dump_packet(si, serv_addr, SWRAP_CONNECT_RECV, NULL, 0); swrap_dump_packet(si, serv_addr, SWRAP_CONNECT_ACK, NULL, 0); } else { @@ -2018,6 +2014,128 @@ _PUBLIC_ ssize_t swrap_send(int s, const void *buf, size_t len, int flags) return ret; } +int swrap_readv(int s, const struct iovec *vector, size_t count) +{ + int ret; + struct socket_info *si = find_socket_info(s); + struct iovec v; + + if (!si) { + return real_readv(s, vector, count); + } + + /* we read 1500 bytes as maximum */ + if (count > 0) { + size_t i, len = 0; + + for (i=0; i < count; i++) { + size_t nlen; + nlen = len + vector[i].iov_len; + if (nlen > 1500) { + break; + } + } + count = i; + if (count == 0) { + v = vector[0]; + v.iov_len = MIN(v.iov_len, 1500); + vector = &v; + count = 1; + } + } + + ret = real_readv(s, vector, count); + if (ret == -1 && errno != EAGAIN && errno != ENOBUFS) { + swrap_dump_packet(si, NULL, SWRAP_RECV_RST, NULL, 0); + } else if (ret == 0) { /* END OF FILE */ + swrap_dump_packet(si, NULL, SWRAP_RECV_RST, NULL, 0); + } else if (ret > 0) { + uint8_t *buf; + off_t ofs = 0; + size_t i; + + /* we capture it as one single packet */ + buf = (uint8_t *)malloc(ret); + if (!buf) { + /* we just not capture the packet */ + errno = 0; + return ret; + } + + for (i=0; i < count; i++) { + memcpy(buf + ofs, + vector[i].iov_base, + vector[i].iov_len); + ofs += vector[i].iov_len; + } + + swrap_dump_packet(si, NULL, SWRAP_RECV, buf, ret); + free(buf); + } + + return ret; +} + +int swrap_writev(int s, const struct iovec *vector, size_t count) +{ + int ret; + struct socket_info *si = find_socket_info(s); + struct iovec v; + + if (!si) { + return real_writev(s, vector, count); + } + + /* we write 1500 bytes as maximum */ + if (count > 0) { + size_t i, len = 0; + + for (i=0; i < count; i++) { + size_t nlen; + nlen = len + vector[i].iov_len; + if (nlen > 1500) { + break; + } + } + count = i; + if (count == 0) { + v = vector[0]; + v.iov_len = MIN(v.iov_len, 1500); + vector = &v; + count = 1; + } + } + + ret = real_writev(s, vector, count); + if (ret == -1) { + swrap_dump_packet(si, NULL, SWRAP_SEND_RST, NULL, 0); + } else { + uint8_t *buf; + off_t ofs = 0; + size_t i; + + /* we capture it as one single packet */ + buf = (uint8_t *)malloc(ret); + if (!buf) { + /* we just not capture the packet */ + errno = 0; + return ret; + } + + for (i=0; i < count; i++) { + memcpy(buf + ofs, + vector[i].iov_base, + vector[i].iov_len); + ofs += vector[i].iov_len; + } + + swrap_dump_packet(si, NULL, SWRAP_SEND, buf, ret); + free(buf); + } + + return ret; +} + _PUBLIC_ int swrap_close(int fd) { struct socket_info *si = find_socket_info(fd); diff --git a/lib/socket_wrapper/socket_wrapper.h b/lib/socket_wrapper/socket_wrapper.h index cc8b937608..b2d44769ff 100644 --- a/lib/socket_wrapper/socket_wrapper.h +++ b/lib/socket_wrapper/socket_wrapper.h @@ -52,6 +52,8 @@ ssize_t swrap_sendto(int s, const void *buf, size_t len, int flags, const struct int swrap_ioctl(int s, int req, void *ptr); ssize_t swrap_recv(int s, void *buf, size_t len, int flags); ssize_t swrap_send(int s, const void *buf, size_t len, int flags); +int swrap_readv(int s, const struct iovec *vector, size_t count); +int swrap_writev(int s, const struct iovec *vector, size_t count); int swrap_close(int); #ifdef SOCKET_WRAPPER_REPLACE @@ -121,6 +123,16 @@ int swrap_close(int); #endif #define send(s,buf,len,flags) swrap_send(s,buf,len,flags) +#ifdef readv +#undef readv +#endif +#define readv(s, vector, count) swrap_readv(s,vector, count) + +#ifdef writev +#undef writev +#endif +#define writev(s, vector, count) swrap_writev(s,vector, count) + #ifdef socket #undef socket #endif -- cgit From 7fc8086e11497c96be72859a510f72cb3c4104d5 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 10 Mar 2009 10:56:33 +0100 Subject: s3:libsmb: fix a lot of cli_push() bugs There were the following problems: 1.) if window_size was a multiple of the chunk_size, we silently dropped the last truncated chunk. 2.) if window_size was 0 pushed only the first chunk to the server and silently dropped the rest. 3.) we had only transferred state->num_reqs writes, even if there would be more data to send. metze --- source3/libsmb/clireadwrite.c | 202 ++++++++++++++++++++++-------------------- 1 file changed, 107 insertions(+), 95 deletions(-) diff --git a/source3/libsmb/clireadwrite.c b/source3/libsmb/clireadwrite.c index 9d17ff86a5..764ef631f2 100644 --- a/source3/libsmb/clireadwrite.c +++ b/source3/libsmb/clireadwrite.c @@ -915,9 +915,15 @@ static NTSTATUS cli_writeall_recv(struct async_req *req) return async_req_simple_recv_ntstatus(req); } -struct cli_push_state { - struct async_req *req; +struct cli_push_write_state { + struct async_req *req;/* This is the main request! Not the subreq */ + uint32_t idx; + off_t ofs; + uint8_t *buf; + size_t size; +}; +struct cli_push_state { struct event_context *ev; struct cli_state *cli; uint16_t fnum; @@ -928,24 +934,72 @@ struct cli_push_state { size_t (*source)(uint8_t *buf, size_t n, void *priv); void *priv; - size_t chunk_size; - - size_t sent; bool eof; + size_t chunk_size; + off_t next_offset; + /* * Outstanding requests */ - int num_reqs; - struct async_req **reqs; - - int pending; - - uint8_t *buf; + uint32_t pending; + uint32_t num_reqs; + struct cli_push_write_state **reqs; }; static void cli_push_written(struct async_req *req); +static bool cli_push_write_setup(struct async_req *req, + struct cli_push_state *state, + uint32_t idx) +{ + struct cli_push_write_state *substate; + struct async_req *subreq; + + substate = talloc(state->reqs, struct cli_push_write_state); + if (!substate) { + return false; + } + substate->req = req; + substate->idx = idx; + substate->ofs = state->next_offset; + substate->buf = talloc_array(substate, uint8_t, state->chunk_size); + if (!substate->buf) { + talloc_free(substate); + return false; + } + substate->size = state->source(substate->buf, + state->chunk_size, + state->priv); + if (substate->size < state->chunk_size) { + state->eof = true; + } + if (substate->size == 0) { + /* nothing to send */ + talloc_free(substate); + return true; + } + + subreq = cli_writeall_send(substate, + state->ev, state->cli, + state->fnum, state->mode, + substate->buf, + substate->ofs, + substate->size); + if (!subreq) { + talloc_free(substate); + return false; + } + subreq->async.fn = cli_push_written; + subreq->async.priv = substate; + + state->reqs[idx] = substate; + state->pending += 1; + state->next_offset += substate->size; + + return true; +} + struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, struct cli_state *cli, uint16_t fnum, uint16_t mode, @@ -954,16 +1008,14 @@ struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, void *priv), void *priv) { - struct async_req *result; + struct async_req *req; struct cli_push_state *state; - int i; + uint32_t i; - if (!async_req_setup(mem_ctx, &result, &state, + if (!async_req_setup(mem_ctx, &req, &state, struct cli_push_state)) { return NULL; } - state->req = result; - state->cli = cli; state->ev = ev; state->fnum = fnum; @@ -972,124 +1024,84 @@ struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, state->source = source; state->priv = priv; state->eof = false; - state->sent = 0; state->pending = 0; + state->next_offset = start_offset; state->chunk_size = cli_write_max_bufsize(cli, mode); - state->num_reqs = MAX(window_size/state->chunk_size, 1); + if (window_size == 0) { + window_size = cli->max_mux * state->chunk_size; + } + state->num_reqs = window_size/state->chunk_size; + if ((window_size % state->chunk_size) > 0) { + state->num_reqs += 1; + } state->num_reqs = MIN(state->num_reqs, cli->max_mux); + state->num_reqs = MAX(state->num_reqs, 1); - state->reqs = TALLOC_ZERO_ARRAY(state, struct async_req *, + state->reqs = TALLOC_ZERO_ARRAY(state, struct cli_push_write_state *, state->num_reqs); if (state->reqs == NULL) { goto failed; } - state->buf = TALLOC_ARRAY( - state, uint8_t, state->chunk_size * state->num_reqs); - if (state->buf == NULL) { - goto failed; - } - for (i=0; inum_reqs; i++) { - size_t to_write; - uint8_t *buf = state->buf + i*state->chunk_size; - - to_write = state->source(buf, state->chunk_size, state->priv); - if (to_write == 0) { - state->eof = true; - break; - } - - state->reqs[i] = cli_writeall_send( - state->reqs, state->ev, state->cli, state->fnum, - state->mode, buf, state->start_offset + state->sent, - to_write); - if (state->reqs[i] == NULL) { + if (!cli_push_write_setup(req, state, i)) { goto failed; } - state->reqs[i]->async.fn = cli_push_written; - state->reqs[i]->async.priv = state; - - state->sent += to_write; - state->pending += 1; + if (state->eof) { + state->num_reqs = i+1; + break; + } } - if (i == 0) { - if (!async_post_ntstatus(result, ev, NT_STATUS_OK)) { + if (state->pending == 0) { + if (!async_post_ntstatus(req, ev, NT_STATUS_OK)) { goto failed; } - return result; + return req; } - return result; + return req; failed: - TALLOC_FREE(result); + TALLOC_FREE(req); return NULL; } -static void cli_push_written(struct async_req *req) +static void cli_push_written(struct async_req *subreq) { + struct cli_push_write_state *substate = talloc_get_type_abort( + subreq->async.priv, struct cli_push_write_state); + struct async_req *req = substate->req; struct cli_push_state *state = talloc_get_type_abort( - req->async.priv, struct cli_push_state); + req->private_data, struct cli_push_state); NTSTATUS status; - int i; - uint8_t *buf; - size_t to_write; - - for (i=0; inum_reqs; i++) { - if (state->reqs[i] == req) { - break; - } - } - - if (i == state->num_reqs) { - async_req_nterror(state->req, NT_STATUS_INTERNAL_ERROR); - return; - } - - status = cli_writeall_recv(req); - TALLOC_FREE(state->reqs[i]); - req = NULL; - if (!NT_STATUS_IS_OK(status)) { - async_req_nterror(state->req, status); - return; - } + uint32_t idx = substate->idx; + state->reqs[idx] = NULL; state->pending -= 1; - if (state->pending == 0) { - async_req_done(state->req); - return; - } - if (state->eof) { + status = cli_writeall_recv(subreq); + TALLOC_FREE(subreq); + TALLOC_FREE(substate); + if (!NT_STATUS_IS_OK(status)) { + async_req_nterror(req, status); return; } - buf = state->buf + i * state->chunk_size; - - to_write = state->source(buf, state->chunk_size, state->priv); - if (to_write == 0) { - state->eof = true; - return; + if (!state->eof) { + if (!cli_push_write_setup(req, state, idx)) { + async_req_nomem(NULL, req); + return; + } } - state->reqs[i] = cli_writeall_send( - state->reqs, state->ev, state->cli, state->fnum, - state->mode, buf, state->start_offset + state->sent, to_write); - if (state->reqs[i] == NULL) { - async_req_nterror(state->req, NT_STATUS_NO_MEMORY); + if (state->pending == 0) { + async_req_done(req); return; } - - state->reqs[i]->async.fn = cli_push_written; - state->reqs[i]->async.priv = state; - - state->sent += to_write; - state->pending += 1; } NTSTATUS cli_push_recv(struct async_req *req) -- cgit From c2993f74af4e17790f8afbf16ba1c6884afa4ad4 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 10 Mar 2009 12:32:48 +0100 Subject: s3:libsmb: only treat a return 0 as end of file metze --- source3/libsmb/clireadwrite.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source3/libsmb/clireadwrite.c b/source3/libsmb/clireadwrite.c index 764ef631f2..f2f447b4c9 100644 --- a/source3/libsmb/clireadwrite.c +++ b/source3/libsmb/clireadwrite.c @@ -971,10 +971,8 @@ static bool cli_push_write_setup(struct async_req *req, substate->size = state->source(substate->buf, state->chunk_size, state->priv); - if (substate->size < state->chunk_size) { - state->eof = true; - } if (substate->size == 0) { + state->eof = true; /* nothing to send */ talloc_free(substate); return true; @@ -1051,7 +1049,6 @@ struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, } if (state->eof) { - state->num_reqs = i+1; break; } } -- cgit From 9579a6f193f570e4ce2af80f4aac7c2f25ae5b22 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 10 Mar 2009 12:34:20 +0100 Subject: s3:libsmb: add an option to cli_push to let the caller provide the buffers metze --- source3/client/client.c | 10 ++++++---- source3/include/proto.h | 9 +++++++-- source3/libsmb/clireadwrite.c | 33 +++++++++++++++++++++++++-------- source3/torture/torture.c | 8 +++++--- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/source3/client/client.c b/source3/client/client.c index aaa9e35d96..67a2458a94 100644 --- a/source3/client/client.c +++ b/source3/client/client.c @@ -220,7 +220,9 @@ struct push_state { SMB_OFF_T nread; }; -static size_t push_source(uint8_t *buf, size_t n, void *priv) +static size_t push_source(uint8_t *inbuf, size_t n, + const uint8_t **outbuf, + void *priv) { struct push_state *state = (struct push_state *)priv; int result; @@ -229,7 +231,7 @@ static size_t push_source(uint8_t *buf, size_t n, void *priv) return 0; } - result = readfile(buf, n, state->f); + result = readfile(inbuf, n, state->f); state->nread += result; return result; } @@ -1681,8 +1683,8 @@ static int do_put(const char *rname, const char *lname, bool reput) state.f = f; state.nread = 0; - status = cli_push(targetcli, fnum, 0, 0, io_bufsize, push_source, - &state); + status = cli_push(targetcli, fnum, 0, 0, io_bufsize, + false, push_source, &state); if (!NT_STATUS_IS_OK(status)) { d_fprintf(stderr, "cli_push returned %s\n", nt_errstr(status)); } diff --git a/source3/include/proto.h b/source3/include/proto.h index a1cafb6837..794a006a68 100644 --- a/source3/include/proto.h +++ b/source3/include/proto.h @@ -2790,13 +2790,18 @@ struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, struct cli_state *cli, uint16_t fnum, uint16_t mode, off_t start_offset, size_t window_size, - size_t (*source)(uint8_t *buf, size_t n, + bool caller_buffers, + size_t (*source)(uint8_t *inbuf, size_t n, + const uint8_t **outbuf, void *priv), void *priv); NTSTATUS cli_push_recv(struct async_req *req); NTSTATUS cli_push(struct cli_state *cli, uint16_t fnum, uint16_t mode, off_t start_offset, size_t window_size, - size_t (*source)(uint8_t *buf, size_t n, void *priv), + bool caller_buffers, + size_t (*source)(uint8_t *inbuf, size_t n, + const uint8_t **outbuf, + void *priv), void *priv); /* The following definitions come from libsmb/clisecdesc.c */ diff --git a/source3/libsmb/clireadwrite.c b/source3/libsmb/clireadwrite.c index f2f447b4c9..7e7cf0d682 100644 --- a/source3/libsmb/clireadwrite.c +++ b/source3/libsmb/clireadwrite.c @@ -930,8 +930,11 @@ struct cli_push_state { uint16_t mode; off_t start_offset; size_t window_size; + bool caller_buffers; - size_t (*source)(uint8_t *buf, size_t n, void *priv); + size_t (*source)(uint8_t *inbuf, size_t n, + const uint8_t **outbuf, + void *priv); void *priv; bool eof; @@ -963,13 +966,21 @@ static bool cli_push_write_setup(struct async_req *req, substate->req = req; substate->idx = idx; substate->ofs = state->next_offset; - substate->buf = talloc_array(substate, uint8_t, state->chunk_size); - if (!substate->buf) { - talloc_free(substate); - return false; + if (state->caller_buffers) { + substate->buf = NULL; + } else { + substate->buf = talloc_array(substate, uint8_t, + state->chunk_size); + if (!substate->buf) { + talloc_free(substate); + return false; + } } + + /* source function can overwrite substate->buf... */ substate->size = state->source(substate->buf, state->chunk_size, + (const uint8_t **)&substate->buf, state->priv); if (substate->size == 0) { state->eof = true; @@ -1002,7 +1013,9 @@ struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, struct cli_state *cli, uint16_t fnum, uint16_t mode, off_t start_offset, size_t window_size, - size_t (*source)(uint8_t *buf, size_t n, + bool caller_buffers, + size_t (*source)(uint8_t *inbuf, size_t n, + const uint8_t **outbuf, void *priv), void *priv) { @@ -1019,6 +1032,7 @@ struct async_req *cli_push_send(TALLOC_CTX *mem_ctx, struct event_context *ev, state->fnum = fnum; state->start_offset = start_offset; state->mode = mode; + state->caller_buffers = caller_buffers; state->source = source; state->priv = priv; state->eof = false; @@ -1108,7 +1122,10 @@ NTSTATUS cli_push_recv(struct async_req *req) NTSTATUS cli_push(struct cli_state *cli, uint16_t fnum, uint16_t mode, off_t start_offset, size_t window_size, - size_t (*source)(uint8_t *buf, size_t n, void *priv), + bool caller_buffers, + size_t (*source)(uint8_t *inbuf, size_t n, + const uint8_t **outbuf, + void *priv), void *priv) { TALLOC_CTX *frame = talloc_stackframe(); @@ -1129,7 +1146,7 @@ NTSTATUS cli_push(struct cli_state *cli, uint16_t fnum, uint16_t mode, } req = cli_push_send(frame, ev, cli, fnum, mode, start_offset, - window_size, source, priv); + window_size, caller_buffers, source, priv); if (req == NULL) { goto nomem; } diff --git a/source3/torture/torture.c b/source3/torture/torture.c index db89b05603..a563557d5f 100644 --- a/source3/torture/torture.c +++ b/source3/torture/torture.c @@ -5004,7 +5004,9 @@ static bool run_chain1(int dummy) return True; } -static size_t null_source(uint8_t *buf, size_t n, void *priv) +static size_t null_source(uint8_t *inbuf, size_t n, + const uint8_t *outbuf, + void *priv) { size_t *to_pull = (size_t *)priv; size_t thistime = *to_pull; @@ -5014,7 +5016,7 @@ static size_t null_source(uint8_t *buf, size_t n, void *priv) return 0; } - memset(buf, 0, thistime); + memset(inbuf, 0, thistime); *to_pull -= thistime; return thistime; } @@ -5057,7 +5059,7 @@ static bool run_windows_write(int dummy) } status = cli_push(cli1, fnum, 0, i * torture_blocksize, torture_blocksize, - null_source, &to_pull); + false, null_source, &to_pull); if (!NT_STATUS_IS_OK(status)) { printf("cli_push returned: %s\n", nt_errstr(status)); goto fail; -- cgit From 97cccb22194380ce84bf6188df90e2438f1176db Mon Sep 17 00:00:00 2001 From: Simo Sorce Date: Tue, 10 Mar 2009 10:05:52 -0400 Subject: Fix extended operation return path. Extended operations return was not going thorugh the same patch as others leaving the ctx around. Plus we were neutralizing the spy too early so that it didn't set the request_terminated flag as it should have. This should hopefully fix the failures in the build farm. --- source4/lib/ldb/ldb_tdb/ldb_tdb.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c index d38cb828bb..9df62be936 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c @@ -1020,13 +1020,14 @@ static void ltdb_timeout(struct tevent_context *ev, ctx = talloc_get_type(private_data, struct ltdb_context); if (!ctx->request_terminated) { - /* neutralize the spy */ - ctx->spy->ctx = NULL; - /* request is done now */ ltdb_request_done(ctx, LDB_ERR_TIME_LIMIT_EXCEEDED); } + if (!ctx->request_terminated) { + /* neutralize the spy */ + ctx->spy->ctx = NULL; + } talloc_free(ctx); } @@ -1086,10 +1087,9 @@ static void ltdb_callback(struct tevent_context *ev, ctx = talloc_get_type(private_data, struct ltdb_context); - if (!ctx->request_terminated) { - /* neutralize the spy */ - ctx->spy->ctx = NULL; - } else goto done; + if (ctx->request_terminated) { + goto done; + } switch (ctx->req->operation) { case LDB_SEARCH: @@ -1109,7 +1109,7 @@ static void ltdb_callback(struct tevent_context *ev, break; case LDB_EXTENDED: ltdb_handle_extended(ctx); - return; + goto done; default: /* no other op supported */ ret = LDB_ERR_UNWILLING_TO_PERFORM; @@ -1121,6 +1121,10 @@ static void ltdb_callback(struct tevent_context *ev, } done: + if (!ctx->request_terminated) { + /* neutralize the spy */ + ctx->spy->ctx = NULL; + } talloc_free(ctx); } -- cgit From 09d1b2324cf02c20daa005e6f5d55dc107303af7 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 10 Mar 2009 13:54:57 +0100 Subject: tevent: add tevent_req_received() function This function can be called as last action of a _recv() function, it destroys the data attached to the tevent_req. metze --- lib/tevent/tevent.h | 8 +++++++- lib/tevent/tevent_internal.h | 2 +- lib/tevent/tevent_req.c | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/tevent/tevent.h b/lib/tevent/tevent.h index 5089d18ec2..8ae9eaf545 100644 --- a/lib/tevent/tevent.h +++ b/lib/tevent/tevent.h @@ -165,7 +165,11 @@ enum tevent_req_state { /** * No memory in between */ - TEVENT_REQ_NO_MEMORY + TEVENT_REQ_NO_MEMORY, + /** + * the request is already received by the caller + */ + TEVENT_REQ_RECEIVED }; /** @@ -238,6 +242,8 @@ bool tevent_req_is_error(struct tevent_req *req, enum tevent_req_state *state, uint64_t *error); +void tevent_req_received(struct tevent_req *req); + struct tevent_req *tevent_wakeup_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev, struct timeval wakeup_time); diff --git a/lib/tevent/tevent_internal.h b/lib/tevent/tevent_internal.h index fa73b22a48..5a645ecb60 100644 --- a/lib/tevent/tevent_internal.h +++ b/lib/tevent/tevent_internal.h @@ -56,7 +56,7 @@ struct tevent_req { /** * @brief A function to overwrite the default print function * - * The implementation doing the work may want to imeplement a + * The implementation doing the work may want to implement a * custom function to print the text representation of the async * request. */ diff --git a/lib/tevent/tevent_req.c b/lib/tevent/tevent_req.c index 9b3e00ec8f..3832088b34 100644 --- a/lib/tevent/tevent_req.c +++ b/lib/tevent/tevent_req.c @@ -256,6 +256,27 @@ bool tevent_req_is_in_progress(struct tevent_req *req) return false; } +/** + * @brief This function destroys the attached private data + * @param[in] req The finished request + * + * This function can be called as last action of a _recv() + * function, it destroys the data attached to the tevent_req. + */ +void tevent_req_received(struct tevent_req *req) +{ + talloc_free(req->data); + req->data = NULL; + req->private_print = NULL; + + talloc_free(req->internal.trigger); + req->internal.trigger = NULL; + talloc_free(req->internal.timer); + req->internal.timer = NULL; + + req->internal.state = TEVENT_REQ_RECEIVED; +} + bool tevent_req_poll(struct tevent_req *req, struct tevent_context *ev) { -- cgit From 825949b810c6e7b7b977283355e658e2edbb8932 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Tue, 10 Mar 2009 14:02:30 +0100 Subject: tevent: add tevent_req_callback_data_void() macro Callers can use it instead of tevent_req_callback_data() if they use a non talloc private pointer. metze --- lib/tevent/tevent.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tevent/tevent.h b/lib/tevent/tevent.h index 8ae9eaf545..2dadfc1abe 100644 --- a/lib/tevent/tevent.h +++ b/lib/tevent/tevent.h @@ -197,6 +197,8 @@ void *_tevent_req_data(struct tevent_req *req); #define tevent_req_callback_data(_req, _type) \ talloc_get_type_abort(_tevent_req_callback_data(_req), _type) +#define tevent_req_callback_data_void(_req) \ + _tevent_req_callback_data(_req) #define tevent_req_data(_req, _type) \ talloc_get_type_abort(_tevent_req_data(_req), _type) -- cgit From 8c41e0e5a6cdcc2bcaf12ec7916c8ae7c04bf1e7 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Sat, 14 Feb 2009 13:18:28 +0100 Subject: Don't log NDR_PRINT_DEBUG at level 0, this always ends up in syslog I think we eventually need to pass down the debuglevel here. --- librpc/ndr/ndr.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/librpc/ndr/ndr.c b/librpc/ndr/ndr.c index 2341f51faa..8188ec998f 100644 --- a/librpc/ndr/ndr.c +++ b/librpc/ndr/ndr.c @@ -179,10 +179,10 @@ _PUBLIC_ void ndr_print_debug_helper(struct ndr_print *ndr, const char *format, } for (i=0;idepth;i++) { - DEBUGADD(0,(" ")); + DEBUGADD(1,(" ")); } - DEBUGADD(0,("%s\n", s)); + DEBUGADD(1,("%s\n", s)); free(s); } @@ -211,7 +211,7 @@ _PUBLIC_ void ndr_print_debug(ndr_print_fn_t fn, const char *name, void *ptr) { struct ndr_print *ndr; - DEBUG(0,(" ")); + DEBUG(1,(" ")); ndr = talloc_zero(NULL, struct ndr_print); if (!ndr) return; @@ -229,7 +229,7 @@ _PUBLIC_ void ndr_print_union_debug(ndr_print_fn_t fn, const char *name, uint32_ { struct ndr_print *ndr; - DEBUG(0,(" ")); + DEBUG(1,(" ")); ndr = talloc_zero(NULL, struct ndr_print); if (!ndr) return; @@ -248,7 +248,7 @@ _PUBLIC_ void ndr_print_function_debug(ndr_print_function_t fn, const char *name { struct ndr_print *ndr; - DEBUG(0,(" ")); + DEBUG(1,(" ")); ndr = talloc_zero(NULL, struct ndr_print); if (!ndr) return; -- cgit From 66a26a0ac27462361cb94e3ef1744d275bd95da1 Mon Sep 17 00:00:00 2001 From: Brad Hards Date: Tue, 10 Mar 2009 17:11:39 +0100 Subject: Update instructions for checking out pidl. --- pidl/README | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pidl/README b/pidl/README index 8f31e408d5..c6b7e11792 100644 --- a/pidl/README +++ b/pidl/README @@ -3,8 +3,9 @@ Introduction: This directory contains the source code of the pidl (Perl IDL) compiler for Samba 4. -The main sources for pidl are available by Subversion on -svn://svnanon.samba.org/samba/branches/SAMBA_4_0/source/pidl +The main sources for pidl are available using Git as part of +the combined Samba 3 / Samba 4 tree. Use: +git clone git://git.samba.org/samba.git Pidl works by building a parse tree from a .pidl file (a simple dump of it's internal parse tree) or a .idl file -- cgit From 3d280639c4652d6cd35577e333bcd46c2517754c Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Tue, 10 Mar 2009 18:02:21 +0100 Subject: Add a vfs_preopen module to hide fs latencies --- docs-xml/manpages-3/vfs_preopen.8.xml | 115 +++++++++ source3/Makefile.in | 5 + source3/configure.in | 3 +- source3/modules/vfs_preopen.c | 456 ++++++++++++++++++++++++++++++++++ 4 files changed, 578 insertions(+), 1 deletion(-) create mode 100644 docs-xml/manpages-3/vfs_preopen.8.xml create mode 100644 source3/modules/vfs_preopen.c diff --git a/docs-xml/manpages-3/vfs_preopen.8.xml b/docs-xml/manpages-3/vfs_preopen.8.xml new file mode 100644 index 0000000000..a84d4720bb --- /dev/null +++ b/docs-xml/manpages-3/vfs_preopen.8.xml @@ -0,0 +1,115 @@ + + + + + + vfs_preopen + 8 + Samba + System Administration tools + 3.3 + + + + vfs_preopen + Hide read latencies for applications reading numbered files + + + + + vfs objects = preopen + + + + + DESCRIPTION + + This VFS module is part of the + samba + 7 suite. + + This module assists applications that want to read numbered + files in sequence with very strict latency requirements. One area + where this happens in video streaming applications that want to read + one file per frame. + + When you use this module, a number of helper processes is + started that speculatively open files and read a number of bytes to + prime the file system cache, so that later on when the real + application's request comes along, no disk access is necessary. + + This module is stackable. + + + + + + OPTIONS + + + + + preopen:names = /pattern/ + + + preopen:names specifies the file name pattern which should + trigger the preopen helpers to do their work. We assume that + the files are numbered incrementally. So if your file names + are numbered FRAME00000.frm FRAME00001.frm and so on you would + list them as preopen:names=/FRAME*.frm/ + + + + + + preopen:num_bytes = BYTES + + + Specifies the number of bytes the helpers should speculatively + read, defaults to 1. + + + + + + preopen:helpers = NUM-PROCS + + + Number of forked helper processes, defaults to 1. + + + + + + preopen:queuelen = NUM-FILES + + + Number of files that should be speculatively opened. Defaults + to the 10 subsequent files. + + + + + + + + + VERSION + This man page is correct for version 3.3 of the Samba suite. + + + + + AUTHOR + + The original Samba software and related utilities + were created by Andrew Tridgell. Samba is now developed + by the Samba Team as an Open Source project similar + to the way the Linux kernel is developed. + + The PREOPEN VFS module was created with contributions from + Volker Lendecke and the developers at IBM. + + + + diff --git a/source3/Makefile.in b/source3/Makefile.in index 6aabcf0c8d..76fd91a31e 100644 --- a/source3/Makefile.in +++ b/source3/Makefile.in @@ -667,6 +667,7 @@ VFS_READAHEAD_OBJ = modules/vfs_readahead.o VFS_TSMSM_OBJ = modules/vfs_tsmsm.o VFS_FILEID_OBJ = modules/vfs_fileid.o VFS_AIO_FORK_OBJ = modules/vfs_aio_fork.o +VFS_PREOPEN_OBJ = modules/vfs_preopen.o VFS_SYNCOPS_OBJ = modules/vfs_syncops.o VFS_ACL_XATTR_OBJ = modules/vfs_acl_xattr.o VFS_ACL_TDB_OBJ = modules/vfs_acl_tdb.o @@ -2567,6 +2568,10 @@ bin/aio_fork.@SHLIBEXT@: $(BINARY_PREREQS) $(VFS_AIO_FORK_OBJ) @echo "Building plugin $@" @$(SHLD_MODULE) $(VFS_AIO_FORK_OBJ) +bin/preopen.@SHLIBEXT@: $(BINARY_PREREQS) $(VFS_PREOPEN_OBJ) + @echo "Building plugin $@" + @$(SHLD_MODULE) $(VFS_PREOPEN_OBJ) + bin/acl_xattr.@SHLIBEXT@: $(BINARY_PREREQS) $(VFS_ACL_XATTR_OBJ) @echo "Building plugin $@" @$(SHLD_MODULE) $(VFS_ACL_XATTR_OBJ) diff --git a/source3/configure.in b/source3/configure.in index e48ff34554..2af1545d58 100644 --- a/source3/configure.in +++ b/source3/configure.in @@ -417,7 +417,7 @@ dnl These have to be built static: default_static_modules="pdb_smbpasswd pdb_tdbsam pdb_wbc_sam rpc_lsarpc rpc_samr rpc_winreg rpc_initshutdown rpc_dssetup rpc_wkssvc rpc_svcctl rpc_ntsvcs rpc_netlogon rpc_netdfs rpc_srvsvc rpc_spoolss2 rpc_eventlog auth_sam auth_unix auth_winbind auth_wbc auth_server auth_domain auth_builtin auth_netlogond vfs_default nss_info_template" dnl These are preferably build shared, and static if dlopen() is not available -default_shared_modules="vfs_recycle vfs_audit vfs_extd_audit vfs_full_audit vfs_netatalk vfs_fake_perms vfs_default_quota vfs_readonly vfs_cap vfs_expand_msdfs vfs_shadow_copy vfs_shadow_copy2 charset_CP850 charset_CP437 auth_script vfs_readahead vfs_xattr_tdb vfs_streams_xattr vfs_streams_depot vfs_acl_xattr vfs_acl_tdb vfs_smb_traffic_analyzer" +default_shared_modules="vfs_recycle vfs_audit vfs_extd_audit vfs_full_audit vfs_netatalk vfs_fake_perms vfs_default_quota vfs_readonly vfs_cap vfs_expand_msdfs vfs_shadow_copy vfs_shadow_copy2 charset_CP850 charset_CP437 auth_script vfs_readahead vfs_xattr_tdb vfs_streams_xattr vfs_streams_depot vfs_acl_xattr vfs_acl_tdb vfs_smb_traffic_analyzer vfs_preopen" if test "x$developer" = xyes; then default_static_modules="$default_static_modules rpc_rpcecho" @@ -6185,6 +6185,7 @@ SMB_MODULE(vfs_readahead, \$(VFS_READAHEAD_OBJ), "bin/readahead.$SHLIBEXT", VFS) SMB_MODULE(vfs_tsmsm, \$(VFS_TSMSM_OBJ), "bin/tsmsm.$SHLIBEXT", VFS) SMB_MODULE(vfs_fileid, \$(VFS_FILEID_OBJ), "bin/fileid.$SHLIBEXT", VFS) SMB_MODULE(vfs_aio_fork, \$(VFS_AIO_FORK_OBJ), "bin/aio_fork.$SHLIBEXT", VFS) +SMB_MODULE(vfs_preopen, \$(VFS_PREOPEN_OBJ), "bin/preopen.$SHLIBEXT", VFS) SMB_MODULE(vfs_syncops, \$(VFS_SYNCOPS_OBJ), "bin/syncops.$SHLIBEXT", VFS) SMB_MODULE(vfs_zfsacl, \$(VFS_ZFSACL_OBJ), "bin/zfsacl.$SHLIBEXT", VFS) SMB_MODULE(vfs_notify_fam, \$(VFS_NOTIFY_FAM_OBJ), "bin/notify_fam.$SHLIBEXT", VFS) diff --git a/source3/modules/vfs_preopen.c b/source3/modules/vfs_preopen.c new file mode 100644 index 0000000000..25b9e7f3e4 --- /dev/null +++ b/source3/modules/vfs_preopen.c @@ -0,0 +1,456 @@ +/* + * Force a readahead of files by opening them and reading the first bytes + * + * Copyright (C) Volker Lendecke 2008 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "includes.h" + +struct preopen_state; + +struct preopen_helper { + struct preopen_state *state; + struct fd_event *fde; + pid_t pid; + int fd; + bool busy; +}; + +struct preopen_state { + int num_helpers; + struct preopen_helper *helpers; + + size_t to_read; /* How many bytes to read in children? */ + int queue_max; + + char *template_fname; /* Filename to be sent to children */ + size_t number_start; /* start offset into "template_fname" */ + int num_digits; /* How many digits is the number long? */ + + int fnum_sent; /* last fname sent to children */ + + int fnum_queue_end; /* last fname to be sent, based on + * last open call + preopen:queuelen + */ + + name_compare_entry *preopen_names; +}; + +static void preopen_helper_destroy(struct preopen_helper *c) +{ + int status; + close(c->fd); + c->fd = -1; + kill(c->pid, SIGKILL); + waitpid(c->pid, &status, 0); + c->busy = true; +} + +static void preopen_queue_run(struct preopen_state *state) +{ + char *pdelimiter; + char delimiter; + + pdelimiter = state->template_fname + state->number_start + + state->num_digits; + delimiter = *pdelimiter; + + while (state->fnum_sent < state->fnum_queue_end) { + + ssize_t written; + size_t to_write; + int helper; + + for (helper=0; helpernum_helpers; helper++) { + if (state->helpers[helper].busy) { + continue; + } + break; + } + if (helper == state->num_helpers) { + /* everyone is busy */ + return; + } + + snprintf(state->template_fname + state->number_start, + state->num_digits + 1, + "%.*lu", state->num_digits, + (long unsigned int)(state->fnum_sent + 1)); + *pdelimiter = delimiter; + + to_write = talloc_get_size(state->template_fname); + written = write_data(state->helpers[helper].fd, + state->template_fname, to_write); + state->helpers[helper].busy = true; + + if (written != to_write) { + preopen_helper_destroy(&state->helpers[helper]); + } + state->fnum_sent += 1; + } +} + +static void preopen_helper_readable(struct event_context *ev, + struct fd_event *fde, uint16_t flags, + void *priv) +{ + struct preopen_helper *helper = (struct preopen_helper *)priv; + struct preopen_state *state = helper->state; + ssize_t nread; + char c; + + if ((flags & EVENT_FD_READ) == 0) { + return; + } + + nread = read(helper->fd, &c, 1); + if (nread <= 0) { + preopen_helper_destroy(helper); + return; + } + + helper->busy = false; + + preopen_queue_run(state); +} + +static int preopen_helpers_destructor(struct preopen_state *c) +{ + int i; + + for (i=0; inum_helpers; i++) { + if (c->helpers[i].fd == -1) { + continue; + } + preopen_helper_destroy(&c->helpers[i]); + } + + return 0; +} + +static bool preopen_helper_open_one(int sock_fd, char **pnamebuf, + size_t to_read, void *filebuf) +{ + char *namebuf = *pnamebuf; + ssize_t nwritten, nread; + char c = 0; + int fd; + + nread = 0; + + while ((nread == 0) || (namebuf[nread-1] != '\0')) { + ssize_t thistime; + + thistime = read(sock_fd, namebuf + nread, + talloc_get_size(namebuf) - nread); + if (thistime <= 0) { + return false; + } + + nread += thistime; + + if (nread == talloc_get_size(namebuf)) { + namebuf = TALLOC_REALLOC_ARRAY( + NULL, namebuf, char, + talloc_get_size(namebuf) * 2); + if (namebuf == NULL) { + return false; + } + *pnamebuf = namebuf; + } + } + + fd = open(namebuf, O_RDONLY); + if (fd == -1) { + goto done; + } + nread = read(fd, filebuf, to_read); + close(fd); + + done: + nwritten = write(sock_fd, &c, 1); + return true; +} + +static bool preopen_helper(int fd, size_t to_read) +{ + char *namebuf; + void *readbuf; + + namebuf = TALLOC_ARRAY(NULL, char, 1024); + if (namebuf == NULL) { + return false; + } + + readbuf = talloc_size(NULL, to_read); + if (readbuf == NULL) { + TALLOC_FREE(namebuf); + return false; + } + + while (preopen_helper_open_one(fd, &namebuf, to_read, readbuf)) { + ; + } + + TALLOC_FREE(readbuf); + TALLOC_FREE(namebuf); + return false; +} + +static NTSTATUS preopen_init_helper(struct preopen_helper *h) +{ + int fdpair[2]; + NTSTATUS status; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fdpair) == -1) { + status = map_nt_error_from_unix(errno); + DEBUG(10, ("socketpair() failed: %s\n", strerror(errno))); + return status; + } + + h->pid = sys_fork(); + + if (h->pid == -1) { + return map_nt_error_from_unix(errno); + } + + if (h->pid == 0) { + close(fdpair[0]); + preopen_helper(fdpair[1], h->state->to_read); + exit(0); + } + close(fdpair[1]); + h->fd = fdpair[0]; + h->fde = event_add_fd(smbd_event_context(), h->state, h->fd, + EVENT_FD_READ, preopen_helper_readable, h); + if (h->fde == NULL) { + close(h->fd); + h->fd = -1; + return NT_STATUS_NO_MEMORY; + } + h->busy = false; + return NT_STATUS_OK; +} + +static NTSTATUS preopen_init_helpers(TALLOC_CTX *mem_ctx, size_t to_read, + int num_helpers, int queue_max, + struct preopen_state **presult) +{ + struct preopen_state *result; + int i; + + result = talloc(mem_ctx, struct preopen_state); + if (result == NULL) { + return NT_STATUS_NO_MEMORY; + } + + result->num_helpers = num_helpers; + result->helpers = TALLOC_ARRAY(result, struct preopen_helper, + num_helpers); + if (result->helpers == NULL) { + TALLOC_FREE(result); + return NT_STATUS_NO_MEMORY; + } + + result->to_read = to_read; + result->queue_max = queue_max; + result->template_fname = NULL; + result->fnum_sent = 0; + + for (i=0; ihelpers[i].state = result; + result->helpers[i].fd = -1; + } + + talloc_set_destructor(result, preopen_helpers_destructor); + + for (i=0; ihelpers[i]); + } + + *presult = result; + return NT_STATUS_OK; +} + +static void preopen_free_helpers(void **ptr) +{ + TALLOC_FREE(*ptr); +} + +static struct preopen_state *preopen_state_get(vfs_handle_struct *handle) +{ + struct preopen_state *state; + NTSTATUS status; + const char *namelist; + + if (SMB_VFS_HANDLE_TEST_DATA(handle)) { + SMB_VFS_HANDLE_GET_DATA(handle, state, struct preopen_state, + return NULL); + return state; + } + + namelist = lp_parm_const_string(SNUM(handle->conn), "preopen", "names", + NULL); + + if (namelist == NULL) { + return NULL; + } + + status = preopen_init_helpers( + NULL, + lp_parm_int(SNUM(handle->conn), "preopen", "num_bytes", 1), + lp_parm_int(SNUM(handle->conn), "preopen", "helpers", 1), + lp_parm_int(SNUM(handle->conn), "preopen", "queuelen", 10), + &state); + if (!NT_STATUS_IS_OK(status)) { + return NULL; + } + + set_namearray(&state->preopen_names, (char *)namelist); + + if (state->preopen_names == NULL) { + TALLOC_FREE(state); + return NULL; + } + + if (!SMB_VFS_HANDLE_TEST_DATA(handle)) { + SMB_VFS_HANDLE_SET_DATA(handle, state, preopen_free_helpers, + struct preopen_state, return NULL); + } + + return state; +} + +static bool preopen_parse_fname(const char *fname, unsigned long *pnum, + size_t *pstart_idx, int *pnum_digits) +{ + const char *p, *q; + unsigned long num; + + p = strrchr_m(fname, '/'); + if (p == NULL) { + p = fname; + } + + p += 1; + while (p[0] != '\0') { + if (isdigit(p[0]) && isdigit(p[1]) && isdigit(p[2])) { + break; + } + p += 1; + } + if (*p == '\0') { + /* no digits around */ + return false; + } + + num = strtoul(p, (char **)&q, 10); + + if (num+1 < num) { + /* overflow */ + return false; + } + + *pnum = num; + *pstart_idx = (p - fname); + *pnum_digits = (q - p); + return true; +} + +static int preopen_open(vfs_handle_struct *handle, const char *fname, + files_struct *fsp, int flags, mode_t mode) +{ + struct preopen_state *state; + int res; + unsigned long num; + + DEBUG(10, ("preopen_open called on %s\n", fname)); + + state = preopen_state_get(handle); + if (state == NULL) { + return SMB_VFS_NEXT_OPEN(handle, fname, fsp, flags, mode); + } + + res = SMB_VFS_NEXT_OPEN(handle, fname, fsp, flags, mode); + if (res == -1) { + return -1; + } + + if (flags != O_RDONLY) { + return res; + } + + if (!is_in_path(fname, state->preopen_names, true)) { + DEBUG(10, ("%s does not match the preopen:names list\n", + fname)); + return res; + } + + TALLOC_FREE(state->template_fname); + state->template_fname = talloc_asprintf( + state, "%s/%s", fsp->conn->connectpath, fname); + + if (state->template_fname == NULL) { + return res; + } + + if (!preopen_parse_fname(state->template_fname, &num, + &state->number_start, &state->num_digits)) { + TALLOC_FREE(state->template_fname); + return res; + } + + if (num > state->fnum_sent) { + /* + * Helpers were too slow, there's no point in reading + * files in helpers that we already read in the + * parent. + */ + state->fnum_sent = num; + } + + if ((state->fnum_queue_end != 0) /* Something was started earlier */ + && (num < (state->fnum_queue_end - state->queue_max))) { + /* + * "num" is before the queue we announced. This means + * a new run is started. + */ + state->fnum_sent = num; + } + + state->fnum_queue_end = num + state->queue_max; + + preopen_queue_run(state); + + return res; +} + +/* VFS operations structure */ + +static vfs_op_tuple preopen_ops[] = { + {SMB_VFS_OP(preopen_open), SMB_VFS_OP_OPEN, + SMB_VFS_LAYER_TRANSPARENT}, + {SMB_VFS_OP(NULL), SMB_VFS_OP_NOOP, + SMB_VFS_LAYER_NOOP} +}; + +NTSTATUS vfs_preopen_init(void); +NTSTATUS vfs_preopen_init(void) +{ + return smb_register_vfs(SMB_VFS_INTERFACE_VERSION, + "preopen", preopen_ops); +} -- cgit