From ae795a09796b35c464c2cc89c53776e63e608686 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Thu, 6 Nov 2008 11:02:20 +1100 Subject: we need to remove the pidl Makefile on "make clean" to handle upgrades to perl versions --- pidl/config.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pidl/config.mk b/pidl/config.mk index 45582f5d56..d7a84e3fcc 100644 --- a/pidl/config.mk +++ b/pidl/config.mk @@ -28,4 +28,7 @@ $(pidldir)/lib/Parse/Pidl/Expr.pm: $(pidldir)/idl.yp testcov-html:: pidl-testcov +pidl-clean: + /bin/rm -f $(pidldir)/Makefile +clean:: pidl-clean -- cgit From 41a1c882d4adec957a1ed180b53fd6874d9524f2 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Thu, 6 Nov 2008 11:13:37 +1100 Subject: don't give errors when an empty modules list is given to ldb --- source4/lib/ldb/common/ldb_modules.c | 11 +++++++++-- source4/lib/ldb/include/ldb_private.h | 3 --- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source4/lib/ldb/common/ldb_modules.c b/source4/lib/ldb/common/ldb_modules.c index 2b453bb0c3..ab0f4c51cc 100644 --- a/source4/lib/ldb/common/ldb_modules.c +++ b/source4/lib/ldb/common/ldb_modules.c @@ -40,6 +40,9 @@ #define LDB_MODULE_PREFIX "modules:" #define LDB_MODULE_PREFIX_LEN 8 +static void *ldb_dso_load_symbol(struct ldb_context *ldb, const char *name, + const char *symbol); + void ldb_set_modules_dir(struct ldb_context *ldb, const char *path) { talloc_free(ldb->modules_dir); @@ -291,8 +294,8 @@ int ldb_register_module(const struct ldb_module_ops *ops) return 0; } -void *ldb_dso_load_symbol(struct ldb_context *ldb, const char *name, - const char *symbol) +static void *ldb_dso_load_symbol(struct ldb_context *ldb, const char *name, + const char *symbol) { char *path; void *handle; @@ -334,6 +337,10 @@ int ldb_load_modules_list(struct ldb_context *ldb, const char **module_list, str for (i = 0; module_list[i] != NULL; i++) { struct ldb_module *current; const struct ldb_module_ops *ops; + + if (strcmp(module_list[i], "") == 0) { + continue; + } ops = ldb_find_module_ops(module_list[i]); if (ops == NULL) { diff --git a/source4/lib/ldb/include/ldb_private.h b/source4/lib/ldb/include/ldb_private.h index 8f7e010bee..90c4980017 100644 --- a/source4/lib/ldb/include/ldb_private.h +++ b/source4/lib/ldb/include/ldb_private.h @@ -259,9 +259,6 @@ const char *ldb_default_modules_dir(void); int ldb_register_backend(const char *url_prefix, ldb_connect_fn); -void *ldb_dso_load_symbol(struct ldb_context *ldb, const char *name, - const char *symbol); - struct ldb_handle *ldb_handle_new(TALLOC_CTX *mem_ctx, struct ldb_context *ldb); int ldb_module_send_entry(struct ldb_request *req, -- cgit From 30eff4f31b497ac94d8ee02ee2ec24bc8865ce0d Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Thu, 6 Nov 2008 14:02:40 +1100 Subject: fixed options argument to ldb connect in python The options argument to ldb connect() needs to be marked for the same special handling as attrs in search --- source4/lib/ldb/ldb.i | 4 +++- source4/lib/ldb/ldb_wrap.c | 24 +++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/source4/lib/ldb/ldb.i b/source4/lib/ldb/ldb.i index 6013462225..35d443e809 100644 --- a/source4/lib/ldb/ldb.i +++ b/source4/lib/ldb/ldb.i @@ -190,6 +190,7 @@ PyObject *ldb_val_to_py_object(struct ldb_context *ldb_ctx, } %apply const char * const *NULL_STR_LIST { const char * const *attrs } +%apply const char * const *NULL_STR_LIST { const char * const *options } %apply const char * const *NULL_STR_LIST { const char * const *control_strings } #endif @@ -712,9 +713,10 @@ typedef struct ldb_context { %feature("docstring") connect "S.connect(url,flags=0,options=None) -> None\n" \ "Connect to a LDB URL."; ldb_error connect(const char *url, unsigned int flags = 0, - const char *options[] = NULL); + const char *const *options = NULL); ~ldb() { talloc_free($self); } + ldb_error search_ex(TALLOC_CTX *mem_ctx, ldb_dn *base = NULL, enum ldb_scope scope = LDB_SCOPE_DEFAULT, diff --git a/source4/lib/ldb/ldb_wrap.c b/source4/lib/ldb/ldb_wrap.c index bc9266a306..194b562c30 100644 --- a/source4/lib/ldb/ldb_wrap.c +++ b/source4/lib/ldb/ldb_wrap.c @@ -4823,7 +4823,7 @@ SWIGINTERN PyObject *_wrap_Ldb_connect(PyObject *SWIGUNUSEDPARM(self), PyObject ldb *arg1 = (ldb *) 0 ; char *arg2 = (char *) 0 ; unsigned int arg3 = (unsigned int) 0 ; - char **arg4 = (char **) (char **)NULL ; + char **arg4 = (char **) NULL ; void *argp1 = 0 ; int res1 = 0 ; int res2 ; @@ -4831,8 +4831,6 @@ SWIGINTERN PyObject *_wrap_Ldb_connect(PyObject *SWIGUNUSEDPARM(self), PyObject int alloc2 = 0 ; unsigned int val3 ; int ecode3 = 0 ; - void *argp4 = 0 ; - int res4 = 0 ; PyObject * obj0 = 0 ; PyObject * obj1 = 0 ; PyObject * obj2 = 0 ; @@ -4861,25 +4859,33 @@ SWIGINTERN PyObject *_wrap_Ldb_connect(PyObject *SWIGUNUSEDPARM(self), PyObject arg3 = (unsigned int)(val3); } if (obj3) { - res4 = SWIG_ConvertPtr(obj3, &argp4,SWIGTYPE_p_p_char, 0 | 0 ); - if (!SWIG_IsOK(res4)) { - SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "Ldb_connect" "', argument " "4"" of type '" "char const *[]""'"); - } - arg4 = (char **)(argp4); + if (obj3 == Py_None) { + arg4 = NULL; + } else if (PySequence_Check(obj3)) { + int i; + arg4 = talloc_array(NULL, char *, PySequence_Size(obj3)+1); + for(i = 0; i < PySequence_Size(obj3); i++) + arg4[i] = PyString_AsString(PySequence_GetItem(obj3, i)); + arg4[i] = NULL; + } else { + SWIG_exception(SWIG_TypeError, "expected sequence"); + } } if (arg1 == NULL) SWIG_exception(SWIG_ValueError, "ldb context must be non-NULL"); - result = ldb_connect(arg1,(char const *)arg2,arg3,(char const *(*))arg4); + result = ldb_connect(arg1,(char const *)arg2,arg3,(char const *const *)arg4); if (result != 0) { PyErr_SetObject(PyExc_LdbError, Py_BuildValue((char *)"(i,s)", result, ldb_errstring(arg1))); SWIG_fail; } resultobj = Py_None; if (alloc2 == SWIG_NEWOBJ) free((char*)buf2); + talloc_free(arg4); return resultobj; fail: if (alloc2 == SWIG_NEWOBJ) free((char*)buf2); + talloc_free(arg4); return NULL; } -- cgit From 2e4247782bd5812bc8e7ea24194c8436748bb2fa Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 14:36:56 +1100 Subject: make tdbbackup use transactions tdbbackup was originally written before we had transactions, and it attempted to use its own fsync() calls to make it safe. Now that we have transactions we can do it in a much safer (and faster!) fashion --- lib/tdb/tools/tdbbackup.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/tdb/tools/tdbbackup.c b/lib/tdb/tools/tdbbackup.c index 6f3ca48314..83c0e16399 100644 --- a/lib/tdb/tools/tdbbackup.c +++ b/lib/tdb/tools/tdbbackup.c @@ -126,9 +126,17 @@ static int backup_tdb(const char *old_name, const char *new_name, int hash_size) return 1; } - /* lock the old tdb */ - if (tdb_lockall(tdb) != 0) { - fprintf(stderr,"Failed to lock %s\n", old_name); + if (tdb_transaction_start(tdb) != 0) { + printf("Failed to start transaction on old tdb\n"); + tdb_close(tdb); + tdb_close(tdb_new); + unlink(tmp_name); + free(tmp_name); + return 1; + } + + if (tdb_transaction_start(tdb_new) != 0) { + printf("Failed to start transaction on new tdb\n"); tdb_close(tdb); tdb_close(tdb_new); unlink(tmp_name); @@ -152,6 +160,14 @@ static int backup_tdb(const char *old_name, const char *new_name, int hash_size) /* close the old tdb */ tdb_close(tdb); + if (tdb_transaction_commit(tdb_new) != 0) { + fprintf(stderr, "Failed to commit new tdb\n"); + tdb_close(tdb_new); + unlink(tmp_name); + free(tmp_name); + return 1; + } + /* close the new tdb and re-open read-only */ tdb_close(tdb_new); tdb_new = tdb_open(tmp_name, 0, TDB_DEFAULT, O_RDONLY, 0); @@ -173,9 +189,6 @@ static int backup_tdb(const char *old_name, const char *new_name, int hash_size) return 1; } - /* make sure the new tdb has reached stable storage */ - fsync(tdb_fd(tdb_new)); - /* close the new tdb and rename it to .bak */ tdb_close(tdb_new); if (rename(tmp_name, new_name) != 0) { -- cgit From 936d76802f98d04d9743b2ca8eeeaadd4362db51 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 14:38:17 +1100 Subject: imported the tdb_repack() code from CTDB The tdb_repack() function repacks a TDB so that it has a single freelist entry. The file doesn't shrink, but it does remove all freelist fragmentation. This code originated in the CTDB vacuuming code, but will now be used in ldb to cope with fragmentation from re-indexing --- lib/tdb/common/tdb.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/tdb/include/tdb.h | 5 ++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/lib/tdb/common/tdb.c b/lib/tdb/common/tdb.c index c7cec297f6..8c61ec1a89 100644 --- a/lib/tdb/common/tdb.c +++ b/lib/tdb/common/tdb.c @@ -800,3 +800,92 @@ failed: tdb_unlockall(tdb); return -1; } + +struct traverse_state { + bool error; + struct tdb_context *dest_db; +}; + +/* + traverse function for repacking + */ +static int repack_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private) +{ + struct traverse_state *state = (struct traverse_state *)private; + if (tdb_store(state->dest_db, key, data, TDB_INSERT) != 0) { + state->error = true; + return -1; + } + return 0; +} + +/* + repack a tdb + */ +int tdb_repack(struct tdb_context *tdb) +{ + struct tdb_context *tmp_db; + struct traverse_state state; + + if (tdb_transaction_start(tdb) != 0) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to start transaction\n")); + return -1; + } + + tmp_db = tdb_open("tmpdb", tdb_hash_size(tdb), TDB_INTERNAL, O_RDWR|O_CREAT, 0); + if (tmp_db == NULL) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to create tmp_db\n")); + tdb_transaction_cancel(tdb); + return -1; + } + + state.error = false; + state.dest_db = tmp_db; + + if (tdb_traverse_read(tdb, repack_traverse, &state) == -1) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to traverse copying out\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + if (state.error) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Error during traversal\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + if (tdb_wipe_all(tdb) != 0) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to wipe database\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + state.error = false; + state.dest_db = tdb; + + if (tdb_traverse_read(tmp_db, repack_traverse, &state) == -1) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to traverse copying back\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + if (state.error) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Error during second traversal\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + tdb_close(tmp_db); + + if (tdb_transaction_commit(tdb) != 0) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to commit\n")); + return -1; + } + + return 0; +} diff --git a/lib/tdb/include/tdb.h b/lib/tdb/include/tdb.h index c41c9941f0..94b5e366b9 100644 --- a/lib/tdb/include/tdb.h +++ b/lib/tdb/include/tdb.h @@ -152,11 +152,14 @@ int tdb_chainlock_unmark(struct tdb_context *tdb, TDB_DATA key); void tdb_setalarm_sigptr(struct tdb_context *tdb, volatile sig_atomic_t *sigptr); +/* wipe and repack */ +int tdb_wipe_all(struct tdb_context *tdb); +int tdb_repack(struct tdb_context *tdb); + /* Debug functions. Not used in production. */ void tdb_dump_all(struct tdb_context *tdb); int tdb_printfreelist(struct tdb_context *tdb); int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries); -int tdb_wipe_all(struct tdb_context *tdb); int tdb_freelist_size(struct tdb_context *tdb); extern TDB_DATA tdb_null; -- cgit From f320e3a659d2ef701e3770071375ca9781d98325 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 14:39:42 +1100 Subject: use transactions in ldbadd, ldbmodify and ldbedit The command line tools ldbadd, ldbmodify and ldbedit should operate within a transaction to make them more efficient. The ldbadd tool in particular is much faster when adding a large number of records if all the adds happen within a transaction. Previously there was a transaction per record. --- source4/lib/ldb/tools/ldbadd.c | 10 ++++++++++ source4/lib/ldb/tools/ldbedit.c | 10 ++++++++++ source4/lib/ldb/tools/ldbmodify.c | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/source4/lib/ldb/tools/ldbadd.c b/source4/lib/ldb/tools/ldbadd.c index 15376e7342..3749aec629 100644 --- a/source4/lib/ldb/tools/ldbadd.c +++ b/source4/lib/ldb/tools/ldbadd.c @@ -92,6 +92,11 @@ int main(int argc, const char **argv) options = ldb_cmdline_process(ldb, argc, argv, usage); + if (ldb_transaction_start(ldb) != 0) { + printf("Failed to start transaction\n"); + exit(1); + } + if (options->argc == 0) { ret = process_file(ldb, stdin, &count); } else { @@ -108,6 +113,11 @@ int main(int argc, const char **argv) } } + if (count != 0 && ldb_transaction_commit(ldb) != 0) { + printf("Failed to commit transaction\n"); + exit(1); + } + talloc_free(ldb); printf("Added %d records with %d failures\n", count, failures); diff --git a/source4/lib/ldb/tools/ldbedit.c b/source4/lib/ldb/tools/ldbedit.c index b2a040cd09..b18aea1b10 100644 --- a/source4/lib/ldb/tools/ldbedit.c +++ b/source4/lib/ldb/tools/ldbedit.c @@ -112,6 +112,11 @@ static int merge_edits(struct ldb_context *ldb, int ret = 0; int adds=0, modifies=0, deletes=0; + if (ldb_transaction_start(ldb) != 0) { + fprintf(stderr, "Failed to start transaction\n"); + return -1; + } + /* do the adds and modifies */ for (i=0;idn); @@ -150,6 +155,11 @@ static int merge_edits(struct ldb_context *ldb, } } + if (ldb_transaction_commit(ldb) != 0) { + fprintf(stderr, "Failed to commit transaction\n"); + return -1; + } + printf("# %d adds %d modifies %d deletes\n", adds, modifies, deletes); return ret; diff --git a/source4/lib/ldb/tools/ldbmodify.c b/source4/lib/ldb/tools/ldbmodify.c index 6e355a10cf..8b6309e016 100644 --- a/source4/lib/ldb/tools/ldbmodify.c +++ b/source4/lib/ldb/tools/ldbmodify.c @@ -91,6 +91,11 @@ int main(int argc, const char **argv) ldb = ldb_init(NULL, NULL); + if (ldb_transaction_start(ldb) != 0) { + printf("Failed to start transaction\n"); + exit(1); + } + options = ldb_cmdline_process(ldb, argc, argv, usage); if (options->argc == 0) { @@ -108,6 +113,11 @@ int main(int argc, const char **argv) } } + if (count != 0 && ldb_transaction_commit(ldb) != 0) { + printf("Failed to commit transaction\n"); + exit(1); + } + talloc_free(ldb); printf("Modified %d records with %d failures\n", count, failures); -- cgit From 4380a374c1dec46ad77939604e548f0c79d9e2ec Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 14:41:21 +1100 Subject: repack the ldb after re-indexing re-indexing in ldb is triggered on any modification to the @ATTRIBUTES or @INDEXLIST records. This happens to produce a worst-case fragmentation of the database, as all @INDEX records are deleted then re-created. By repacking after re-indexing we ensure that the database ends up without extreme fragmentation. --- source4/lib/ldb/ldb_tdb/ldb_index.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source4/lib/ldb/ldb_tdb/ldb_index.c b/source4/lib/ldb/ldb_tdb/ldb_index.c index 65711d9f4b..eedbda4170 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_index.c +++ b/source4/lib/ldb/ldb_tdb/ldb_index.c @@ -1250,5 +1250,9 @@ int ltdb_reindex(struct ldb_module *module) return LDB_ERR_OPERATIONS_ERROR; } + if (tdb_repack(ltdb->tdb) != 0) { + return LDB_ERR_OPERATIONS_ERROR; + } + return LDB_SUCCESS; } -- cgit