From 2e4247782bd5812bc8e7ea24194c8436748bb2fa Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 14:36:56 +1100 Subject: make tdbbackup use transactions tdbbackup was originally written before we had transactions, and it attempted to use its own fsync() calls to make it safe. Now that we have transactions we can do it in a much safer (and faster!) fashion --- lib/tdb/tools/tdbbackup.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/tdb/tools/tdbbackup.c b/lib/tdb/tools/tdbbackup.c index 6f3ca48314..83c0e16399 100644 --- a/lib/tdb/tools/tdbbackup.c +++ b/lib/tdb/tools/tdbbackup.c @@ -126,9 +126,17 @@ static int backup_tdb(const char *old_name, const char *new_name, int hash_size) return 1; } - /* lock the old tdb */ - if (tdb_lockall(tdb) != 0) { - fprintf(stderr,"Failed to lock %s\n", old_name); + if (tdb_transaction_start(tdb) != 0) { + printf("Failed to start transaction on old tdb\n"); + tdb_close(tdb); + tdb_close(tdb_new); + unlink(tmp_name); + free(tmp_name); + return 1; + } + + if (tdb_transaction_start(tdb_new) != 0) { + printf("Failed to start transaction on new tdb\n"); tdb_close(tdb); tdb_close(tdb_new); unlink(tmp_name); @@ -152,6 +160,14 @@ static int backup_tdb(const char *old_name, const char *new_name, int hash_size) /* close the old tdb */ tdb_close(tdb); + if (tdb_transaction_commit(tdb_new) != 0) { + fprintf(stderr, "Failed to commit new tdb\n"); + tdb_close(tdb_new); + unlink(tmp_name); + free(tmp_name); + return 1; + } + /* close the new tdb and re-open read-only */ tdb_close(tdb_new); tdb_new = tdb_open(tmp_name, 0, TDB_DEFAULT, O_RDONLY, 0); @@ -173,9 +189,6 @@ static int backup_tdb(const char *old_name, const char *new_name, int hash_size) return 1; } - /* make sure the new tdb has reached stable storage */ - fsync(tdb_fd(tdb_new)); - /* close the new tdb and rename it to .bak */ tdb_close(tdb_new); if (rename(tmp_name, new_name) != 0) { -- cgit From 936d76802f98d04d9743b2ca8eeeaadd4362db51 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 14:38:17 +1100 Subject: imported the tdb_repack() code from CTDB The tdb_repack() function repacks a TDB so that it has a single freelist entry. The file doesn't shrink, but it does remove all freelist fragmentation. This code originated in the CTDB vacuuming code, but will now be used in ldb to cope with fragmentation from re-indexing --- lib/tdb/common/tdb.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++ lib/tdb/include/tdb.h | 5 ++- 2 files changed, 93 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tdb/common/tdb.c b/lib/tdb/common/tdb.c index c7cec297f6..8c61ec1a89 100644 --- a/lib/tdb/common/tdb.c +++ b/lib/tdb/common/tdb.c @@ -800,3 +800,92 @@ failed: tdb_unlockall(tdb); return -1; } + +struct traverse_state { + bool error; + struct tdb_context *dest_db; +}; + +/* + traverse function for repacking + */ +static int repack_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, void *private) +{ + struct traverse_state *state = (struct traverse_state *)private; + if (tdb_store(state->dest_db, key, data, TDB_INSERT) != 0) { + state->error = true; + return -1; + } + return 0; +} + +/* + repack a tdb + */ +int tdb_repack(struct tdb_context *tdb) +{ + struct tdb_context *tmp_db; + struct traverse_state state; + + if (tdb_transaction_start(tdb) != 0) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to start transaction\n")); + return -1; + } + + tmp_db = tdb_open("tmpdb", tdb_hash_size(tdb), TDB_INTERNAL, O_RDWR|O_CREAT, 0); + if (tmp_db == NULL) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to create tmp_db\n")); + tdb_transaction_cancel(tdb); + return -1; + } + + state.error = false; + state.dest_db = tmp_db; + + if (tdb_traverse_read(tdb, repack_traverse, &state) == -1) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to traverse copying out\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + if (state.error) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Error during traversal\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + if (tdb_wipe_all(tdb) != 0) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to wipe database\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + state.error = false; + state.dest_db = tdb; + + if (tdb_traverse_read(tmp_db, repack_traverse, &state) == -1) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to traverse copying back\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + if (state.error) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Error during second traversal\n")); + tdb_transaction_cancel(tdb); + tdb_close(tmp_db); + return -1; + } + + tdb_close(tmp_db); + + if (tdb_transaction_commit(tdb) != 0) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, __location__ " Failed to commit\n")); + return -1; + } + + return 0; +} diff --git a/lib/tdb/include/tdb.h b/lib/tdb/include/tdb.h index c41c9941f0..94b5e366b9 100644 --- a/lib/tdb/include/tdb.h +++ b/lib/tdb/include/tdb.h @@ -152,11 +152,14 @@ int tdb_chainlock_unmark(struct tdb_context *tdb, TDB_DATA key); void tdb_setalarm_sigptr(struct tdb_context *tdb, volatile sig_atomic_t *sigptr); +/* wipe and repack */ +int tdb_wipe_all(struct tdb_context *tdb); +int tdb_repack(struct tdb_context *tdb); + /* Debug functions. Not used in production. */ void tdb_dump_all(struct tdb_context *tdb); int tdb_printfreelist(struct tdb_context *tdb); int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries); -int tdb_wipe_all(struct tdb_context *tdb); int tdb_freelist_size(struct tdb_context *tdb); extern TDB_DATA tdb_null; -- cgit From 42366bcbbdd42bb9d5821dfcc9dbe71a1eafa330 Mon Sep 17 00:00:00 2001 From: Holger Hetterich Date: Sun, 2 Nov 2008 00:12:32 +0100 Subject: Added a simple tdb integrity check to tdbtool. The command "check" runs traverse on the currently open tdb, and returns the number of entries if the integrity check is successful. --- lib/tdb/tools/tdbtool.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/tdb/tools/tdbtool.c b/lib/tdb/tools/tdbtool.c index d104ccd7c4..1ecad62a3d 100644 --- a/lib/tdb/tools/tdbtool.c +++ b/lib/tdb/tools/tdbtool.c @@ -57,6 +57,7 @@ enum commands { CMD_FIRST, CMD_NEXT, CMD_SYSTEM, + CMD_CHECK, CMD_QUIT, CMD_HELP }; @@ -87,6 +88,7 @@ COMMAND_TABLE cmd_table[] = { {"1", CMD_FIRST}, {"next", CMD_NEXT}, {"n", CMD_NEXT}, + {"check", CMD_CHECK}, {"quit", CMD_QUIT}, {"q", CMD_QUIT}, {"!", CMD_SYSTEM}, @@ -179,7 +181,8 @@ static void help(void) " delete key : delete a record by key\n" " list : print the database hash table and freelist\n" " free : print the database freelist\n" -" ! command : execute system command\n" +" check : check the integrity of an opened database\n" +" ! command : execute system command\n" " 1 | first : print the first record\n" " n | next : print the next record\n" " q | quit : terminate\n" @@ -452,6 +455,27 @@ static void next_record(TDB_CONTEXT *the_tdb, TDB_DATA *pkey) print_rec(the_tdb, *pkey, dbuf, NULL); } +static int test_fn(TDB_CONTEXT *tdb, TDB_DATA key, TDB_DATA dbuf, void *state) +{ + return 0; +} + +static void check_db(TDB_CONTEXT *the_tdb) +{ + int tdbcount=-1; + if (the_tdb) { + tdbcount = tdb_traverse(the_tdb, test_fn, NULL); + } else { + printf("Error: No database opened!\n"); + } + + if (tdbcount<0) { + printf("Integrity check for the opened database failed.\n"); + } else { + printf("Database integrity is OK and has %d records.\n", tdbcount); + } +} + static int do_command(void) { COMMAND_TABLE *ctp = cmd_table; @@ -552,6 +576,9 @@ static int do_command(void) if (bIterate) next_record(tdb, &iterate_kbuf); return 0; + case CMD_CHECK: + check_db(tdb); + return 0; case CMD_HELP: help(); return 0; -- cgit