From e294c4799bf6ad8da043aadf8341391644147056 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 16 Dec 2008 18:45:28 +1100 Subject: an experimental patch for fixing ldb bloat ldb indexing can cause huge files, and huge memory usage. This experiment allows us to keep indexes in memory during a transaction, then to write the indexes to disk when the transaction completes. The result is that the db is much smaller (we have seen improvements of about 100x in file size) and memory usage during large transactions is also greatly reduced Note that this patch uses the unusual strategy of putting pointers into a ldb (and thus into a tdb). This works because the pointers are only there during a transaction, so the pointers are not exposed to any other users of the database. The pointers allow us to avoid some really bad allocation problems with tdb record allocation during the re-indexing. --- source4/lib/ldb/ldb_tdb/ldb_tdb.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'source4/lib/ldb/ldb_tdb/ldb_tdb.c') diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.c b/source4/lib/ldb/ldb_tdb/ldb_tdb.c index 9e3ad80705..ea460def65 100644 --- a/source4/lib/ldb/ldb_tdb/ldb_tdb.c +++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.c @@ -850,6 +850,8 @@ static int ltdb_start_trans(struct ldb_module *module) ltdb->in_transaction++; + ltdb_index_transaction_start(module); + return LDB_SUCCESS; } @@ -860,6 +862,10 @@ static int ltdb_end_trans(struct ldb_module *module) ltdb->in_transaction--; + if (ltdb_index_transaction_commit(module) != 0) { + return ltdb_err_map(tdb_error(ltdb->tdb)); + } + if (tdb_transaction_commit(ltdb->tdb) != 0) { return ltdb_err_map(tdb_error(ltdb->tdb)); } @@ -874,6 +880,10 @@ static int ltdb_del_trans(struct ldb_module *module) ltdb->in_transaction--; + if (ltdb_index_transaction_cancel(module) != 0) { + return ltdb_err_map(tdb_error(ltdb->tdb)); + } + if (tdb_transaction_cancel(ltdb->tdb) != 0) { return ltdb_err_map(tdb_error(ltdb->tdb)); } -- cgit