an experimental patch for fixing ldb bloat

ldb indexing can cause huge files, and huge memory usage. This
experiment allows us to keep indexes in memory during a transaction,
then to write the indexes to disk when the transaction completes. The
result is that the db is much smaller (we have seen improvements of
about 100x in file size) and memory usage during large transactions is
also greatly reduced

Note that this patch uses the unusual strategy of putting pointers
into a ldb (and thus into a tdb). This works because the pointers are
only there during a transaction, so the pointers are not exposed to
any other users of the database. The pointers allow us to avoid some
really bad allocation problems with tdb record allocation during the
re-indexing.

1 files changed, 5 insertions, 0 deletions

diff --git a/source4/lib/ldb/ldb_tdb/ldb_tdb.h b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
index c78a8172c7..7b9a76501f 100644
--- a/source4/lib/ldb/ldb_tdb/ldb_tdb.h
+++ b/source4/lib/ldb/ldb_tdb/ldb_tdb.h
@@ -33,6 +33,7 @@ struct ltdb_private {
 	int in_transaction;
 
 	bool check_base;
+	struct ltdb_idxptr *idxptr;
 };
 
 /*
@@ -57,6 +58,7 @@ struct ltdb_context {
 #define LTDB_INDEX      "@INDEX"
 #define LTDB_INDEXLIST  "@INDEXLIST"
 #define LTDB_IDX        "@IDX"
+#define LTDB_IDXPTR     "@IDXPTR"
 #define LTDB_IDXATTR    "@IDXATTR"
 #define LTDB_IDXONE     "@IDXONE"
 #define LTDB_BASEINFO   "@BASEINFO"
@@ -85,6 +87,9 @@ int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg);
 int ltdb_index_one(struct ldb_module *module, const struct ldb_message *msg, int add);
 int ltdb_reindex(struct ldb_module *module);
+int ltdb_index_transaction_start(struct ldb_module *module);
+int ltdb_index_transaction_commit(struct ldb_module *module);
+int ltdb_index_transaction_cancel(struct ldb_module *module);
 
 /* The following definitions come from lib/ldb/ldb_tdb/ldb_pack.c  */