Better behavior on cleanup

With the previous code in domains with many users and enumeration enable we
would eventually end up making thousands of individual searches for entries in
the clean-up process.

Change the code to do a full enumeration before a cleanup so we do one single
big search to update all entries and only then search for entries to purge.

This also fixes the fact that the cleanup task was running at every enumeration
instead of running every "ldap_purge_cache_timeout" seconds.

4 files changed, 44 insertions, 20 deletions

diff --git a/server/providers/ldap/ldap_common.h b/server/providers/ldap/ldap_common.h
index 9cc30f84..96b332cf 100644
--- a/server/providers/ldap/ldap_common.h
+++ b/server/providers/ldap/ldap_common.h
@@ -37,7 +37,9 @@ struct sdap_id_ctx {
     struct sdap_handle *gsh;
 
     /* enumeration loop timer */
-    struct timeval last_run;
+    struct timeval last_enum;
+    /* cleanup loop timer */
+    struct timeval last_purge;
 
     char *max_user_timestamp;
     char *max_group_timestamp;
diff --git a/server/providers/ldap/ldap_id_cleanup.c b/server/providers/ldap/ldap_id_cleanup.c
index 7267b354..60d3b28a 100644
--- a/server/providers/ldap/ldap_id_cleanup.c
+++ b/server/providers/ldap/ldap_id_cleanup.c
@@ -60,8 +60,6 @@ static void ldap_id_cleanup_timer(struct tevent_context *ev,
         return;
     }
 
-    ctx->last_run = tv;
-
     req = ldap_id_cleanup_send(ctx, ev, ctx);
     if (!req) {
         DEBUG(1, ("Failed to schedule cleanup, retrying later!\n"));
@@ -113,7 +111,7 @@ static void ldap_id_cleanup_reschedule(struct tevent_req *req)
         /* On error schedule starting from now, not the last run */
         tv = tevent_timeval_current();
     } else {
-        tv = ctx->last_run;
+        tv = ctx->last_purge;
     }
     talloc_zfree(req);
 
@@ -177,6 +175,8 @@ struct tevent_req *ldap_id_cleanup_send(TALLOC_CTX *memctx,
     }
     tevent_req_set_callback(subreq, ldap_id_cleanup_users_done, req);
 
+    ctx->last_purge = tevent_timeval_current();
+
     return req;
 }
 
diff --git a/server/providers/ldap/ldap_id_enum.c b/server/providers/ldap/ldap_id_enum.c
index 10b8d96c..916389fe 100644
--- a/server/providers/ldap/ldap_id_enum.c
+++ b/server/providers/ldap/ldap_id_enum.c
@@ -64,8 +64,6 @@ static void ldap_id_enumerate_timer(struct tevent_context *ev,
         return;
     }
 
-    ctx->last_run = tv;
-
     req = ldap_id_enumerate_send(ev, ctx);
     if (!req) {
         DEBUG(1, ("Failed to schedule enumeration, retrying later!\n"));
@@ -117,7 +115,7 @@ static void ldap_id_enumerate_reschedule(struct tevent_req *req)
         /* On error schedule starting from now, not the last run */
         tv = tevent_timeval_current();
     } else {
-        tv = ctx->last_run;
+        tv = ctx->last_enum;
     }
     talloc_zfree(req);
 
@@ -149,15 +147,19 @@ int ldap_id_enumerate_set_timer(struct sdap_id_ctx *ctx, struct timeval tv)
 struct global_enum_state {
     struct tevent_context *ev;
     struct sdap_id_ctx *ctx;
+
+    bool purge;
 };
 
 static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx);
+                                          struct sdap_id_ctx *ctx,
+                                          bool purge);
 static void ldap_id_enum_users_done(struct tevent_req *subreq);
 static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx);
+                                          struct sdap_id_ctx *ctx,
+                                          bool purge);
 static void ldap_id_enum_groups_done(struct tevent_req *subreq);
 static void ldap_id_enum_cleanup_done(struct tevent_req *subreq);
 
@@ -166,6 +168,7 @@ static struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
 {
     struct global_enum_state *state;
     struct tevent_req *req, *subreq;
+    int t;
 
     req = tevent_req_create(ctx, &state, struct global_enum_state);
     if (!req) return NULL;
@@ -173,7 +176,16 @@ static struct tevent_req *ldap_id_enumerate_send(struct tevent_context *ev,
     state->ev = ev;
     state->ctx = ctx;
 
-    subreq = enum_users_send(state, ev, ctx);
+    ctx->last_enum = tevent_timeval_current();
+
+    t = dp_opt_get_int(ctx->opts->basic, SDAP_CACHE_PURGE_TIMEOUT);
+    if ((ctx->last_purge.tv_sec + t) < ctx->last_enum.tv_sec) {
+        state->purge = true;
+    } else {
+        state->purge = false;
+    }
+
+    subreq = enum_users_send(state, ev, ctx, state->purge);
     if (!subreq) {
         talloc_zfree(req);
         return NULL;
@@ -202,7 +214,7 @@ static void ldap_id_enum_users_done(struct tevent_req *subreq)
     }
     talloc_zfree(subreq);
 
-    subreq = enum_groups_send(state, state->ev, state->ctx);
+    subreq = enum_groups_send(state, state->ev, state->ctx, state->purge);
     if (!subreq) {
         goto fail;
     }
@@ -241,12 +253,18 @@ static void ldap_id_enum_groups_done(struct tevent_req *subreq)
     }
     talloc_zfree(subreq);
 
-    subreq = ldap_id_cleanup_send(state, state->ev, state->ctx);
-    if (!subreq) {
-        goto fail;
+    if (state->purge) {
+
+        subreq = ldap_id_cleanup_send(state, state->ev, state->ctx);
+        if (!subreq) {
+            goto fail;
+        }
+        tevent_req_set_callback(subreq, ldap_id_enum_cleanup_done, req);
+
+        return;
     }
-    tevent_req_set_callback(subreq, ldap_id_enum_cleanup_done, req);
 
+    tevent_req_done(req);
     return;
 
 fail:
@@ -281,7 +299,8 @@ static void enum_users_op_done(struct tevent_req *subreq);
 
 static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx)
+                                          struct sdap_id_ctx *ctx,
+                                          bool purge)
 {
     struct tevent_req *req, *subreq;
     struct enum_users_state *state;
@@ -293,7 +312,8 @@ static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
     state->ev = ev;
     state->ctx = ctx;
 
-    if (ctx->max_user_timestamp) {
+    if (ctx->max_user_timestamp && !purge) {
+
         state->filter = talloc_asprintf(state,
                                "(&(%s=*)(objectclass=%s)(%s>=%s)(!(%s=%s)))",
                                ctx->opts->user_map[SDAP_AT_USER_NAME].name,
@@ -429,7 +449,8 @@ static void enum_groups_op_done(struct tevent_req *subreq);
 
 static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
                                           struct tevent_context *ev,
-                                          struct sdap_id_ctx *ctx)
+                                          struct sdap_id_ctx *ctx,
+                                          bool purge)
 {
     struct tevent_req *req, *subreq;
     struct enum_groups_state *state;
@@ -444,7 +465,8 @@ static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
 
     attr_name = ctx->opts->group_map[SDAP_AT_GROUP_NAME].name;
 
-    if (ctx->max_group_timestamp) {
+    if (ctx->max_group_timestamp && !purge) {
+
         state->filter = talloc_asprintf(state,
                               "(&(%s=*)(objectclass=%s)(%s>=%s)(!(%s=%s)))",
                               ctx->opts->group_map[SDAP_AT_GROUP_NAME].name,
diff --git a/server/providers/ldap/sdap_async.c b/server/providers/ldap/sdap_async.c
index 99d82979..5a90339e 100644
--- a/server/providers/ldap/sdap_async.c
+++ b/server/providers/ldap/sdap_async.c
@@ -769,7 +769,7 @@ struct tevent_req *sdap_get_generic_send(TALLOC_CTX *memctx,
     state->reply_count = 0;
     state->reply = NULL;
 
-    DEBUG(7, ("calling ldap_search_ext with [%s][%s].\n", state->filter,
+    DEBUG(6, ("calling ldap_search_ext with [%s][%s].\n", state->filter,
                                                           state->search_base));
     if (debug_level >= 7) {
         int i;