Add initial failover support for ldap and ipa

The retun values are still not directly used with ldap libraries that still do
their own name resolution, but this patch introduces a very basic framework to
have a multiple providers in one domain use and share a single failover
service if they want to.

9 files changed, 266 insertions, 17 deletions

diff --git a/server/providers/ldap/ldap_auth.c b/server/providers/ldap/ldap_auth.c
index a9f03a76..6a80df44 100644
--- a/server/providers/ldap/ldap_auth.c
+++ b/server/providers/ldap/ldap_auth.c
@@ -414,8 +414,11 @@ struct auth_state {
     char *dn;
     enum pwexpire pw_expire_type;
     void *pw_expire_data;
+
+    struct fo_server *srv;
 };
 
+static void auth_resolve_done(struct tevent_req *subreq);
 static void auth_connect_done(struct tevent_req *subreq);
 static void auth_get_user_dn_done(struct tevent_req *subreq);
 static void auth_bind_user_done(struct tevent_req *subreq);
@@ -436,11 +439,12 @@ static struct tevent_req *auth_send(TALLOC_CTX *memctx,
     state->ctx = ctx;
     state->username = username;
     state->password = password;
+    state->srv = NULL;
 
-    subreq = sdap_connect_send(state, ev, ctx->opts, true);
+    subreq = be_resolve_server_send(state, ev, ctx->be, ctx->service->name);
     if (!subreq) goto fail;
 
-    tevent_req_set_callback(subreq, auth_connect_done, req);
+    tevent_req_set_callback(subreq, auth_resolve_done, req);
 
     return req;
 
@@ -449,6 +453,31 @@ fail:
     return NULL;
 }
 
+static void auth_resolve_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = tevent_req_callback_data(subreq,
+                                                      struct tevent_req);
+    struct auth_state *state = tevent_req_data(req,
+                                                    struct auth_state);
+    int ret;
+
+    ret = be_resolve_server_recv(subreq, &state->srv);
+    talloc_zfree(subreq);
+    if (ret) {
+        tevent_req_error(req, ret);
+        return;
+    }
+
+    subreq = sdap_connect_send(state, state->ev, state->ctx->opts,
+                               state->ctx->service->uri, true);
+    if (!subreq) {
+        tevent_req_error(req, ENOMEM);
+        return;
+    }
+
+    tevent_req_set_callback(subreq, auth_connect_done, req);
+}
+
 static void auth_connect_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
@@ -460,6 +489,11 @@ static void auth_connect_done(struct tevent_req *subreq)
     ret = sdap_connect_recv(subreq, state, &state->sh);
     talloc_zfree(subreq);
     if (ret) {
+        if (state->srv) {
+            /* mark the server as bad if connection failed */
+            fo_set_server_status(state->srv, SERVER_NOT_WORKING);
+        }
+
         tevent_req_error(req, ret);
         return;
     }
diff --git a/server/providers/ldap/ldap_common.c b/server/providers/ldap/ldap_common.c
index 6b619f90..6236707f 100644
--- a/server/providers/ldap/ldap_common.c
+++ b/server/providers/ldap/ldap_common.c
@@ -23,6 +23,7 @@
 */
 
 #include "providers/ldap/ldap_common.h"
+#include "providers/fail_over.h"
 
 struct dp_option default_basic_opts[] = {
     { "ldap_uri", DP_OPT_STRING, { "ldap://localhost" }, NULL_STRING },
@@ -309,3 +310,110 @@ int sdap_id_setup_tasks(struct sdap_id_ctx *ctx)
 
     return ret;
 }
+
+static void sdap_uri_callback(void *private_data, struct fo_server *server)
+{
+    struct sdap_service *service;
+    const char *tmp;
+    char *new_uri;
+
+    service = talloc_get_type(private_data, struct sdap_service);
+    if (!service) return;
+
+    tmp = (const char *)fo_get_server_user_data(server);
+    if (tmp && ldap_is_ldap_url(tmp)) {
+        new_uri = talloc_strdup(service, tmp);
+    } else {
+        new_uri = talloc_asprintf(service, "ldap://%s",
+                                  fo_get_server_name(server));
+    }
+    if (!new_uri) {
+        DEBUG(2, ("Failed to copy URI ...\n"));
+        return;
+    }
+
+    /* free old one and replace with new one */
+    talloc_zfree(service->uri);
+    service->uri = new_uri;
+}
+
+int sdap_service_init(TALLOC_CTX *memctx, struct be_ctx *ctx,
+                      const char *service_name, const char *urls,
+                      struct sdap_service **_service)
+{
+    TALLOC_CTX *tmp_ctx;
+    struct sdap_service *service;
+    LDAPURLDesc *lud;
+    char **list = NULL;
+    int count = 0;
+    int ret;
+    int i;
+
+    tmp_ctx = talloc_new(memctx);
+    if (!tmp_ctx) {
+        return ENOMEM;
+    }
+
+    service = talloc_zero(tmp_ctx, struct sdap_service);
+    if (!service) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    ret = be_fo_add_service(ctx, service_name);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to create failover service!\n"));
+        goto done;
+    }
+
+    service->name = talloc_strdup(service, service_name);
+    if (!service->name) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    /* split server parm into a list */
+    ret = sss_split_list(tmp_ctx, urls, ", ", &list, &count);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to parse server list!\n"));
+        goto done;
+    }
+
+    /* now for each URI add a new server to the failover service */
+    for (i = 0; i < count; i++) {
+        ret = ldap_url_parse(list[i], &lud);
+        if (ret != LDAP_SUCCESS) {
+            DEBUG(0, ("Failed to parse ldap URI (%s)!\n", list[i]));
+            ret = EINVAL;
+            goto done;
+        }
+
+        DEBUG(6, ("Added URI %s\n", list[i]));
+
+        talloc_steal(service, list[i]);
+
+        ret = be_fo_add_server(ctx, service->name,
+                               lud->lud_host, lud->lud_port, list[i]);
+        if (ret) {
+            goto done;
+        }
+        ldap_free_urldesc(lud);
+    }
+
+    ret = be_fo_service_add_callback(memctx, ctx, service->name,
+                                     sdap_uri_callback, service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to add failover callback!\n"));
+        goto done;
+    }
+
+    ret = EOK;
+
+done:
+    if (ret == EOK) {
+        *_service = talloc_steal(memctx, service);
+    }
+    talloc_zfree(tmp_ctx);
+    return ret;
+}
+
diff --git a/server/providers/ldap/ldap_common.h b/server/providers/ldap/ldap_common.h
index 96b332cf..6adb785a 100644
--- a/server/providers/ldap/ldap_common.h
+++ b/server/providers/ldap/ldap_common.h
@@ -24,11 +24,13 @@
 
 #include "providers/dp_backend.h"
 #include "providers/ldap/sdap.h"
+#include "providers/fail_over.h"
 
 struct sdap_id_ctx {
     struct be_ctx *be;
-
     struct sdap_options *opts;
+    struct fo_service *fo_service;
+    struct sdap_service *service;
 
     /* what rootDSE returns */
     struct sysdb_attrs *rootDSE;
@@ -48,6 +50,8 @@ struct sdap_id_ctx {
 struct sdap_auth_ctx {
     struct be_ctx *be;
     struct sdap_options *opts;
+    struct fo_service *fo_service;
+    struct sdap_service *service;
 };
 
 /* id */
@@ -65,6 +69,10 @@ void sdap_pam_chpass_handler(struct be_req *breq);
 void sdap_handler_done(struct be_req *req, int dp_err,
                        int error, const char *errstr);
 
+int sdap_service_init(TALLOC_CTX *mmectx, struct be_ctx *ctx,
+                      const char *service_name, const char *urls,
+                      struct sdap_service **service);
+
 /* options parser */
 int ldap_get_options(TALLOC_CTX *memctx,
                      struct confdb_ctx *cdb,
diff --git a/server/providers/ldap/ldap_id.c b/server/providers/ldap/ldap_id.c
index 52391c28..f99ea7b9 100644
--- a/server/providers/ldap/ldap_id.c
+++ b/server/providers/ldap/ldap_id.c
@@ -104,7 +104,9 @@ struct tevent_req *users_get_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
@@ -325,7 +327,9 @@ struct tevent_req *groups_get_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
@@ -511,7 +515,9 @@ static struct tevent_req *groups_by_user_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
diff --git a/server/providers/ldap/ldap_id_enum.c b/server/providers/ldap/ldap_id_enum.c
index 916389fe..1ddcbf8f 100644
--- a/server/providers/ldap/ldap_id_enum.c
+++ b/server/providers/ldap/ldap_id_enum.c
@@ -345,7 +345,9 @@ static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
@@ -498,7 +500,9 @@ static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
diff --git a/server/providers/ldap/ldap_init.c b/server/providers/ldap/ldap_init.c
index 295ff19d..5a64585d 100644
--- a/server/providers/ldap/ldap_init.c
+++ b/server/providers/ldap/ldap_init.c
@@ -49,6 +49,7 @@ int sssm_ldap_init(struct be_ctx *bectx,
                    void **pvt_data)
 {
     struct sdap_id_ctx *ctx;
+    const char *urls;
     int ret;
 
     ctx = talloc_zero(bectx, struct sdap_id_ctx);
@@ -62,6 +63,19 @@ int sssm_ldap_init(struct be_ctx *bectx,
         goto done;
     }
 
+    urls = dp_opt_get_string(ctx->opts->basic, SDAP_URI);
+    if (!urls) {
+        DEBUG(0, ("Missing ldap_uri\n"));
+        ret = EINVAL;
+        goto done;
+    }
+
+    ret = sdap_service_init(ctx, ctx->be, "LDAP", urls, &ctx->service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to initialize failover service!\n"));
+        goto done;
+    }
+
     ret = setup_tls_config(ctx->opts->basic);
     if (ret != EOK) {
         DEBUG(1, ("setup_tls_config failed [%d][%s].\n",
@@ -90,6 +104,7 @@ int sssm_ldap_auth_init(struct be_ctx *bectx,
                         void **pvt_data)
 {
     struct sdap_auth_ctx *ctx;
+    const char *urls;
     int ret;
 
     ctx = talloc(bectx, struct sdap_auth_ctx);
@@ -103,6 +118,19 @@ int sssm_ldap_auth_init(struct be_ctx *bectx,
         goto done;
     }
 
+    urls = dp_opt_get_string(ctx->opts->basic, SDAP_URI);
+    if (!urls) {
+        DEBUG(0, ("Missing ldap_uri\n"));
+        ret = EINVAL;
+        goto done;
+    }
+
+    ret = sdap_service_init(ctx, ctx->be, "LDAP", urls, &ctx->service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to initialize failover service!\n"));
+        goto done;
+    }
+
     ret = setup_tls_config(ctx->opts->basic);
     if (ret != EOK) {
         DEBUG(1, ("setup_tls_config failed [%d][%s].\n",
diff --git a/server/providers/ldap/sdap.h b/server/providers/ldap/sdap.h
index 8330bd6f..a63d8580 100644
--- a/server/providers/ldap/sdap.h
+++ b/server/providers/ldap/sdap.h
@@ -62,6 +62,11 @@ struct sdap_handle {
     struct sdap_op *ops;
 };
 
+struct sdap_service {
+    char *name;
+    char *uri;
+};
+
 #define SYSDB_SHADOWPW_LASTCHANGE "shadowLastChange"
 #define SYSDB_SHADOWPW_MIN "shadowMin"
 #define SYSDB_SHADOWPW_MAX "shadowMax"
diff --git a/server/providers/ldap/sdap_async.h b/server/providers/ldap/sdap_async.h
index 383a2fce..d8550794 100644
--- a/server/providers/ldap/sdap_async.h
+++ b/server/providers/ldap/sdap_async.h
@@ -26,10 +26,12 @@
 #include <tevent.h>
 #include "providers/dp_backend.h"
 #include "providers/ldap/sdap.h"
+#include "providers/fail_over.h"
 
 struct tevent_req *sdap_connect_send(TALLOC_CTX *memctx,
                                      struct tevent_context *ev,
                                      struct sdap_options *opts,
+                                     const char *uri,
                                      bool use_start_tls);
 int sdap_connect_recv(struct tevent_req *req,
                       TALLOC_CTX *memctx,
@@ -91,11 +93,14 @@ struct tevent_req *sdap_exop_modify_passwd_send(TALLOC_CTX *memctx,
                                                 char *user_dn,
                                                 char *password,
                                                 char *new_password);
-int sdap_exop_modify_passwd_recv(struct tevent_req *req, enum sdap_result *result);
+int sdap_exop_modify_passwd_recv(struct tevent_req *req,
+                                 enum sdap_result *result);
 
 struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
                                          struct tevent_context *ev,
                                          struct sdap_options *opts,
+                                         struct be_ctx *be,
+                                         struct sdap_service *service,
                                          struct sysdb_attrs **rootdse);
 int sdap_cli_connect_recv(struct tevent_req *req,
                           TALLOC_CTX *memctx,
diff --git a/server/providers/ldap/sdap_async_connection.c b/server/providers/ldap/sdap_async_connection.c
index 9f53ad6f..a5e6ccfa 100644
--- a/server/providers/ldap/sdap_async_connection.c
+++ b/server/providers/ldap/sdap_async_connection.c
@@ -46,6 +46,7 @@ static void sdap_connect_done(struct sdap_op *op,
 struct tevent_req *sdap_connect_send(TALLOC_CTX *memctx,
                                      struct tevent_context *ev,
                                      struct sdap_options *opts,
+                                     const char *uri,
                                      bool use_start_tls)
 {
     struct tevent_req *req;
@@ -73,10 +74,9 @@ struct tevent_req *sdap_connect_send(TALLOC_CTX *memctx,
         return NULL;
     }
     /* Initialize LDAP handler */
-    lret = ldap_initialize(&state->sh->ldap,
-                           dp_opt_get_string(opts->basic, SDAP_URI));
+    lret = ldap_initialize(&state->sh->ldap, uri);
     if (lret != LDAP_SUCCESS) {
-        DEBUG(1, ("ldap_initialize failed: %s\n", ldap_err2string(ret)));
+        DEBUG(1, ("ldap_initialize failed: %s\n", ldap_err2string(lret)));
         goto fail;
     }
 
@@ -871,12 +871,17 @@ int sdap_auth_recv(struct tevent_req *req, enum sdap_result *result)
 struct sdap_cli_connect_state {
     struct tevent_context *ev;
     struct sdap_options *opts;
+    struct sdap_service *service;
 
-    struct sysdb_attrs *rootdse;
     bool use_rootdse;
+    struct sysdb_attrs *rootdse;
+
     struct sdap_handle *sh;
+
+    struct fo_server *srv;
 };
 
+static void sdap_cli_resolve_done(struct tevent_req *subreq);
 static void sdap_cli_connect_done(struct tevent_req *subreq);
 static void sdap_cli_rootdse_step(struct tevent_req *req);
 static void sdap_cli_rootdse_done(struct tevent_req *subreq);
@@ -888,6 +893,8 @@ static void sdap_cli_auth_done(struct tevent_req *subreq);
 struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
                                          struct tevent_context *ev,
                                          struct sdap_options *opts,
+                                         struct be_ctx *be,
+                                         struct sdap_service *service,
                                          struct sysdb_attrs **rootdse)
 {
     struct tevent_req *req, *subreq;
@@ -898,6 +905,9 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
 
     state->ev = ev;
     state->opts = opts;
+    state->service = service;
+    state->srv = NULL;
+
     if (rootdse) {
         state->use_rootdse = true;
         state->rootdse = *rootdse;
@@ -906,17 +916,46 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
         state->rootdse = NULL;
     }
 
-    subreq = sdap_connect_send(state, ev, opts,
-                               dp_opt_get_bool(opts->basic, SDAP_ID_TLS));
+    /* NOTE: this call may cause service->uri to be refreshed
+     * with a new valid server. Do not use service->uri before */
+    subreq = be_resolve_server_send(state, ev, be, service->name);
     if (!subreq) {
         talloc_zfree(req);
         return NULL;
     }
-    tevent_req_set_callback(subreq, sdap_cli_connect_done, req);
+    tevent_req_set_callback(subreq, sdap_cli_resolve_done, req);
 
     return req;
 }
 
+static void sdap_cli_resolve_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = tevent_req_callback_data(subreq,
+                                                      struct tevent_req);
+    struct sdap_cli_connect_state *state = tevent_req_data(req,
+                                             struct sdap_cli_connect_state);
+    int ret;
+
+    ret = be_resolve_server_recv(subreq, &state->srv);
+    talloc_zfree(subreq);
+    if (ret) {
+        /* all servers have been tried and none
+         * was found good, go offline */
+        tevent_req_error(req, EIO);
+        return;
+    }
+
+    subreq = sdap_connect_send(state, state->ev, state->opts,
+                               state->service->uri,
+                               dp_opt_get_bool(state->opts->basic,
+                                                            SDAP_ID_TLS));
+    if (!subreq) {
+        tevent_req_error(req, ENOMEM);
+        return;
+    }
+    tevent_req_set_callback(subreq, sdap_cli_connect_done, req);
+}
+
 static void sdap_cli_connect_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
@@ -1118,8 +1157,20 @@ int sdap_cli_connect_recv(struct tevent_req *req,
 {
     struct sdap_cli_connect_state *state = tevent_req_data(req,
                                              struct sdap_cli_connect_state);
+    enum tevent_req_state tstate;
+    uint64_t err;
 
-    TEVENT_REQ_RETURN_ON_ERROR(req);
+    if (tevent_req_is_error(req, &tstate, &err)) {
+        /* mark the server as bad if connection failed */
+        if (state->srv) {
+            fo_set_server_status(state->srv, SERVER_NOT_WORKING);
+        }
+
+        if (tstate == TEVENT_REQ_USER_ERROR) {
+            return err;
+        }
+        return EIO;
+    }
 
     if (gsh) {
         *gsh = talloc_steal(memctx, state->sh);