Add initial failover support for ldap and ipa

The retun values are still not directly used with ldap libraries that still do
their own name resolution, but this patch introduces a very basic framework to
have a multiple providers in one domain use and share a single failover
service if they want to.

22 files changed, 938 insertions, 69 deletions

diff --git a/server/Makefile.am b/server/Makefile.am
index 4259f6cf..c40942a7 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -183,6 +183,7 @@ SSSD_UTIL_OBJ = \
     sbus/sssd_dbus_common.c \
     sbus/sssd_dbus_connection.c \
     sbus/sssd_dbus_server.c \
+    util/util.c \
     util/memory.c \
     util/server.c \
     util/signal.c \
@@ -341,9 +342,11 @@ sssd_pam_LDADD = \
 
 sssd_be_SOURCES = \
     providers/data_provider_be.c \
+    providers/data_provider_fo.c \
     providers/data_provider_opts.c \
+    $(SSSD_FAILOVER_OBJ) \
     $(SSSD_UTIL_OBJ)
-sssd_be_LDADD = $(SSSD_LIBS)
+sssd_be_LDADD = $(SSSD_LIBS) $(CARES_LIBS)
 sssd_be_LDFLAGS = \
     -Wl,--version-script,$(srcdir)/providers/sssd_be.exports \
     -export-dynamic
diff --git a/server/providers/data_provider_be.c b/server/providers/data_provider_be.c
index d5c2492b..b5d24600 100644
--- a/server/providers/data_provider_be.c
+++ b/server/providers/data_provider_be.c
@@ -41,6 +41,8 @@
 #include "dbus/dbus.h"
 #include "sbus/sssd_dbus.h"
 #include "providers/dp_backend.h"
+#include "providers/fail_over.h"
+#include "resolv/async_resolv.h"
 #include "monitor/monitor_interfaces.h"
 
 #define MSG_TARGET_NO_CONFIGURED "sssd_be: The requested target is not configured"
@@ -138,7 +140,6 @@ static int be_file_request(struct be_ctx *ctx,
     return EOK;
 }
 
-
 bool be_is_offline(struct be_ctx *ctx)
 {
     time_t now = time(NULL);
@@ -991,6 +992,12 @@ int be_process_init(TALLOC_CTX *mem_ctx,
         return ENOMEM;
     }
 
+    ret = be_init_failover(ctx);
+    if (ret != EOK) {
+        DEBUG(0, ("fatal error initializing failover context\n"));
+        return ret;
+    }
+
     ret = confdb_get_domain(cdb, be_domain, &ctx->domain);
     if (ret != EOK) {
         DEBUG(0, ("fatal error retrieving domain configuration\n"));
diff --git a/server/providers/data_provider_fo.c b/server/providers/data_provider_fo.c
new file mode 100644
index 00000000..ebf92a43
--- /dev/null
+++ b/server/providers/data_provider_fo.c
@@ -0,0 +1,358 @@
+/*
+    SSSD
+
+    Data Provider Helpers
+
+    Copyright (C) Simo Sorce <ssorce@redhat.com> 2009
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <netdb.h>
+#include <arpa/inet.h>
+#include "providers/dp_backend.h"
+#include "resolv/async_resolv.h"
+
+struct be_svc_callback {
+    struct be_svc_callback *prev;
+    struct be_svc_callback *next;
+
+    struct be_svc_data *svc;
+
+    be_svc_callback_fn_t *fn;
+    void *private_data;
+};
+
+struct be_svc_data {
+    struct be_svc_data *prev;
+    struct be_svc_data *next;
+
+    const char *name;
+    struct fo_service *fo_service;
+
+    struct hostent *last_good_srvaddr;
+
+    struct be_svc_callback *callbacks;
+};
+
+struct be_failover_ctx {
+    struct fo_ctx *fo_ctx;
+    struct resolv_ctx *resolv;
+
+    struct be_svc_data *svcs;
+};
+
+int be_init_failover(struct be_ctx *ctx)
+{
+    int ret;
+
+    if (ctx->be_fo != NULL) {
+        return EOK;
+    }
+
+    ctx->be_fo = talloc_zero(ctx, struct be_failover_ctx);
+    if (!ctx->be_fo) {
+        return ENOMEM;
+    }
+
+    ret = resolv_init(ctx, ctx->ev, &ctx->be_fo->resolv);
+    if (ret != EOK) {
+        talloc_zfree(ctx->be_fo);
+        return ret;
+    }
+
+    /* todo get timeout from configuration */
+    ctx->be_fo->fo_ctx = fo_context_init(ctx->be_fo, 30);
+    if (!ctx->be_fo->fo_ctx) {
+        talloc_zfree(ctx->be_fo);
+        return ENOMEM;
+    }
+
+    return EOK;
+}
+
+static int be_svc_data_destroy(void *memptr)
+{
+    struct be_svc_data *svc;
+
+    svc = talloc_get_type(memptr, struct be_svc_data);
+
+    while (svc->callbacks) {
+        /* callbacks removes themselves from the list,
+         * so this while will freem them all and then terminate */
+        talloc_free(svc->callbacks);
+    }
+
+    return 0;
+}
+
+int be_fo_add_service(struct be_ctx *ctx, const char *service_name)
+{
+    struct fo_service *service;
+    struct be_svc_data *svc;
+    int ret;
+
+    DLIST_FOR_EACH(svc, ctx->be_fo->svcs) {
+        if (strcmp(svc->name, service_name) == 0) {
+            DEBUG(6, ("Failover service already initialized!\n"));
+            /* we already have a service up and configured,
+             * can happen when using both id and auth provider
+             */
+            return EOK;
+        }
+    }
+
+    /* if not in the be service list, try to create new one */
+
+    ret = fo_new_service(ctx->be_fo->fo_ctx, service_name, &service);
+    if (ret != EOK && ret != EEXIST) {
+        DEBUG(1, ("Failed to create failover service!\n"));
+        return ret;
+    }
+
+    svc = talloc_zero(ctx->be_fo, struct be_svc_data);
+    if (!svc) {
+        return ENOMEM;
+    }
+    talloc_set_destructor((TALLOC_CTX *)svc, be_svc_data_destroy);
+
+    svc->name = talloc_strdup(svc, service_name);
+    if (!svc->name) {
+        talloc_zfree(svc);
+        return ENOMEM;
+    }
+    svc->fo_service = service;
+
+    DLIST_ADD(ctx->be_fo->svcs, svc);
+
+    return EOK;
+}
+
+static int be_svc_callback_destroy(void *memptr)
+{
+    struct be_svc_callback *callback;
+
+    callback = talloc_get_type(memptr, struct be_svc_callback);
+
+    if (callback->svc) {
+        DLIST_REMOVE(callback->svc->callbacks, callback);
+    }
+
+    return 0;
+}
+
+int be_fo_service_add_callback(TALLOC_CTX *memctx,
+                               struct be_ctx *ctx, const char *service_name,
+                               be_svc_callback_fn_t *fn, void *private_data)
+{
+    struct be_svc_callback *callback;
+    struct be_svc_data *svc;
+
+    DLIST_FOR_EACH(svc, ctx->be_fo->svcs) {
+        if (strcmp(svc->name, service_name) == 0) {
+            break;
+        }
+    }
+    if (NULL == svc) {
+        return ENOENT;
+    }
+
+    callback = talloc_zero(memctx, struct be_svc_callback);
+    if (!callback) {
+        return ENOMEM;
+    }
+    talloc_set_destructor((TALLOC_CTX *)callback, be_svc_callback_destroy);
+
+    callback->fn = fn;
+    callback->private_data = private_data;
+
+    DLIST_ADD(svc->callbacks, callback);
+
+    return EOK;
+}
+
+int be_fo_add_server(struct be_ctx *ctx, const char *service_name,
+                     const char *server, int port, void *user_data)
+{
+    struct be_svc_data *svc;
+    int ret;
+
+    DLIST_FOR_EACH(svc, ctx->be_fo->svcs) {
+        if (strcmp(svc->name, service_name) == 0) {
+            break;
+        }
+    }
+    if (NULL == svc) {
+        return ENOENT;
+    }
+
+    ret = fo_add_server(svc->fo_service, server, port, user_data);
+    if (ret && ret != EEXIST) {
+        DEBUG(1, ("Failed to add server to failover service\n"));
+        return ret;
+    }
+
+    return EOK;
+}
+
+struct be_resolve_server_state {
+    struct tevent_context *ev;
+    struct be_ctx *ctx;
+
+    struct be_svc_data *svc;
+    int attempts;
+
+    struct fo_server *srv;
+};
+
+static void be_resolve_server_done(struct tevent_req *subreq);
+
+struct tevent_req *be_resolve_server_send(TALLOC_CTX *memctx,
+                                          struct tevent_context *ev,
+                                          struct be_ctx *ctx,
+                                          const char *service_name)
+{
+    struct tevent_req *req, *subreq;
+    struct be_resolve_server_state *state;
+    struct be_svc_data *svc;
+
+    req = tevent_req_create(memctx, &state, struct be_resolve_server_state);
+    if (!req) return NULL;
+
+    state->ev = ev;
+    state->ctx = ctx;
+
+    DLIST_FOR_EACH(svc, ctx->be_fo->svcs) {
+        if (strcmp(svc->name, service_name) == 0) {
+            state->svc = svc;
+            break;
+        }
+    }
+
+    if (NULL == svc) {
+        tevent_req_error(req, EINVAL);
+        tevent_req_post(req, ev);
+        return req;
+    }
+
+    state->attempts = 0;
+
+    subreq = fo_resolve_service_send(state, ev,
+                                     ctx->be_fo->resolv, svc->fo_service);
+    if (!subreq) {
+        talloc_zfree(req);
+        return NULL;
+    }
+    tevent_req_set_callback(subreq, be_resolve_server_done, req);
+
+    return req;
+}
+
+static void be_resolve_server_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = tevent_req_callback_data(subreq,
+                                                      struct tevent_req);
+    struct be_resolve_server_state *state = tevent_req_data(req,
+                                             struct be_resolve_server_state);
+    struct be_svc_callback *callback;
+    struct hostent *srvaddr;
+    int ret;
+
+    ret = fo_resolve_service_recv(subreq, &state->srv);
+    talloc_zfree(subreq);
+    switch (ret) {
+    case EOK:
+        if (!state->srv) {
+            tevent_req_error(req, EFAULT);
+            return;
+        }
+        break;
+
+    case ENOENT:
+        /* all servers have been tried and none
+         * was found good, go offline */
+        tevent_req_error(req, EIO);
+        return;
+
+    default:
+        /* mark server as bad and retry */
+        if (!state->srv) {
+            tevent_req_error(req, EFAULT);
+            return;
+        }
+        DEBUG(6, ("Couldn't resolve server (%s), resolver returned (%d)\n",
+                  fo_get_server_name(state->srv), ret));
+
+        /* mark as bad server */
+        fo_set_server_status(state->srv, SERVER_NOT_WORKING);
+
+        state->attempts++;
+        if (state->attempts >= 10) {
+            DEBUG(2, ("Failed to find a server after 10 attempts\n"));
+            tevent_req_error(req, EIO);
+            return;
+        }
+
+        /* now try next one */
+        DEBUG(6, ("Trying with the next one!\n"));
+        subreq = fo_resolve_service_send(state, state->ev,
+                                         state->ctx->be_fo->resolv,
+                                         state->svc->fo_service);
+        if (!subreq) {
+            tevent_req_error(req, ENOMEM);
+            return;
+        }
+        tevent_req_set_callback(subreq, be_resolve_server_done, req);
+
+        return;
+    }
+
+    /* all fine we got the server */
+    srvaddr = fo_get_server_hostent(state->srv);
+
+    if (debug_level >= 4) {
+        char ipaddr[128];
+        inet_ntop(srvaddr->h_addrtype, srvaddr->h_addr_list[0],
+                  ipaddr, 128);
+
+        DEBUG(4, ("Found address for server %s: [%s]\n",
+                  fo_get_server_name(state->srv), ipaddr));
+    }
+
+    /* now call all svc callbacks if server changed */
+    if (srvaddr != state->svc->last_good_srvaddr) {
+        state->svc->last_good_srvaddr = srvaddr;
+
+        DLIST_FOR_EACH(callback, state->svc->callbacks) {
+            callback->fn(callback->private_data, state->srv);
+        }
+    }
+
+    tevent_req_done(req);
+}
+
+int be_resolve_server_recv(struct tevent_req *req, struct fo_server **srv)
+{
+    struct be_resolve_server_state *state = tevent_req_data(req,
+                                             struct be_resolve_server_state);
+
+    TEVENT_REQ_RETURN_ON_ERROR(req);
+
+    if (srv) {
+        *srv = state->srv;
+    }
+
+    return EOK;
+}
+
diff --git a/server/providers/dp_backend.h b/server/providers/dp_backend.h
index 1a8c6c46..f1069d0d 100644
--- a/server/providers/dp_backend.h
+++ b/server/providers/dp_backend.h
@@ -23,6 +23,7 @@
 #define __DP_BACKEND_H__
 
 #include "providers/data_provider.h"
+#include "providers/fail_over.h"
 #include "db/sysdb.h"
 
 struct be_ctx;
@@ -73,6 +74,8 @@ struct be_client {
     bool initialized;
 };
 
+struct be_failover_ctx;
+
 struct be_ctx {
     struct tevent_context *ev;
     struct confdb_ctx *cdb;
@@ -80,6 +83,7 @@ struct be_ctx {
     struct sss_domain_info *domain;
     const char *identity;
     const char *conf_path;
+    struct be_failover_ctx *be_fo;
 
     struct be_offline_status offstat;
 
@@ -118,4 +122,21 @@ struct be_acct_req {
 bool be_is_offline(struct be_ctx *ctx);
 void be_mark_offline(struct be_ctx *ctx);
 
+/* from data_provider_fo.c */
+typedef void (be_svc_callback_fn_t)(void *, struct fo_server *);
+
+int be_init_failover(struct be_ctx *ctx);
+int be_fo_add_service(struct be_ctx *ctx, const char *service_name);
+int be_fo_service_add_callback(TALLOC_CTX *memctx,
+                               struct be_ctx *ctx, const char *service_name,
+                               be_svc_callback_fn_t *fn, void *private_data);
+int be_fo_add_server(struct be_ctx *ctx, const char *service_name,
+                     const char *server, int port, void *user_data);
+
+struct tevent_req *be_resolve_server_send(TALLOC_CTX *memctx,
+                                          struct tevent_context *ev,
+                                          struct be_ctx *ctx,
+                                          const char *service_name);
+int be_resolve_server_recv(struct tevent_req *req, struct fo_server **srv);
+
 #endif /* __DP_BACKEND_H___ */
diff --git a/server/providers/ipa/ipa_access.c b/server/providers/ipa/ipa_access.c
index 230cabc1..675e7c20 100644
--- a/server/providers/ipa/ipa_access.c
+++ b/server/providers/ipa/ipa_access.c
@@ -386,7 +386,8 @@ static struct tevent_req *hbac_get_host_info_send(TALLOC_CTX *memctx,
             talloc_zfree(sdap_ctx->gsh);
         }
 
-        subreq = sdap_cli_connect_send(state, ev, sdap_ctx->opts, NULL);
+        subreq = sdap_cli_connect_send(state, ev, sdap_ctx->opts,
+                                       sdap_ctx->be, sdap_ctx->service, NULL);
         if (!subreq) {
             DEBUG(1, ("sdap_cli_connect_send failed.\n"));
             ret = ENOMEM;
@@ -850,7 +851,8 @@ static struct tevent_req *hbac_get_rules_send(TALLOC_CTX *memctx,
             talloc_zfree(sdap_ctx->gsh);
         }
 
-        subreq = sdap_cli_connect_send(state, ev, sdap_ctx->opts, NULL);
+        subreq = sdap_cli_connect_send(state, ev, sdap_ctx->opts,
+                                       sdap_ctx->be, sdap_ctx->service, NULL);
         if (!subreq) {
             DEBUG(1, ("sdap_cli_connect_send failed.\n"));
             ret = ENOMEM;
diff --git a/server/providers/ipa/ipa_common.c b/server/providers/ipa/ipa_common.c
index 38e4d53d..2bd9c76d 100644
--- a/server/providers/ipa/ipa_common.c
+++ b/server/providers/ipa/ipa_common.c
@@ -22,6 +22,7 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#include <netdb.h>
 #include <ctype.h>
 #include "providers/ipa/ipa_common.h"
 
@@ -251,24 +252,6 @@ int ipa_get_id_options(struct ipa_options *ipa_opts,
         goto done;
     }
 
-    /* set ldap_uri */
-    if (NULL == dp_opt_get_string(ipa_opts->id->basic, SDAP_URI)) {
-        value = talloc_asprintf(tmpctx, "ldap://%s",
-                                dp_opt_get_string(ipa_opts->basic,
-                                                  IPA_SERVER));
-        if (!value) {
-            ret = ENOMEM;
-            goto done;
-        }
-        ret = dp_opt_set_string(ipa_opts->id->basic, SDAP_URI, value);
-        if (ret != EOK) {
-            goto done;
-        }
-        DEBUG(6, ("Option %s set to %s\n",
-                  ipa_opts->id->basic[SDAP_URI].opt_name,
-                  dp_opt_get_string(ipa_opts->id->basic, SDAP_URI)));
-    }
-
     if (NULL == dp_opt_get_string(ipa_opts->id->basic, SDAP_SEARCH_BASE)) {
         ret = domain_to_basedn(tmpctx,
                                dp_opt_get_string(ipa_opts->basic, IPA_DOMAIN),
@@ -429,22 +412,6 @@ int ipa_get_auth_options(struct ipa_options *ipa_opts,
         goto done;
     }
 
-    /* set KDC */
-    if (NULL == dp_opt_get_string(ipa_opts->auth, KRB5_KDC)) {
-        value = dp_opt_get_string(ipa_opts->basic, IPA_SERVER);
-        if (!value) {
-            ret = ENOMEM;
-            goto done;
-        }
-        ret = dp_opt_set_string(ipa_opts->auth, KRB5_KDC, value);
-        if (ret != EOK) {
-            goto done;
-        }
-        DEBUG(6, ("Option %s set to %s\n",
-                  ipa_opts->auth[KRB5_KDC].opt_name,
-                  dp_opt_get_string(ipa_opts->auth, KRB5_KDC)));
-    }
-
     /* set krb realm */
     if (NULL == dp_opt_get_string(ipa_opts->auth, KRB5_REALM)) {
         value = dp_opt_get_string(ipa_opts->basic, IPA_DOMAIN);
@@ -473,3 +440,133 @@ done:
     }
     return ret;
 }
+
+static void ipa_resolve_callback(void *private_data, struct fo_server *server)
+{
+    struct ipa_service *service;
+    struct hostent *srvaddr;
+    char *address;
+    char *new_uri;
+    int ret;
+
+    service = talloc_get_type(private_data, struct ipa_service);
+    if (!service) {
+        DEBUG(1, ("FATAL: Bad private_data\n"));
+        return;
+    }
+
+    srvaddr = fo_get_server_hostent(server);
+    if (!srvaddr) {
+        DEBUG(1, ("FATAL: No hostent available for server (%s)\n",
+                  fo_get_server_name(server)));
+        return;
+    }
+
+    address = talloc_asprintf(service, srvaddr->h_name);
+    if (!address) {
+        DEBUG(1, ("Failed to copy address ...\n"));
+        return;
+    }
+
+    new_uri = talloc_asprintf(service, "ldap://%s", address);
+    if (!new_uri) {
+        DEBUG(2, ("Failed to copy URI ...\n"));
+        talloc_free(address);
+        return;
+    }
+
+    /* free old one and replace with new one */
+    talloc_zfree(service->sdap->uri);
+    service->sdap->uri = new_uri;
+    talloc_zfree(service->krb_server->address);
+    service->krb_server->address = address;
+
+    /* set also env variable */
+    ret = setenv(SSSD_KRB5_KDC, address, 1);
+    if (ret != EOK) {
+        DEBUG(2, ("setenv %s failed, authentication might fail.\n",
+                  SSSD_KRB5_KDC));
+    }
+}
+
+int ipa_service_init(TALLOC_CTX *memctx, struct be_ctx *ctx,
+                     const char *servers, struct ipa_service **_service)
+{
+    TALLOC_CTX *tmp_ctx;
+    struct ipa_service *service;
+    char **list = NULL;
+    int count = 0;
+    int ret;
+    int i;
+
+    tmp_ctx = talloc_new(memctx);
+    if (!tmp_ctx) {
+        return ENOMEM;
+    }
+
+    service = talloc_zero(tmp_ctx, struct ipa_service);
+    if (!service) {
+        ret = ENOMEM;
+        goto done;
+    }
+    service->sdap = talloc_zero(service, struct sdap_service);
+    if (!service->sdap) {
+        ret = ENOMEM;
+        goto done;
+    }
+    service->krb_server = talloc_zero(service, struct krb_server);
+    if (!service->krb_server) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    ret = be_fo_add_service(ctx, "IPA");
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to create failover service!\n"));
+        goto done;
+    }
+
+    service->sdap->name = talloc_strdup(service, "IPA");
+    if (!service->sdap->name) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    /* split server parm into a list */
+    ret = sss_split_list(tmp_ctx, servers, ", ", &list, &count);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to parse server list!\n"));
+        goto done;
+    }
+
+    /* now for each one add a new server to the failover service */
+    for (i = 0; i < count; i++) {
+
+        talloc_steal(service, list[i]);
+
+        ret = be_fo_add_server(ctx, "IPA", list[i], 0, NULL);
+        if (ret && ret != EEXIST) {
+            DEBUG(0, ("Failed to add server\n"));
+            goto done;
+        }
+
+        DEBUG(6, ("Added Server %s\n", list[i]));
+    }
+
+    ret = be_fo_service_add_callback(memctx, ctx, "IPA",
+                                     ipa_resolve_callback, service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to add failover callback!\n"));
+        goto done;
+    }
+
+    ret = EOK;
+
+done:
+    if (ret == EOK) {
+        *_service = talloc_steal(memctx, service);
+    }
+    talloc_zfree(tmp_ctx);
+    return ret;
+}
+
diff --git a/server/providers/ipa/ipa_common.h b/server/providers/ipa/ipa_common.h
index 21e6e1a3..8d0840c5 100644
--- a/server/providers/ipa/ipa_common.h
+++ b/server/providers/ipa/ipa_common.h
@@ -27,6 +27,11 @@
 #include "providers/ldap/ldap_common.h"
 #include "providers/krb5/krb5_common.h"
 
+struct ipa_service {
+    struct sdap_service *sdap;
+    struct krb_server *krb_server;
+};
+
 enum ipa_basic_opt {
     IPA_DOMAIN = 0,
     IPA_SERVER,
@@ -38,6 +43,8 @@ enum ipa_basic_opt {
 struct ipa_options {
     struct dp_option *basic;
 
+    struct ipa_service *service;
+
     /* id provider */
     struct sdap_options *id;
     struct sdap_id_ctx *id_ctx;
@@ -64,4 +71,7 @@ int ipa_get_auth_options(struct ipa_options *ipa_opts,
                          const char *conf_path,
                          struct dp_option **_opts);
 
+int ipa_service_init(TALLOC_CTX *memctx, struct be_ctx *ctx,
+                     const char *servers, struct ipa_service **_service);
+
 #endif /* _IPA_COMMON_H_ */
diff --git a/server/providers/ipa/ipa_init.c b/server/providers/ipa/ipa_init.c
index 70145287..ea279978 100644
--- a/server/providers/ipa/ipa_init.c
+++ b/server/providers/ipa/ipa_init.c
@@ -56,6 +56,34 @@ struct bet_ops ipa_access_ops = {
     .finalize = NULL
 };
 
+int common_ipa_init(struct be_ctx *bectx)
+{
+    const char *ipa_servers;
+    int ret;
+
+    ret = ipa_get_options(bectx, bectx->cdb,
+                          bectx->conf_path,
+                          bectx->domain, &ipa_options);
+    if (ret != EOK) {
+        return ret;
+    }
+
+    ipa_servers = dp_opt_get_string(ipa_options->basic, IPA_SERVER);
+    if (!ipa_servers) {
+        DEBUG(0, ("Missing ipa_server option!\n"));
+        return EINVAL;
+    }
+
+    ret = ipa_service_init(ipa_options, bectx,
+                           ipa_servers, &ipa_options->service);
+    if (ret != EOK) {
+        DEBUG(0, ("Failed to init IPA failover service!\n"));
+        return ret;
+    }
+
+    return EOK;
+}
+
 int sssm_ipa_init(struct be_ctx *bectx,
                   struct bet_ops **ops,
                   void **pvt_data)
@@ -64,12 +92,10 @@ int sssm_ipa_init(struct be_ctx *bectx,
     int ret;
 
     if (!ipa_options) {
-        ipa_get_options(bectx, bectx->cdb,
-                        bectx->conf_path,
-                        bectx->domain, &ipa_options);
-    }
-    if (!ipa_options) {
-        return ENOMEM;
+        ret = common_ipa_init(bectx);
+        if (ret != EOK) {
+            return ret;
+        }
     }
 
     if (ipa_options->id_ctx) {
@@ -84,6 +110,7 @@ int sssm_ipa_init(struct be_ctx *bectx,
         return ENOMEM;
     }
     ctx->be = bectx;
+    ctx->service = ipa_options->service->sdap;
     ipa_options->id_ctx = ctx;
 
     ret = ipa_get_id_options(ipa_options, bectx->cdb,
@@ -127,12 +154,10 @@ int sssm_ipa_auth_init(struct be_ctx *bectx,
     int ret;
 
     if (!ipa_options) {
-        ipa_get_options(bectx, bectx->cdb,
-                        bectx->conf_path,
-                        bectx->domain, &ipa_options);
-    }
-    if (!ipa_options) {
-        return ENOMEM;
+        ret = common_ipa_init(bectx);
+        if (ret != EOK) {
+            return ret;
+        }
     }
 
     if (ipa_options->auth_ctx) {
@@ -146,6 +171,7 @@ int sssm_ipa_auth_init(struct be_ctx *bectx,
     if (!ctx) {
         return ENOMEM;
     }
+    ctx->server = ipa_options->service->krb_server;
     ipa_options->auth_ctx = ctx;
 
     ret = ipa_get_auth_options(ipa_options, bectx->cdb,
diff --git a/server/providers/krb5/krb5_auth.h b/server/providers/krb5/krb5_auth.h
index 54ce2b8b..7851ebba 100644
--- a/server/providers/krb5/krb5_auth.h
+++ b/server/providers/krb5/krb5_auth.h
@@ -52,6 +52,8 @@ struct krb5child_req {
     bool is_offline;
 };
 
+struct fo_service;
+
 struct krb5_ctx {
     /* opts taken from kinit */
     /* in seconds */
@@ -76,7 +78,9 @@ struct krb5_ctx {
     char* k4_cache_name;
 
     action_type action;
+
     struct dp_option *opts;
+    struct krb_server *server;
     int child_debug_fd;
 };
 
diff --git a/server/providers/krb5/krb5_common.c b/server/providers/krb5/krb5_common.c
index 30878de3..6c235364 100644
--- a/server/providers/krb5/krb5_common.c
+++ b/server/providers/krb5/krb5_common.c
@@ -50,8 +50,7 @@ errno_t check_and_export_options(struct dp_option *opts,
 
     dummy = dp_opt_get_cstring(opts, KRB5_KDC);
     if (dummy == NULL) {
-        DEBUG(1, ("No KDC configured, "
-                  "using kerberos defaults from /etc/krb5.conf"));
+        DEBUG(2, ("No KDC expicitly configured, using defaults"));
     } else {
         ret = setenv(SSSD_KRB5_KDC, dummy, 1);
         if (ret != EOK) {
diff --git a/server/providers/krb5/krb5_common.h b/server/providers/krb5/krb5_common.h
index cb60f425..42b00373 100644
--- a/server/providers/krb5/krb5_common.h
+++ b/server/providers/krb5/krb5_common.h
@@ -50,6 +50,10 @@ enum krb5_opts {
     KRB5_OPTS
 };
 
+struct krb_server {
+    char *address;
+};
+
 errno_t check_and_export_options(struct dp_option *opts,
                                  struct sss_domain_info *dom);
 
diff --git a/server/providers/ldap/ldap_auth.c b/server/providers/ldap/ldap_auth.c
index a9f03a76..6a80df44 100644
--- a/server/providers/ldap/ldap_auth.c
+++ b/server/providers/ldap/ldap_auth.c
@@ -414,8 +414,11 @@ struct auth_state {
     char *dn;
     enum pwexpire pw_expire_type;
     void *pw_expire_data;
+
+    struct fo_server *srv;
 };
 
+static void auth_resolve_done(struct tevent_req *subreq);
 static void auth_connect_done(struct tevent_req *subreq);
 static void auth_get_user_dn_done(struct tevent_req *subreq);
 static void auth_bind_user_done(struct tevent_req *subreq);
@@ -436,11 +439,12 @@ static struct tevent_req *auth_send(TALLOC_CTX *memctx,
     state->ctx = ctx;
     state->username = username;
     state->password = password;
+    state->srv = NULL;
 
-    subreq = sdap_connect_send(state, ev, ctx->opts, true);
+    subreq = be_resolve_server_send(state, ev, ctx->be, ctx->service->name);
     if (!subreq) goto fail;
 
-    tevent_req_set_callback(subreq, auth_connect_done, req);
+    tevent_req_set_callback(subreq, auth_resolve_done, req);
 
     return req;
 
@@ -449,6 +453,31 @@ fail:
     return NULL;
 }
 
+static void auth_resolve_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = tevent_req_callback_data(subreq,
+                                                      struct tevent_req);
+    struct auth_state *state = tevent_req_data(req,
+                                                    struct auth_state);
+    int ret;
+
+    ret = be_resolve_server_recv(subreq, &state->srv);
+    talloc_zfree(subreq);
+    if (ret) {
+        tevent_req_error(req, ret);
+        return;
+    }
+
+    subreq = sdap_connect_send(state, state->ev, state->ctx->opts,
+                               state->ctx->service->uri, true);
+    if (!subreq) {
+        tevent_req_error(req, ENOMEM);
+        return;
+    }
+
+    tevent_req_set_callback(subreq, auth_connect_done, req);
+}
+
 static void auth_connect_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
@@ -460,6 +489,11 @@ static void auth_connect_done(struct tevent_req *subreq)
     ret = sdap_connect_recv(subreq, state, &state->sh);
     talloc_zfree(subreq);
     if (ret) {
+        if (state->srv) {
+            /* mark the server as bad if connection failed */
+            fo_set_server_status(state->srv, SERVER_NOT_WORKING);
+        }
+
         tevent_req_error(req, ret);
         return;
     }
diff --git a/server/providers/ldap/ldap_common.c b/server/providers/ldap/ldap_common.c
index 6b619f90..6236707f 100644
--- a/server/providers/ldap/ldap_common.c
+++ b/server/providers/ldap/ldap_common.c
@@ -23,6 +23,7 @@
 */
 
 #include "providers/ldap/ldap_common.h"
+#include "providers/fail_over.h"
 
 struct dp_option default_basic_opts[] = {
     { "ldap_uri", DP_OPT_STRING, { "ldap://localhost" }, NULL_STRING },
@@ -309,3 +310,110 @@ int sdap_id_setup_tasks(struct sdap_id_ctx *ctx)
 
     return ret;
 }
+
+static void sdap_uri_callback(void *private_data, struct fo_server *server)
+{
+    struct sdap_service *service;
+    const char *tmp;
+    char *new_uri;
+
+    service = talloc_get_type(private_data, struct sdap_service);
+    if (!service) return;
+
+    tmp = (const char *)fo_get_server_user_data(server);
+    if (tmp && ldap_is_ldap_url(tmp)) {
+        new_uri = talloc_strdup(service, tmp);
+    } else {
+        new_uri = talloc_asprintf(service, "ldap://%s",
+                                  fo_get_server_name(server));
+    }
+    if (!new_uri) {
+        DEBUG(2, ("Failed to copy URI ...\n"));
+        return;
+    }
+
+    /* free old one and replace with new one */
+    talloc_zfree(service->uri);
+    service->uri = new_uri;
+}
+
+int sdap_service_init(TALLOC_CTX *memctx, struct be_ctx *ctx,
+                      const char *service_name, const char *urls,
+                      struct sdap_service **_service)
+{
+    TALLOC_CTX *tmp_ctx;
+    struct sdap_service *service;
+    LDAPURLDesc *lud;
+    char **list = NULL;
+    int count = 0;
+    int ret;
+    int i;
+
+    tmp_ctx = talloc_new(memctx);
+    if (!tmp_ctx) {
+        return ENOMEM;
+    }
+
+    service = talloc_zero(tmp_ctx, struct sdap_service);
+    if (!service) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    ret = be_fo_add_service(ctx, service_name);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to create failover service!\n"));
+        goto done;
+    }
+
+    service->name = talloc_strdup(service, service_name);
+    if (!service->name) {
+        ret = ENOMEM;
+        goto done;
+    }
+
+    /* split server parm into a list */
+    ret = sss_split_list(tmp_ctx, urls, ", ", &list, &count);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to parse server list!\n"));
+        goto done;
+    }
+
+    /* now for each URI add a new server to the failover service */
+    for (i = 0; i < count; i++) {
+        ret = ldap_url_parse(list[i], &lud);
+        if (ret != LDAP_SUCCESS) {
+            DEBUG(0, ("Failed to parse ldap URI (%s)!\n", list[i]));
+            ret = EINVAL;
+            goto done;
+        }
+
+        DEBUG(6, ("Added URI %s\n", list[i]));
+
+        talloc_steal(service, list[i]);
+
+        ret = be_fo_add_server(ctx, service->name,
+                               lud->lud_host, lud->lud_port, list[i]);
+        if (ret) {
+            goto done;
+        }
+        ldap_free_urldesc(lud);
+    }
+
+    ret = be_fo_service_add_callback(memctx, ctx, service->name,
+                                     sdap_uri_callback, service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to add failover callback!\n"));
+        goto done;
+    }
+
+    ret = EOK;
+
+done:
+    if (ret == EOK) {
+        *_service = talloc_steal(memctx, service);
+    }
+    talloc_zfree(tmp_ctx);
+    return ret;
+}
+
diff --git a/server/providers/ldap/ldap_common.h b/server/providers/ldap/ldap_common.h
index 96b332cf..6adb785a 100644
--- a/server/providers/ldap/ldap_common.h
+++ b/server/providers/ldap/ldap_common.h
@@ -24,11 +24,13 @@
 
 #include "providers/dp_backend.h"
 #include "providers/ldap/sdap.h"
+#include "providers/fail_over.h"
 
 struct sdap_id_ctx {
     struct be_ctx *be;
-
     struct sdap_options *opts;
+    struct fo_service *fo_service;
+    struct sdap_service *service;
 
     /* what rootDSE returns */
     struct sysdb_attrs *rootDSE;
@@ -48,6 +50,8 @@ struct sdap_id_ctx {
 struct sdap_auth_ctx {
     struct be_ctx *be;
     struct sdap_options *opts;
+    struct fo_service *fo_service;
+    struct sdap_service *service;
 };
 
 /* id */
@@ -65,6 +69,10 @@ void sdap_pam_chpass_handler(struct be_req *breq);
 void sdap_handler_done(struct be_req *req, int dp_err,
                        int error, const char *errstr);
 
+int sdap_service_init(TALLOC_CTX *mmectx, struct be_ctx *ctx,
+                      const char *service_name, const char *urls,
+                      struct sdap_service **service);
+
 /* options parser */
 int ldap_get_options(TALLOC_CTX *memctx,
                      struct confdb_ctx *cdb,
diff --git a/server/providers/ldap/ldap_id.c b/server/providers/ldap/ldap_id.c
index 52391c28..f99ea7b9 100644
--- a/server/providers/ldap/ldap_id.c
+++ b/server/providers/ldap/ldap_id.c
@@ -104,7 +104,9 @@ struct tevent_req *users_get_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
@@ -325,7 +327,9 @@ struct tevent_req *groups_get_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
@@ -511,7 +515,9 @@ static struct tevent_req *groups_by_user_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
diff --git a/server/providers/ldap/ldap_id_enum.c b/server/providers/ldap/ldap_id_enum.c
index 916389fe..1ddcbf8f 100644
--- a/server/providers/ldap/ldap_id_enum.c
+++ b/server/providers/ldap/ldap_id_enum.c
@@ -345,7 +345,9 @@ static struct tevent_req *enum_users_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
@@ -498,7 +500,9 @@ static struct tevent_req *enum_groups_send(TALLOC_CTX *memctx,
 
         /* FIXME: add option to decide if tls should be used
          * or SASL/GSSAPI, etc ... */
-        subreq = sdap_cli_connect_send(state, ev, ctx->opts, &ctx->rootDSE);
+        subreq = sdap_cli_connect_send(state, ev, ctx->opts,
+                                       ctx->be, ctx->service,
+                                       &ctx->rootDSE);
         if (!subreq) {
             ret = ENOMEM;
             goto fail;
diff --git a/server/providers/ldap/ldap_init.c b/server/providers/ldap/ldap_init.c
index 295ff19d..5a64585d 100644
--- a/server/providers/ldap/ldap_init.c
+++ b/server/providers/ldap/ldap_init.c
@@ -49,6 +49,7 @@ int sssm_ldap_init(struct be_ctx *bectx,
                    void **pvt_data)
 {
     struct sdap_id_ctx *ctx;
+    const char *urls;
     int ret;
 
     ctx = talloc_zero(bectx, struct sdap_id_ctx);
@@ -62,6 +63,19 @@ int sssm_ldap_init(struct be_ctx *bectx,
         goto done;
     }
 
+    urls = dp_opt_get_string(ctx->opts->basic, SDAP_URI);
+    if (!urls) {
+        DEBUG(0, ("Missing ldap_uri\n"));
+        ret = EINVAL;
+        goto done;
+    }
+
+    ret = sdap_service_init(ctx, ctx->be, "LDAP", urls, &ctx->service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to initialize failover service!\n"));
+        goto done;
+    }
+
     ret = setup_tls_config(ctx->opts->basic);
     if (ret != EOK) {
         DEBUG(1, ("setup_tls_config failed [%d][%s].\n",
@@ -90,6 +104,7 @@ int sssm_ldap_auth_init(struct be_ctx *bectx,
                         void **pvt_data)
 {
     struct sdap_auth_ctx *ctx;
+    const char *urls;
     int ret;
 
     ctx = talloc(bectx, struct sdap_auth_ctx);
@@ -103,6 +118,19 @@ int sssm_ldap_auth_init(struct be_ctx *bectx,
         goto done;
     }
 
+    urls = dp_opt_get_string(ctx->opts->basic, SDAP_URI);
+    if (!urls) {
+        DEBUG(0, ("Missing ldap_uri\n"));
+        ret = EINVAL;
+        goto done;
+    }
+
+    ret = sdap_service_init(ctx, ctx->be, "LDAP", urls, &ctx->service);
+    if (ret != EOK) {
+        DEBUG(1, ("Failed to initialize failover service!\n"));
+        goto done;
+    }
+
     ret = setup_tls_config(ctx->opts->basic);
     if (ret != EOK) {
         DEBUG(1, ("setup_tls_config failed [%d][%s].\n",
diff --git a/server/providers/ldap/sdap.h b/server/providers/ldap/sdap.h
index 8330bd6f..a63d8580 100644
--- a/server/providers/ldap/sdap.h
+++ b/server/providers/ldap/sdap.h
@@ -62,6 +62,11 @@ struct sdap_handle {
     struct sdap_op *ops;
 };
 
+struct sdap_service {
+    char *name;
+    char *uri;
+};
+
 #define SYSDB_SHADOWPW_LASTCHANGE "shadowLastChange"
 #define SYSDB_SHADOWPW_MIN "shadowMin"
 #define SYSDB_SHADOWPW_MAX "shadowMax"
diff --git a/server/providers/ldap/sdap_async.h b/server/providers/ldap/sdap_async.h
index 383a2fce..d8550794 100644
--- a/server/providers/ldap/sdap_async.h
+++ b/server/providers/ldap/sdap_async.h
@@ -26,10 +26,12 @@
 #include <tevent.h>
 #include "providers/dp_backend.h"
 #include "providers/ldap/sdap.h"
+#include "providers/fail_over.h"
 
 struct tevent_req *sdap_connect_send(TALLOC_CTX *memctx,
                                      struct tevent_context *ev,
                                      struct sdap_options *opts,
+                                     const char *uri,
                                      bool use_start_tls);
 int sdap_connect_recv(struct tevent_req *req,
                       TALLOC_CTX *memctx,
@@ -91,11 +93,14 @@ struct tevent_req *sdap_exop_modify_passwd_send(TALLOC_CTX *memctx,
                                                 char *user_dn,
                                                 char *password,
                                                 char *new_password);
-int sdap_exop_modify_passwd_recv(struct tevent_req *req, enum sdap_result *result);
+int sdap_exop_modify_passwd_recv(struct tevent_req *req,
+                                 enum sdap_result *result);
 
 struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
                                          struct tevent_context *ev,
                                          struct sdap_options *opts,
+                                         struct be_ctx *be,
+                                         struct sdap_service *service,
                                          struct sysdb_attrs **rootdse);
 int sdap_cli_connect_recv(struct tevent_req *req,
                           TALLOC_CTX *memctx,
diff --git a/server/providers/ldap/sdap_async_connection.c b/server/providers/ldap/sdap_async_connection.c
index 9f53ad6f..a5e6ccfa 100644
--- a/server/providers/ldap/sdap_async_connection.c
+++ b/server/providers/ldap/sdap_async_connection.c
@@ -46,6 +46,7 @@ static void sdap_connect_done(struct sdap_op *op,
 struct tevent_req *sdap_connect_send(TALLOC_CTX *memctx,
                                      struct tevent_context *ev,
                                      struct sdap_options *opts,
+                                     const char *uri,
                                      bool use_start_tls)
 {
     struct tevent_req *req;
@@ -73,10 +74,9 @@ struct tevent_req *sdap_connect_send(TALLOC_CTX *memctx,
         return NULL;
     }
     /* Initialize LDAP handler */
-    lret = ldap_initialize(&state->sh->ldap,
-                           dp_opt_get_string(opts->basic, SDAP_URI));
+    lret = ldap_initialize(&state->sh->ldap, uri);
     if (lret != LDAP_SUCCESS) {
-        DEBUG(1, ("ldap_initialize failed: %s\n", ldap_err2string(ret)));
+        DEBUG(1, ("ldap_initialize failed: %s\n", ldap_err2string(lret)));
         goto fail;
     }
 
@@ -871,12 +871,17 @@ int sdap_auth_recv(struct tevent_req *req, enum sdap_result *result)
 struct sdap_cli_connect_state {
     struct tevent_context *ev;
     struct sdap_options *opts;
+    struct sdap_service *service;
 
-    struct sysdb_attrs *rootdse;
     bool use_rootdse;
+    struct sysdb_attrs *rootdse;
+
     struct sdap_handle *sh;
+
+    struct fo_server *srv;
 };
 
+static void sdap_cli_resolve_done(struct tevent_req *subreq);
 static void sdap_cli_connect_done(struct tevent_req *subreq);
 static void sdap_cli_rootdse_step(struct tevent_req *req);
 static void sdap_cli_rootdse_done(struct tevent_req *subreq);
@@ -888,6 +893,8 @@ static void sdap_cli_auth_done(struct tevent_req *subreq);
 struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
                                          struct tevent_context *ev,
                                          struct sdap_options *opts,
+                                         struct be_ctx *be,
+                                         struct sdap_service *service,
                                          struct sysdb_attrs **rootdse)
 {
     struct tevent_req *req, *subreq;
@@ -898,6 +905,9 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
 
     state->ev = ev;
     state->opts = opts;
+    state->service = service;
+    state->srv = NULL;
+
     if (rootdse) {
         state->use_rootdse = true;
         state->rootdse = *rootdse;
@@ -906,17 +916,46 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
         state->rootdse = NULL;
     }
 
-    subreq = sdap_connect_send(state, ev, opts,
-                               dp_opt_get_bool(opts->basic, SDAP_ID_TLS));
+    /* NOTE: this call may cause service->uri to be refreshed
+     * with a new valid server. Do not use service->uri before */
+    subreq = be_resolve_server_send(state, ev, be, service->name);
     if (!subreq) {
         talloc_zfree(req);
         return NULL;
     }
-    tevent_req_set_callback(subreq, sdap_cli_connect_done, req);
+    tevent_req_set_callback(subreq, sdap_cli_resolve_done, req);
 
     return req;
 }
 
+static void sdap_cli_resolve_done(struct tevent_req *subreq)
+{
+    struct tevent_req *req = tevent_req_callback_data(subreq,
+                                                      struct tevent_req);
+    struct sdap_cli_connect_state *state = tevent_req_data(req,
+                                             struct sdap_cli_connect_state);
+    int ret;
+
+    ret = be_resolve_server_recv(subreq, &state->srv);
+    talloc_zfree(subreq);
+    if (ret) {
+        /* all servers have been tried and none
+         * was found good, go offline */
+        tevent_req_error(req, EIO);
+        return;
+    }
+
+    subreq = sdap_connect_send(state, state->ev, state->opts,
+                               state->service->uri,
+                               dp_opt_get_bool(state->opts->basic,
+                                                            SDAP_ID_TLS));
+    if (!subreq) {
+        tevent_req_error(req, ENOMEM);
+        return;
+    }
+    tevent_req_set_callback(subreq, sdap_cli_connect_done, req);
+}
+
 static void sdap_cli_connect_done(struct tevent_req *subreq)
 {
     struct tevent_req *req = tevent_req_callback_data(subreq,
@@ -1118,8 +1157,20 @@ int sdap_cli_connect_recv(struct tevent_req *req,
 {
     struct sdap_cli_connect_state *state = tevent_req_data(req,
                                              struct sdap_cli_connect_state);
+    enum tevent_req_state tstate;
+    uint64_t err;
 
-    TEVENT_REQ_RETURN_ON_ERROR(req);
+    if (tevent_req_is_error(req, &tstate, &err)) {
+        /* mark the server as bad if connection failed */
+        if (state->srv) {
+            fo_set_server_status(state->srv, SERVER_NOT_WORKING);
+        }
+
+        if (tstate == TEVENT_REQ_USER_ERROR) {
+            return err;
+        }
+        return EIO;
+    }
 
     if (gsh) {
         *gsh = talloc_steal(memctx, state->sh);
diff --git a/server/util/util.c b/server/util/util.c
new file mode 100644
index 00000000..9049602c
--- /dev/null
+++ b/server/util/util.c
@@ -0,0 +1,85 @@
+/*
+    Authors:
+        Simo Sorce <ssorce@redhat.com>
+
+    Copyright (C) 2009 Red Hat
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "talloc.h"
+#include "util/util.h"
+
+/* Split string in a list using a set of legal seprators */
+
+int sss_split_list(TALLOC_CTX *memctx, const char *string,
+                   const char *sep, char ***_list, int *c)
+{
+    const char *p;
+    const char *s;
+    char **list;
+    char **t;
+    int i;
+
+    /* split server parm into a list */
+    list = NULL;
+    s = string;
+    i = 0;
+
+    while (s) {
+        p = strpbrk(s, sep);
+        if (p) {
+            if (p - s == 1) {
+                s++;
+                continue;
+            }
+
+            t = talloc_realloc(memctx, list, char *, i + 1);
+            if (!t) {
+                talloc_zfree(list);
+                return ENOMEM;
+            }
+            list = t;
+            list[i] = talloc_asprintf(list, "%.*s", (int)(p - s), s);
+            if (!list[i]) {
+                talloc_zfree(list);
+                return ENOMEM;
+            }
+            i++;
+
+            s = p + 1;
+        }
+        else {
+
+            t = talloc_realloc(memctx, list, char *, i + 1);
+            if (!t) {
+                talloc_zfree(list);
+                return ENOMEM;
+            }
+            list = t;
+            list[i] = talloc_strdup(list, s);
+            if (!list[i]) {
+                talloc_zfree(list);
+                return ENOMEM;
+            }
+            i++;
+
+            s = NULL;
+        }
+    }
+
+    *_list = list;
+    *c = i;
+    return EOK;
+}
diff --git a/server/util/util.h b/server/util/util.h
index 3ca43a40..b116a66a 100644
--- a/server/util/util.h
+++ b/server/util/util.h
@@ -221,4 +221,8 @@ int backup_file(const char *src, int dbglvl);
 errno_t check_and_open_readonly(const char *filename, int *fd, const uid_t uid,
                                const gid_t gid, const mode_t mode);
 
+/* from util.c */
+int sss_split_list(TALLOC_CTX *memctx, const char *string,
+                   const char *sep, char ***_list, int *c);
+
 #endif /* __SSSD_UTIL_H__ */