Only do one cycle when resolving a server

https://fedorahosted.org/sssd/ticket/1214

7 files changed, 105 insertions, 37 deletions

diff --git a/src/providers/data_provider_fo.c b/src/providers/data_provider_fo.c
index 4c2c0b4a..c358e68d 100644
--- a/src/providers/data_provider_fo.c
+++ b/src/providers/data_provider_fo.c
@@ -46,6 +46,7 @@ struct be_svc_data {
     bool run_callbacks;
 
     struct be_svc_callback *callbacks;
+    struct fo_server *first_resolved;
 };
 
 struct be_failover_ctx {
@@ -398,42 +399,43 @@ static void be_resolve_server_done(struct tevent_req *subreq)
     switch (ret) {
     case EOK:
         if (!state->srv) {
-            tevent_req_error(req, EFAULT);
-            return;
+            ret = EFAULT;
+            goto fail;
         }
         break;
 
     case ENOENT:
         /* all servers have been tried and none
          * was found good, go offline */
-        tevent_req_error(req, EIO);
-        return;
+        ret = EIO;
+        goto fail;
 
     default:
         /* mark server as bad and retry */
         if (!state->srv) {
-            tevent_req_error(req, EFAULT);
-            return;
+            ret = EFAULT;
+            goto fail;
         }
-        DEBUG(6, ("Couldn't resolve server (%s), resolver returned (%d)\n",
-                  fo_get_server_str_name(state->srv), ret));
+        DEBUG(SSSDBG_MINOR_FAILURE,
+              ("Couldn't resolve server (%s), resolver returned (%d)\n",
+              fo_get_server_str_name(state->srv), ret));
 
         state->attempts++;
         if (state->attempts >= 10) {
             DEBUG(2, ("Failed to find a server after 10 attempts\n"));
-            tevent_req_error(req, EIO);
-            return;
+            ret = EIO;
+            goto fail;
         }
 
         /* now try next one */
-        DEBUG(6, ("Trying with the next one!\n"));
+        DEBUG(SSSDBG_TRACE_LIBS, ("Trying with the next one!\n"));
         subreq = fo_resolve_service_send(state, state->ev,
                                          state->ctx->be_fo->resolv,
                                          state->ctx->be_fo->fo_ctx,
                                          state->svc->fo_service);
         if (!subreq) {
-            tevent_req_error(req, ENOMEM);
-            return;
+            ret = ENOMEM;
+            goto fail;
         }
         tevent_req_set_callback(subreq, be_resolve_server_done, req);
 
@@ -441,24 +443,34 @@ static void be_resolve_server_done(struct tevent_req *subreq)
     }
 
     /* all fine we got the server */
+    if (state->svc->first_resolved == NULL) {
+        DEBUG(SSSDBG_TRACE_LIBS, ("Saving the first resolved server\n"));
+        state->svc->first_resolved = state->srv;
+    } else if (state->svc->first_resolved == state->srv) {
+        DEBUG(SSSDBG_OP_FAILURE,
+              ("The fail over cycled through all available servers\n"));
+        ret = ENOENT;
+        goto fail;
+    }
 
-    if (DEBUG_IS_SET(SSSDBG_CONF_SETTINGS) && fo_get_server_name(state->srv)) {
+    if (DEBUG_IS_SET(SSSDBG_FUNC_DATA) && fo_get_server_name(state->srv)) {
         struct resolv_hostent *srvaddr;
         char ipaddr[128];
         srvaddr = fo_get_server_hostent(state->srv);
         if (!srvaddr) {
-            DEBUG(3, ("FATAL: No hostent available for server (%s)\n",
-                      fo_get_server_str_name(state->srv)));
-            tevent_req_error(req, EFAULT);
-            return;
+            DEBUG(SSSDBG_CRIT_FAILURE,
+                  ("FATAL: No hostent available for server (%s)\n",
+                  fo_get_server_str_name(state->srv)));
+            ret = EFAULT;
+            goto fail;
         }
 
         inet_ntop(srvaddr->family, srvaddr->addr_list[0]->ipaddr,
                   ipaddr, 128);
 
-        DEBUG(4, ("Found address for server %s: [%s] TTL %d\n",
-                fo_get_server_str_name(state->srv), ipaddr,
-                srvaddr->addr_list[0]->ttl));
+        DEBUG(SSSDBG_FUNC_DATA, ("Found address for server %s: [%s] TTL %d\n",
+              fo_get_server_str_name(state->srv), ipaddr,
+              srvaddr->addr_list[0]->ttl));
     }
 
     srv_status_change = fo_get_server_hostname_last_change(state->srv);
@@ -478,6 +490,12 @@ static void be_resolve_server_done(struct tevent_req *subreq)
     }
 
     tevent_req_done(req);
+    return;
+
+fail:
+    DEBUG(SSSDBG_TRACE_LIBS, ("Server resolution failed: %d\n", ret));
+    state->svc->first_resolved = NULL;
+    tevent_req_error(req, ret);
 }
 
 int be_resolve_server_recv(struct tevent_req *req, struct fo_server **srv)
@@ -524,3 +542,33 @@ void reset_fo(struct be_ctx *be_ctx)
     fo_reset_services(be_ctx->be_fo->fo_ctx);
 }
 
+void be_fo_set_port_status(struct be_ctx *ctx,
+                           struct fo_server *server,
+                           enum port_status status)
+{
+    struct be_svc_data *svc;
+    struct fo_service *fsvc;
+
+    fo_set_port_status(server, status);
+
+    fsvc = fo_get_server_service(server);
+    if (!fsvc) {
+        DEBUG(SSSDBG_OP_FAILURE, ("BUG: No service associated with server\n"));
+        return;
+    }
+
+    DLIST_FOR_EACH(svc, ctx->be_fo->svcs) {
+        if (svc->fo_service == fsvc) break;
+    }
+
+    if (!svc) {
+        DEBUG(SSSDBG_OP_FAILURE, ("BUG: Unknown service\n"));
+        return;
+    }
+
+    if (status == PORT_WORKING) {
+        /* We were successful in connecting to the server. Cycle through all
+         * available servers next time */
+        svc->first_resolved = NULL;
+    }
+}
diff --git a/src/providers/dp_backend.h b/src/providers/dp_backend.h
index b6b3db50..96c77478 100644
--- a/src/providers/dp_backend.h
+++ b/src/providers/dp_backend.h
@@ -209,6 +209,11 @@ struct tevent_req *be_resolve_server_send(TALLOC_CTX *memctx,
                                           struct be_ctx *ctx,
                                           const char *service_name);
 int be_resolve_server_recv(struct tevent_req *req, struct fo_server **srv);
+
+void be_fo_set_port_status(struct be_ctx *ctx,
+                           struct fo_server *server,
+                           enum port_status status);
+
 /*
  * Instruct fail-over to try next server on the next connect attempt.
  * Should be used after connection to service was unexpectedly dropped
diff --git a/src/providers/fail_over.c b/src/providers/fail_over.c
index 637b5b4d..b8310684 100644
--- a/src/providers/fail_over.c
+++ b/src/providers/fail_over.c
@@ -1476,3 +1476,10 @@ void fo_reset_services(struct fo_ctx *fo_ctx)
     }
 }
 
+struct fo_service *
+fo_get_server_service(struct fo_server *server)
+{
+    if (!server) return NULL;
+    return server->service;
+}
+
diff --git a/src/providers/fail_over.h b/src/providers/fail_over.h
index fee64e82..8fbbe251 100644
--- a/src/providers/fail_over.h
+++ b/src/providers/fail_over.h
@@ -185,4 +185,7 @@ time_t fo_get_server_hostname_last_change(struct fo_server *server);
 int fo_is_srv_lookup(struct fo_server *s);
 
 void fo_reset_services(struct fo_ctx *fo_ctx);
+
+struct fo_service *fo_get_server_service(struct fo_server *server);
+
 #endif /* !__FAIL_OVER_H__ */
diff --git a/src/providers/krb5/krb5_auth.c b/src/providers/krb5/krb5_auth.c
index 14a7c547..72992ba7 100644
--- a/src/providers/krb5/krb5_auth.c
+++ b/src/providers/krb5/krb5_auth.c
@@ -883,14 +883,16 @@ static void krb5_child_done(struct tevent_req *subreq)
     if (kr->kpasswd_srv != NULL) {
         /* ..which is unreachable by now.. */
         if (msg_status == PAM_AUTHTOK_LOCK_BUSY) {
-            fo_set_port_status(kr->kpasswd_srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be_ctx,
+                                  kr->kpasswd_srv, PORT_NOT_WORKING);
             /* ..try to resolve next kpasswd server */
             if (krb5_next_kpasswd(req) == NULL) {
                 tevent_req_error(req, ENOMEM);
             }
             return;
         } else {
-            fo_set_port_status(kr->kpasswd_srv, PORT_WORKING);
+            be_fo_set_port_status(state->be_ctx,
+                                  kr->kpasswd_srv, PORT_WORKING);
         }
     }
 
@@ -900,7 +902,7 @@ static void krb5_child_done(struct tevent_req *subreq)
     if (msg_status == PAM_AUTHINFO_UNAVAIL ||
         (kr->kpasswd_srv == NULL && msg_status == PAM_AUTHTOK_LOCK_BUSY)) {
         if (kr->srv != NULL) {
-            fo_set_port_status(kr->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be_ctx, kr->srv, PORT_NOT_WORKING);
             /* ..try to resolve next KDC */
             if (krb5_next_kdc(req) == NULL) {
                 tevent_req_error(req, ENOMEM);
@@ -908,7 +910,7 @@ static void krb5_child_done(struct tevent_req *subreq)
             return;
         }
     } else if (kr->srv != NULL) {
-        fo_set_port_status(kr->srv, PORT_WORKING);
+        be_fo_set_port_status(state->be_ctx, kr->srv, PORT_WORKING);
     }
 
     /* Now only a successful authentication or password change is left.
@@ -971,17 +973,20 @@ static struct tevent_req *krb5_next_server(struct tevent_req *req)
     switch (pd->cmd) {
         case SSS_PAM_AUTHENTICATE:
         case SSS_CMD_RENEW:
-            fo_set_port_status(state->kr->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be_ctx,
+                                  state->kr->srv, PORT_NOT_WORKING);
             next_req = krb5_next_kdc(req);
             break;
         case SSS_PAM_CHAUTHTOK:
         case SSS_PAM_CHAUTHTOK_PRELIM:
             if (state->kr->kpasswd_srv) {
-                fo_set_port_status(state->kr->kpasswd_srv, PORT_NOT_WORKING);
+                be_fo_set_port_status(state->be_ctx,
+                                      state->kr->kpasswd_srv, PORT_NOT_WORKING);
                 next_req = krb5_next_kpasswd(req);
                 break;
             } else {
-                fo_set_port_status(state->kr->srv, PORT_NOT_WORKING);
+                be_fo_set_port_status(state->be_ctx,
+                                      state->kr->srv, PORT_NOT_WORKING);
                 next_req = krb5_next_kdc(req);
                 break;
             }
diff --git a/src/providers/ldap/ldap_auth.c b/src/providers/ldap/ldap_auth.c
index 170a34b9..8b6173e1 100644
--- a/src/providers/ldap/ldap_auth.c
+++ b/src/providers/ldap/ldap_auth.c
@@ -588,7 +588,8 @@ static void auth_connect_done(struct tevent_req *subreq)
     if (ret) {
         if (state->srv) {
             /* mark this server as bad if connection failed */
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->ctx->be,
+                                  state->srv, PORT_NOT_WORKING);
         }
         if (ret == ETIMEDOUT) {
             if (auth_get_server(req) == NULL) {
@@ -600,7 +601,7 @@ static void auth_connect_done(struct tevent_req *subreq)
         tevent_req_error(req, ret);
         return;
     } else if (state->srv) {
-        fo_set_port_status(state->srv, PORT_WORKING);
+        be_fo_set_port_status(state->ctx->be, state->srv, PORT_WORKING);
     }
 
     ret = get_user_dn(state, state->ctx->be->sysdb, state->ctx->opts,
diff --git a/src/providers/ldap/sdap_async_connection.c b/src/providers/ldap/sdap_async_connection.c
index ca1cd6e1..22aa4f91 100644
--- a/src/providers/ldap/sdap_async_connection.c
+++ b/src/providers/ldap/sdap_async_connection.c
@@ -962,7 +962,7 @@ static void sdap_kinit_done(struct tevent_req *subreq)
         return;
     } else {
         if (kerr == KRB5_KDC_UNREACH) {
-            fo_set_port_status(state->kdc_srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->kdc_srv, PORT_NOT_WORKING);
             nextreq = sdap_kinit_next_kdc(req);
             if (!nextreq) {
                 tevent_req_error(req, ENOMEM);
@@ -1191,7 +1191,6 @@ struct tevent_req *sdap_cli_connect_send(TALLOC_CTX *memctx,
     state->be = be;
     state->srv = NULL;
     state->srv_opts = NULL;
-    state->be = be;
     state->use_rootdse = !skip_rootdse;
     state->force_tls = force_tls;
     state->do_auth = !skip_auth;
@@ -1290,7 +1289,7 @@ static void sdap_cli_connect_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
     if (ret) {
         /* retry another server */
-        fo_set_port_status(state->srv, PORT_NOT_WORKING);
+        be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
         ret = sdap_cli_resolve_next(req);
         if (ret != EOK) {
             tevent_req_error(req, ret);
@@ -1364,7 +1363,7 @@ static void sdap_cli_rootdse_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
     if (ret) {
         if (ret == ETIMEDOUT) { /* retry another server */
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
             ret = sdap_cli_resolve_next(req);
             if (ret != EOK) {
                 tevent_req_error(req, ret);
@@ -1483,7 +1482,7 @@ static void sdap_cli_kinit_done(struct tevent_req *subreq)
     talloc_zfree(subreq);
     if (ret) {
         if (ret == ETIMEDOUT) { /* child timed out, retry another server */
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
             ret = sdap_cli_resolve_next(req);
             if (ret != EOK) {
                 tevent_req_error(req, ret);
@@ -1585,7 +1584,7 @@ int sdap_cli_connect_recv(struct tevent_req *req,
     if (tevent_req_is_error(req, &tstate, &err)) {
         /* mark the server as bad if connection failed */
         if (state->srv) {
-            fo_set_port_status(state->srv, PORT_NOT_WORKING);
+            be_fo_set_port_status(state->be, state->srv, PORT_NOT_WORKING);
         } else {
             if (can_retry) {
                 *can_retry = false;
@@ -1597,7 +1596,7 @@ int sdap_cli_connect_recv(struct tevent_req *req,
         }
         return EIO;
     } else if (state->srv) {
-        fo_set_port_status(state->srv, PORT_WORKING);
+        be_fo_set_port_status(state->be, state->srv, PORT_WORKING);
     }
 
     if (gsh) {