Failover: Introduce a per-service timeout

https://fedorahosted.org/sssd/ticket/976

4 files changed, 67 insertions, 5 deletions

diff --git a/src/confdb/confdb.h b/src/confdb/confdb.h
index e3f8ba3c..7cfc73d2 100644
--- a/src/confdb/confdb.h
+++ b/src/confdb/confdb.h
@@ -117,6 +117,7 @@
 #define CONFDB_DOMAIN_FQ "use_fully_qualified_names"
 #define CONFDB_DOMAIN_ENTRY_CACHE_TIMEOUT "entry_cache_timeout"
 #define CONFDB_DOMAIN_RESOLV_TIMEOUT "dns_resolver_timeout"
+#define CONFDB_DOMAIN_RESOLV_OP_TIMEOUT "dns_resolver_op_timeout"
 #define CONFDB_DOMAIN_DNS_DISCOVERY_NAME "dns_discovery_domain"
 #define CONFDB_DOMAIN_FAMILY_ORDER "lookup_family_order"
 #define CONFDB_DOMAIN_ACCOUNT_CACHE_EXPIRATION "account_cache_expiration"
diff --git a/src/providers/data_provider_fo.c b/src/providers/data_provider_fo.c
index 7a5c82c3..dd73e9d9 100644
--- a/src/providers/data_provider_fo.c
+++ b/src/providers/data_provider_fo.c
@@ -67,7 +67,14 @@ static int be_fo_get_options(struct be_ctx *ctx,
 {
     errno_t ret;
 
-    /* todo get timeout from configuration */
+    ret = confdb_get_int(ctx->cdb, ctx, ctx->conf_path,
+                         CONFDB_DOMAIN_RESOLV_TIMEOUT,
+                         FO_DEFAULT_SVC_TIMEOUT,
+                         &opts->service_resolv_timeout);
+    if (ret != EOK) {
+        return ret;
+    }
+
     opts->retry_timeout = 30;
     opts->srv_retry_timeout = 14400;
 
@@ -83,7 +90,7 @@ static int be_fo_get_options(struct be_ctx *ctx,
 int be_init_failover(struct be_ctx *ctx)
 {
     int ret;
-    int fo_timeout;
+    int resolv_timeout;
     struct fo_options fopts;
 
     if (ctx->be_fo != NULL) {
@@ -96,13 +103,14 @@ int be_init_failover(struct be_ctx *ctx)
     }
 
     ret = confdb_get_int(ctx->cdb, ctx, ctx->conf_path,
-                         CONFDB_DOMAIN_RESOLV_TIMEOUT,
-                         RESOLV_DEFAULT_TIMEOUT, &fo_timeout);
+                         CONFDB_DOMAIN_RESOLV_OP_TIMEOUT,
+                         RESOLV_DEFAULT_TIMEOUT,
+                         &resolv_timeout);
     if (ret != EOK) {
         return ret;
     }
 
-    ret = resolv_init(ctx, ctx->ev, fo_timeout, &ctx->be_fo->resolv);
+    ret = resolv_init(ctx, ctx->ev, resolv_timeout, &ctx->be_fo->resolv);
     if (ret != EOK) {
         talloc_zfree(ctx->be_fo);
         return ret;
diff --git a/src/providers/fail_over.c b/src/providers/fail_over.c
index 5cfc1a41..637b5b4d 100644
--- a/src/providers/fail_over.c
+++ b/src/providers/fail_over.c
@@ -141,6 +141,7 @@ fo_context_init(TALLOC_CTX *mem_ctx, struct fo_options *opts)
     ctx->opts->srv_retry_timeout = opts->srv_retry_timeout;
     ctx->opts->retry_timeout = opts->retry_timeout;
     ctx->opts->family_order  = opts->family_order;
+    ctx->opts->service_resolv_timeout = opts->service_resolv_timeout;
 
     DEBUG(3, ("Created new fail over context, retry timeout is %d\n",
               ctx->opts->retry_timeout));
@@ -735,10 +736,13 @@ struct resolve_service_state {
 
     struct resolv_ctx *resolv;
     struct tevent_context *ev;
+    struct tevent_timer *timeout_handler;
     struct fo_ctx *fo_ctx;
 };
 
 
+static errno_t fo_resolve_service_activate_timeout(struct tevent_req *req,
+            struct tevent_context *ev, const unsigned long timeout_seconds);
 static void fo_resolve_service_cont(struct tevent_req *subreq);
 static void fo_resolve_service_done(struct tevent_req *subreq);
 static bool fo_resolve_service_server(struct tevent_req *req);
@@ -777,6 +781,14 @@ fo_resolve_service_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
         goto done;
     }
 
+    /* Activate per-service timeout handler */
+    ret = fo_resolve_service_activate_timeout(req, ev,
+                                        ctx->opts->service_resolv_timeout);
+    if (ret != EOK) {
+        DEBUG(SSSDBG_OP_FAILURE, ("Could not set service timeout\n"));
+        goto done;
+    }
+
     if (fo_is_srv_lookup(server)) {
         /* Don't know the server yet, must do a SRV lookup */
         subreq = resolve_srv_send(state, ev, resolv,
@@ -810,6 +822,40 @@ done:
 static void set_server_common_status(struct server_common *common,
                                      enum server_status status);
 
+static void
+fo_resolve_service_timeout(struct tevent_context *ev,
+                           struct tevent_timer *te,
+                           struct timeval tv, void *pvt)
+{
+    struct tevent_req *req = talloc_get_type(pvt, struct tevent_req);
+
+    DEBUG(SSSDBG_MINOR_FAILURE, ("Service resolving timeout reached\n"));
+    tevent_req_error(req, ETIMEDOUT);
+}
+
+static errno_t
+fo_resolve_service_activate_timeout(struct tevent_req *req,
+                                    struct tevent_context *ev,
+                                    const unsigned long timeout_seconds)
+{
+    struct timeval tv;
+    struct resolve_service_state *state = tevent_req_data(req,
+                                        struct resolve_service_state);
+
+    tv = tevent_timeval_current();
+    tv = tevent_timeval_add(&tv, timeout_seconds, 0);
+    state->timeout_handler = tevent_add_timer(ev, state, tv,
+                                           fo_resolve_service_timeout, req);
+    if (state->timeout_handler == NULL) {
+        DEBUG(SSSDBG_CRIT_FAILURE, ("tevent_add_timer failed.\n"));
+        return ENOMEM;
+    }
+
+    DEBUG(SSSDBG_TRACE_INTERNAL, ("Resolve timeout set to %lu seconds\n",
+          timeout_seconds));
+    return EOK;
+}
+
 /* SRV resolving finished, see if we got server to work with */
 static void
 fo_resolve_service_cont(struct tevent_req *subreq)
diff --git a/src/providers/fail_over.h b/src/providers/fail_over.h
index 3ca6096c..fee64e82 100644
--- a/src/providers/fail_over.h
+++ b/src/providers/fail_over.h
@@ -33,6 +33,8 @@
 #define FO_PROTO_TCP "tcp"
 #define FO_PROTO_UDP "udp"
 
+#define FO_DEFAULT_SVC_TIMEOUT 10
+
 /* Some forward declarations that don't have to do anything with fail over. */
 struct hostent;
 struct tevent_context;
@@ -63,6 +65,10 @@ struct fo_server;
  * duration in seconds of how long a server or port will be considered
  * non-working after being marked as such.
  *
+ * The 'service_resolv_timeout' member specifies how long we wait for
+ * service resolution. When this timeout is reached, the resolve request
+ * is cancelled with an error
+ *
  * The 'srv_retry_timeout' member specifies how long a SRV lookup
  * is considered valid until we ask the server again.
  *
@@ -72,6 +78,7 @@ struct fo_server;
 struct fo_options {
     time_t srv_retry_timeout;
     time_t retry_timeout;
+    int service_resolv_timeout;
     enum restrict_family family_order;
 };