Run checks before resetting offline state

Before setting the backend to online during a reset offline request the
check_online method if the ID provider is called. If the check_online
method returns that the ID provider is still not reachable the backend
stays offline. Otherwise the backend is switched to online and the
related callbacks are run.

Additionally the check online test is called during the res_init request
because a change in /etc/resolve.conf might also make a server reachable
which was assumed offline before.

6 files changed, 143 insertions, 5 deletions

diff --git a/src/providers/data_provider_be.c b/src/providers/data_provider_be.c
index e094c65b..98c3f392 100644
--- a/src/providers/data_provider_be.c
+++ b/src/providers/data_provider_be.c
@@ -173,10 +173,9 @@ void be_mark_offline(struct be_ctx *ctx)
     be_run_offline_cb(ctx);
 }
 
-void be_reset_offline(struct be_ctx *ctx)
+static void be_reset_offline(struct be_ctx *ctx)
 {
-    DEBUG(8, ("Going back online!\n"));
-
+    ctx->offstat.went_offline = 0;
     ctx->offstat.offline = false;
     be_run_online_cb(ctx);
 }
@@ -728,6 +727,114 @@ static int client_registration(DBusMessage *message,
     return EOK;
 }
 
+static errno_t be_file_check_online_request(struct be_req *req)
+{
+    int ret;
+
+    req->be_ctx->offstat.went_offline = time(NULL);
+    reset_fo(req->be_ctx);
+
+    ret = be_file_request(req->be_ctx,
+              req->be_ctx->bet_info[BET_ID].bet_ops->check_online, req);
+    if (ret != EOK) {
+        DEBUG(1, ("be_file_request failed.\n"));
+    }
+
+    return ret;
+}
+
+static void check_online_callback(struct be_req *req, int dp_err_type,
+                                  int errnum, const char *errstr)
+{
+    int ret;
+
+    DEBUG(4, ("Backend returned: (%d, %d, %s) [%s]\n",
+              dp_err_type, errnum, errstr?errstr:"<NULL>",
+              dp_pam_err_to_string(req, dp_err_type, errnum)));
+
+    req->be_ctx->check_online_ref_count--;
+
+    if (dp_err_type != DP_ERR_OK && req->be_ctx->check_online_ref_count > 0) {
+        ret = be_file_check_online_request(req);
+        if (ret != EOK) {
+            DEBUG(1, ("be_file_check_online_request failed.\n"));
+            goto done;
+        }
+        return;
+    }
+
+done:
+    req->be_ctx->check_online_ref_count = 0;
+    if (dp_err_type != DP_ERR_OFFLINE) {
+        if (dp_err_type != DP_ERR_OK) {
+            reset_fo(req->be_ctx);
+        }
+        be_reset_offline(req->be_ctx);
+    }
+
+    talloc_free(req);
+
+    return;
+}
+
+static void check_if_online(struct be_ctx *ctx)
+{
+    int ret;
+    struct be_req *be_req = NULL;
+
+    if (ctx->offstat.offline == false) {
+        DEBUG(8, ("Backend is already online, nothing to do.\n"));
+        return;
+    }
+
+    /* Make sure nobody tries to go online while we are checking */
+    ctx->offstat.went_offline = time(NULL);
+
+    DEBUG(8, ("Trying to go back online!\n"));
+
+    ctx->check_online_ref_count++;
+
+    if (ctx->check_online_ref_count != 1) {
+        DEBUG(8, ("There is an online check already running.\n"));
+        return;
+    }
+
+    if (ctx->bet_info[BET_ID].bet_ops->check_online == NULL) {
+        DEBUG(8, ("ID providers does not provide a check_online method.\n"));
+        goto failed;
+    }
+
+    be_req = talloc_zero(ctx, struct be_req);
+    if (be_req == NULL) {
+        DEBUG(1, ("talloc_zero failed.\n"));
+        goto failed;
+    }
+
+    be_req->be_ctx = ctx;
+    be_req->fn = check_online_callback;
+
+    ret = be_file_check_online_request(be_req);
+    if (ret != EOK) {
+        DEBUG(1, ("be_file_check_online_request failed.\n"));
+        goto failed;
+    }
+
+    return;
+
+failed:
+    ctx->check_online_ref_count--;
+    DEBUG(1, ("Failed to run a check_online test.\n"));
+
+    talloc_free(be_req);
+
+    if (ctx->check_online_ref_count == 0) {
+        reset_fo(ctx);
+        be_reset_offline(ctx);
+    }
+
+    return;
+}
+
 static void init_timeout(struct tevent_context *ev,
                          struct tevent_timer *te,
                          struct timeval t, void *ptr)
@@ -1228,7 +1335,11 @@ int main(int argc, const char *argv[])
 static int data_provider_res_init(DBusMessage *message,
                                   struct sbus_connection *conn)
 {
+    struct be_ctx *be_ctx;
+    be_ctx = talloc_get_type(sbus_conn_get_private_data(conn), struct be_ctx);
+
     resolv_reread_configuration();
+    check_if_online(be_ctx);
 
     return monitor_common_res_init(message, conn);
 }
@@ -1247,6 +1358,6 @@ static int data_provider_reset_offline(DBusMessage *message,
 {
     struct be_ctx *be_ctx;
     be_ctx = talloc_get_type(sbus_conn_get_private_data(conn), struct be_ctx);
-    be_reset_offline(be_ctx);
+    check_if_online(be_ctx);
     return monitor_common_pong(message, conn);
 }
diff --git a/src/providers/data_provider_fo.c b/src/providers/data_provider_fo.c
index 8617cd23..0b4b2434 100644
--- a/src/providers/data_provider_fo.c
+++ b/src/providers/data_provider_fo.c
@@ -512,3 +512,8 @@ int be_fo_run_callbacks_at_next_request(struct be_ctx *ctx,
     return EOK;
 }
 
+void reset_fo(struct be_ctx *be_ctx)
+{
+    fo_reset_services(be_ctx->be_fo->fo_ctx);
+}
+
diff --git a/src/providers/dp_backend.h b/src/providers/dp_backend.h
index e11b3b6c..3d5e40ba 100644
--- a/src/providers/dp_backend.h
+++ b/src/providers/dp_backend.h
@@ -111,6 +111,8 @@ struct be_ctx {
 
     struct loaded_be loaded_be[BET_MAX];
     struct bet_info bet_info[BET_MAX];
+
+    size_t check_online_ref_count;
 };
 
 struct bet_ops {
@@ -193,4 +195,5 @@ void be_fo_try_next_server(struct be_ctx *ctx, const char *service_name);
 int be_fo_run_callbacks_at_next_request(struct be_ctx *ctx,
                                         const char *service_name);
 
+void reset_fo(struct be_ctx *be_ctx);
 #endif /* __DP_BACKEND_H___ */
diff --git a/src/providers/fail_over.c b/src/providers/fail_over.c
index 9b818d0c..858d6ba6 100644
--- a/src/providers/fail_over.c
+++ b/src/providers/fail_over.c
@@ -1329,3 +1329,20 @@ fo_get_server_hostent(struct fo_server *server)
     }
     return server->common->hostent;
 }
+
+void fo_reset_services(struct fo_ctx *fo_ctx)
+{
+    struct fo_service *service;
+    struct fo_server *server;
+
+    DLIST_FOR_EACH(service, fo_ctx->service_list) {
+        DLIST_FOR_EACH(server, service->server_list) {
+            fo_set_server_status(server, SERVER_NAME_NOT_RESOLVED);
+            fo_set_port_status(server, PORT_NEUTRAL);
+            if (server->srv_data != NULL) {
+                set_srv_data_status(server->srv_data, SRV_NOT_RESOLVED);
+            }
+        }
+    }
+}
+
diff --git a/src/providers/fail_over.h b/src/providers/fail_over.h
index f1a071da..dce02dc6 100644
--- a/src/providers/fail_over.h
+++ b/src/providers/fail_over.h
@@ -172,4 +172,5 @@ struct hostent *fo_get_server_hostent(struct fo_server *server);
 
 int fo_is_srv_lookup(struct fo_server *s);
 
+void fo_reset_services(struct fo_ctx *fo_ctx);
 #endif /* !__FAIL_OVER_H__ */
diff --git a/src/providers/proxy/proxy_init.c b/src/providers/proxy/proxy_init.c
index de1904c7..0993feed 100644
--- a/src/providers/proxy/proxy_init.c
+++ b/src/providers/proxy/proxy_init.c
@@ -54,7 +54,8 @@ static void proxy_auth_shutdown(struct be_req *req)
 
 struct bet_ops proxy_id_ops = {
     .handler = proxy_get_account_info,
-    .finalize = proxy_shutdown
+    .finalize = proxy_shutdown,
+    .check_online = NULL
 };
 
 struct bet_ops proxy_auth_ops = {