diff --git a/servers/lloadd/backend.c b/servers/lloadd/backend.c
index 29ef49a2fd..72fd989ba9 100644
--- a/servers/lloadd/backend.c
+++ b/servers/lloadd/backend.c
@@ -311,8 +311,8 @@ backend_retry( LloadBackend *b )
             Debug( LDAP_DEBUG_CONNS, "backend_retry: "
                     "scheduling re-connection straight away\n" );
             b->b_opening++;
-            rc = ldap_pvt_thread_pool_submit(
-                    &connection_pool, backend_connect_task, b );
+            rc = ldap_pvt_thread_pool_submit2(
+                    &connection_pool, backend_connect_task, b, &b->b_cookie );
             if ( rc ) {
                 ldap_pvt_thread_mutex_unlock( &b->b_mutex );
                 backend_connect( -1, 0, b );
@@ -345,6 +345,10 @@ backend_connect( evutil_socket_t s, short what, void *arg )
             (what & EV_TIMEOUT) ? "retry timeout finished, " : "",
             b->b_host );
 
+    if ( b->b_cookie ) {
+        b->b_cookie = NULL;
+    }
+
 #ifdef LDAP_PF_LOCAL
     if ( b->b_proto == LDAP_PROTO_IPC ) {
         struct sockaddr_un addr;
@@ -451,6 +455,7 @@ backend_reset( LloadBackend *b )
         evutil_closesocket( pending->fd );
         LDAP_LIST_REMOVE( pending, next );
         ch_free( pending );
+        b->b_opening--;
     }
     while ( !LDAP_CIRCLEQ_EMPTY( &b->b_preparing ) ) {
         LloadConnection *c = LDAP_CIRCLEQ_FIRST( &b->b_preparing );
@@ -493,6 +498,23 @@ backend_reset( LloadBackend *b )
         b->b_dns_req = NULL;
         b->b_opening--;
     }
+    if ( b->b_cookie ) {
+        int rc;
+        rc = ldap_pvt_thread_pool_retract( b->b_cookie );
+        assert( rc == 1 );
+        b->b_cookie = NULL;
+        b->b_opening--;
+    }
+    if ( b->b_retry_event &&
+            event_pending( b->b_retry_event, EV_TIMEOUT, NULL ) ) {
+        assert( b->b_failed );
+        event_del( b->b_retry_event );
+        b->b_opening--;
+    }
+    assert( b->b_opening == 0 );
+    assert( b->b_active == 0 );
+    assert( b->b_bindavail == 0 );
+    b->b_failed = 0;
 }
 
 void
@@ -529,8 +551,11 @@ lload_backend_destroy( LloadBackend *b )
 #endif /* BALANCER_MODULE */
     ldap_pvt_thread_mutex_destroy( &b->b_mutex );
 
-    event_del( b->b_retry_event );
-    event_free( b->b_retry_event );
+    if ( b->b_retry_event ) {
+        event_del( b->b_retry_event );
+        event_free( b->b_retry_event );
+        b->b_retry_event = NULL;
+    }
 
     ch_free( b->b_host );
     ch_free( b->b_uri.bv_val );
diff --git a/servers/lloadd/daemon.c b/servers/lloadd/daemon.c
index 74701351f3..a4f70f826c 100644
--- a/servers/lloadd/daemon.c
+++ b/servers/lloadd/daemon.c
@@ -1419,12 +1419,6 @@ daemon_wakeup_cb( evutil_socket_t sig, short what, void *arg )
 LloadChange lload_change = { .type = LLOAD_UNDEFINED };
 
 #ifdef BALANCER_MODULE
-int
-backend_connect_cb( ldap_pvt_thread_start_t *start, void *startarg, void *arg )
-{
-    return arg == NULL || arg == startarg;
-}
-
 int
 backend_conn_cb( ldap_pvt_thread_start_t *start, void *startarg, void *arg )
 {
@@ -1478,9 +1472,12 @@ lload_handle_backend_invalidation( LloadChange *change )
                 &connection_pool, handle_pdus, backend_conn_cb, b );
         ldap_pvt_thread_pool_walk(
                 &connection_pool, upstream_bind, backend_conn_cb, b );
-        /* Check there are no pending connection tasks either */
-        ldap_pvt_thread_pool_walk(
-                &connection_pool, backend_connect_task, backend_connect_cb, b );
+        /* Drop the connection task if it's queued */
+        if ( b->b_cookie ) {
+            int rc = ldap_pvt_thread_pool_retract( b->b_cookie );
+            assert( rc == 1 );
+            b->b_opening--;
+        }
         lload_backend_destroy( b );
         return;
     }
diff --git a/servers/lloadd/lload.h b/servers/lloadd/lload.h
index 56b6dad62d..d60010061b 100644
--- a/servers/lloadd/lload.h
+++ b/servers/lloadd/lload.h
@@ -213,6 +213,7 @@ struct LloadBackend {
 #endif /* BALANCER_MODULE */
 
     struct evdns_getaddrinfo_request *b_dns_req;
+    void *b_cookie;
 
     LDAP_CIRCLEQ_ENTRY(LloadBackend) b_next;
 };