Keep a list of active clients for shutdown purposes.

Potentially for timeout detection purposes in the future.

servers/lloadd/client.c

@@ -24,6 +24,10 @@
 #include "lutil.h"
 #include "slap.h"
 
+slap_c_head clients = LDAP_CIRCLEQ_HEAD_INITIALIZER( clients );
+
+ldap_pvt_thread_mutex_t clients_mutex;
+
 typedef int (*RequestHandler)( Connection *c, Operation *op );
 
 static void
@@ -178,7 +182,6 @@ handle_one_request( Connection *c )
         case LDAP_REQ_UNBIND:
             /* There is never a response for this operation */
             operation_destroy_from_client( op );
-            c->c_state = SLAP_C_CLOSING;
             CLIENT_DESTROY(c);
             return -1;
         case LDAP_REQ_BIND:
@@ -283,6 +286,13 @@ client_init(
     c->c_write_event = event;
 
     c->c_private = listener;
+
+    /* There should be no lock inversion yet since no other thread could
+     * approach it from clients side */
+    ldap_pvt_thread_mutex_lock( &clients_mutex );
+    LDAP_CIRCLEQ_INSERT_TAIL( &clients, c, c_next );
+    ldap_pvt_thread_mutex_unlock( &clients_mutex );
+
     CONNECTION_UNLOCK(c);
 
     return c;
@@ -303,10 +313,16 @@ fail:
 void
 client_destroy( Connection *c )
 {
+    enum sc_state state;
+
     Debug( LDAP_DEBUG_CONNS, "client_destroy: "
             "destroying client %lu\n",
             c->c_connid );
 
+    assert( c->c_state != SLAP_C_INVALID );
+    state = c->c_state;
+    c->c_state = SLAP_C_INVALID;
+
     if ( c->c_read_event ) {
         event_free( c->c_read_event );
         c->c_read_event = NULL;
@@ -317,11 +333,12 @@ client_destroy( Connection *c )
         c->c_write_event = NULL;
     }
 
-    c->c_state = SLAP_C_INVALID;
-    /* FIXME: we drop c_mutex in client_reset, operation_destroy_from_upstream
-     * might copy op->o_client and bump c_refcnt, it is then responsible to
-     * call destroy_client again, does that mean that we can be triggered for
-     * recursion over all connections? */
+    if ( state != SLAP_C_CLOSING ) {
+        ldap_pvt_thread_mutex_lock( &clients_mutex );
+        LDAP_CIRCLEQ_REMOVE( &clients, c, c_next );
+        ldap_pvt_thread_mutex_unlock( &clients_mutex );
+    }
+
     client_reset( c );
 
     /*
@@ -338,5 +355,27 @@ client_destroy( Connection *c )
         CONNECTION_UNLOCK(c);
         return;
     }
+
     connection_destroy( c );
 }
+
+void
+clients_destroy( void )
+{
+    ldap_pvt_thread_mutex_lock( &clients_mutex );
+    while ( !LDAP_CIRCLEQ_EMPTY( &clients ) ) {
+        Connection *c = LDAP_CIRCLEQ_FIRST( &clients );
+
+        ldap_pvt_thread_mutex_unlock( &clients_mutex );
+        CONNECTION_LOCK(c);
+        /* We have shut down all processing, a dying connection connection
+         * should have been reclaimed by now! */
+        assert( c->c_live );
+        /* Upstream connections have already been destroyed, there should be no
+         * ops left */
+        assert( !c->c_ops );
+        CLIENT_DESTROY(c);
+        ldap_pvt_thread_mutex_lock( &clients_mutex );
+    }
+    ldap_pvt_thread_mutex_unlock( &clients_mutex );
+}

servers/lloadd/daemon.c

@@ -1358,6 +1358,7 @@ slapd_daemon( struct event_base *daemon_base )
     }
     ldap_pvt_thread_pool_close( &connection_pool, 1 );
     backends_destroy();
+    clients_destroy();
     evdns_base_free( dnsbase, 0 );
 
     ch_free( daemon_tid );

servers/lloadd/init.c

@@ -99,6 +99,7 @@ slap_init( int mode, const char *name )
             LDAP_STAILQ_INIT( &slapd_rq.run_list );
 
             ldap_pvt_thread_mutex_init( &backend_mutex );
+            ldap_pvt_thread_mutex_init( &clients_mutex );
 
             break;
 

servers/lloadd/proto-slap.h

@@ -75,6 +75,7 @@ LDAP_SLAPD_F (int) handle_one_request( Connection *c );
 LDAP_SLAPD_F (Connection *) client_init( ber_socket_t s, Listener *url, const char *peername, struct event_base *base, int use_tls );
 LDAP_SLAPD_F (void) client_write_cb( evutil_socket_t s, short what, void *arg );
 LDAP_SLAPD_F (void) client_destroy( Connection *c );
+LDAP_SLAPD_F (void) clients_destroy( void );
 
 /*
  * config.c
@@ -89,6 +90,7 @@ LDAP_SLAPD_F (void) bindconf_free( slap_bindconf *bc );
 /*
  * connection.c
  */
+LDAP_SLAPD_V (ldap_pvt_thread_mutex_t) clients_mutex;
 LDAP_SLAPD_F (Connection *) connection_init( ber_socket_t s, const char *peername, int use_tls );
 LDAP_SLAPD_F (void) connection_destroy( Connection *c );
 

servers/lloadd/slap.h

@@ -122,8 +122,10 @@ extern int slap_inet4or6;
 #endif
 
 typedef LDAP_CIRCLEQ_HEAD(BeSt, Backend) slap_b_head;
+typedef LDAP_CIRCLEQ_HEAD(ClientSt, Connection) slap_c_head;
 
 LDAP_SLAPD_V (slap_b_head) backend;
+LDAP_SLAPD_V (slap_c_head) clients;
 LDAP_SLAPD_V (ldap_pvt_thread_mutex_t) backend_mutex;
 LDAP_SLAPD_V (Backend *) current_backend;
 
@@ -394,8 +396,12 @@ struct Connection {
     long c_n_ops_executing; /* num of ops currently executing */
     long c_n_ops_completed; /* num of ops completed */
 
-    /* Upstream: Protected by its backend's mutex */
-    LDAP_CIRCLEQ_ENTRY( Connection ) c_next;
+    /*
+     * Protected by the CIRCLEQ mutex:
+     * - Client: clients_mutex
+     * - Upstream: b->b_mutex
+     */
+    LDAP_CIRCLEQ_ENTRY(Connection) c_next;
 
     void *c_private;
 };