Introduce a new connection status - gentle shutdown

servers/lloadd/client.c

@@ -227,8 +227,7 @@ handle_one_request( LloadConnection *c )
         case LDAP_REQ_ABANDON:
             /* We can't send a response to abandon requests even if a bind is
              * currently in progress */
-            handler = request_abandon;
-            break;
+            return request_abandon( c, op );
         case LDAP_REQ_EXTENDED:
             handler = request_extended;
             break;
@@ -241,6 +240,11 @@ handle_one_request( LloadConnection *c )
             break;
     }
 
+    if ( c->c_state == LLOAD_C_CLOSING ) {
+        return operation_send_reject_locked(
+                op, LDAP_UNAVAILABLE, "connection is shutting down", 0 );
+    }
+
     return handler( c, op );
 }
 
@@ -510,7 +514,7 @@ client_destroy( LloadConnection *c )
         event_del( write_event );
     }
 
-    if ( state != LLOAD_C_CLOSING ) {
+    if ( state != LLOAD_C_DYING ) {
         ldap_pvt_thread_mutex_lock( &clients_mutex );
         LDAP_CIRCLEQ_REMOVE( &clients, c, c_next );
         ldap_pvt_thread_mutex_unlock( &clients_mutex );
@@ -537,7 +541,7 @@ client_destroy( LloadConnection *c )
      */
     assert( c->c_refcnt >= 0 );
     if ( c->c_refcnt ) {
-        c->c_state = LLOAD_C_CLOSING;
+        c->c_state = LLOAD_C_DYING;
         Debug( LDAP_DEBUG_CONNS, "client_destroy: "
                 "connid=%lu aborting with refcnt=%d\n",
                 c->c_connid, c->c_refcnt );

servers/lloadd/connection.c

@@ -333,6 +333,38 @@ connection_destroy( LloadConnection *c )
     listeners_reactivate();
 }
 
+/*
+ * Expected to be run from lload_unpause_server, so there are no other threads
+ * running.
+ */
+void
+lload_connection_close( LloadConnection *c )
+{
+    TAvlnode *node;
+
+    /* We lock so we can use CONNECTION_UNLOCK_OR_DESTROY to drop the
+     * connection if we can */
+    CONNECTION_LOCK(c);
+
+    /* The first thing we do is make sure we don't get new Operations in */
+    c->c_state = LLOAD_C_CLOSING;
+
+    for ( node = tavl_end( c->c_ops, TAVL_DIR_LEFT ); node;
+            node = tavl_next( node, TAVL_DIR_RIGHT ) ) {
+        LloadOperation *op = node->avl_data;
+
+        if ( op->o_client_msgid == 0 ) {
+            if ( op->o_client == c ) {
+                operation_destroy_from_client( op );
+            } else {
+                assert( op->o_upstream == c );
+                operation_destroy_from_upstream( op );
+            }
+        }
+    }
+    CONNECTION_UNLOCK_OR_DESTROY(c);
+}
+
 LloadConnection *
 lload_connection_init( ber_socket_t s, const char *peername, int flags )
 {

servers/lloadd/lload.h

@@ -195,6 +195,7 @@ enum sc_state {
     LLOAD_C_CLOSING,     /* closing */
     LLOAD_C_ACTIVE,      /* exclusive operation (tls setup, ...) in progress */
     LLOAD_C_BINDING,     /* binding */
+    LLOAD_C_DYING, /* part-processed dead but someone still holds a reference */
 };
 enum sc_type {
     LLOAD_C_OPEN = 0,  /* regular connection */
@@ -246,6 +247,10 @@ struct LloadConnection {
 #define CONNECTION_UNLOCK_OR_DESTROY(c) \
     do { \
         assert( (c)->c_refcnt >= 0 ); \
+        if ( (c)->c_state == LLOAD_C_CLOSING && !( c )->c_ops ) { \
+            (c)->c_refcnt -= (c)->c_live; \
+            (c)->c_live = 0; \
+        } \
         if ( !( c )->c_refcnt ) { \
             Debug( LDAP_DEBUG_TRACE, "%s: destroying connection connid=%lu\n", \
                     __func__, (c)->c_connid ); \

servers/lloadd/proto-lload.h

@@ -87,6 +87,7 @@ LDAP_SLAPD_F (int) lload_back_init_cf( BackendInfo *bi );
 LDAP_SLAPD_V (ldap_pvt_thread_mutex_t) clients_mutex;
 LDAP_SLAPD_F (void) connection_write_cb( evutil_socket_t s, short what, void *arg );
 LDAP_SLAPD_F (void) connection_read_cb( evutil_socket_t s, short what, void *arg );
+LDAP_SLAPD_F (void) lload_connection_close( LloadConnection *c );
 LDAP_SLAPD_F (LloadConnection *) lload_connection_init( ber_socket_t s, const char *peername, int use_tls );
 LDAP_SLAPD_F (void) connection_destroy( LloadConnection *c );
 

servers/lloadd/upstream.c

@@ -101,7 +101,7 @@ forward_final_response(
 static int
 handle_unsolicited( LloadConnection *c, BerElement *ber )
 {
-    if ( c->c_state == LLOAD_C_READY ) {
+    if ( c->c_state != LLOAD_C_PREPARING ) {
         c->c_state = LLOAD_C_CLOSING;
     }
 
@@ -799,7 +799,7 @@ upstream_destroy( LloadConnection *c )
     }
 
     /* Remove from the backend on first pass */
-    if ( state != LLOAD_C_CLOSING ) {
+    if ( state != LLOAD_C_DYING ) {
         ldap_pvt_thread_mutex_lock( &b->b_mutex );
         if ( c->c_type == LLOAD_C_PREPARING ) {
             LDAP_CIRCLEQ_REMOVE( &b->b_preparing, c, c_next );
@@ -854,7 +854,7 @@ upstream_destroy( LloadConnection *c )
      */
     assert( c->c_refcnt >= 0 );
     if ( c->c_refcnt ) {
-        c->c_state = LLOAD_C_CLOSING;
+        c->c_state = LLOAD_C_DYING;
         Debug( LDAP_DEBUG_CONNS, "upstream_destroy: "
                 "connid=%lu aborting with refcnt=%d\n",
                 c->c_connid, c->c_refcnt );