diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index a3fc6d203c..4fef5985b2 100644
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -711,6 +711,7 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	tlssock = isc_mem_get(handle->sock->mgr->mctx, sizeof(*tlssock));
 	isc__nmsocket_init(tlssock, handle->sock->mgr, isc_nm_tlssocket,
 			   &handle->sock->iface);
+	isc__nmsocket_attach(tlslistensock, &tlssock->server);
 
 	tid = isc_nm_tid();
 	/* We need to initialize SSL now to reference SSL_CTX properly */
@@ -945,6 +946,10 @@ tls_close_direct(isc_nmsocket_t *sock) {
 		isc__nmsocket_detach(&sock->listener);
 	}
 
+	if (sock->server != NULL) {
+		isc__nmsocket_detach(&sock->server);
+	}
+
 	/* Further cleanup performed in isc__nm_tls_cleanup_data() */
 	atomic_store(&sock->closed, true);
 	atomic_store(&sock->active, false);