TLS stream/DoH: implement TLS client session resumption

This commit extends TLS stream code and DoH code with TLS client
session resumption support implemented on top of the TLS client
session cache.

(cherry picked from commit 90bc13a5d5)

bin/dig/dighost.c

@@ -3028,7 +3028,7 @@ start_tcp(dig_query_t *query) {
 			isc_nm_httpconnect(netmgr, &localaddr, &query->sockaddr,
 					   uri, !query->lookup->https_get,
 					   tcp_connected, connectquery, tlsctx,
-					   local_timeout, 0);
+					   sess_cache, 0, local_timeout);
 #endif
 		} else {
 			isc_nm_tcpdnsconnect(netmgr, &localaddr,

bin/tests/test_client.c

@@ -428,7 +428,7 @@ run(void) {
 		}
 		isc_nm_httpconnect(netmgr, &sockaddr_local, &sockaddr_remote,
 				   req_url, is_post, connect_cb, NULL, tls_ctx,
-				   timeout, 0);
+				   NULL, timeout, 0);
 	} break;
 #endif
 	default:

lib/isc/include/isc/netmgr.h

@@ -559,13 +559,15 @@ isc_nm_listentls(isc_nm_t *mgr, isc_sockaddr_t *iface,
 void
 isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 		  isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx,
+		  isc_tlsctx_client_session_cache_t *client_sess_cache,
 		  unsigned int timeout, size_t extrahandlesize);
 
 void
 isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 		   const char *uri, bool POST, isc_nm_cb_t cb, void *cbarg,
-		   isc_tlsctx_t *ctx, unsigned int timeout,
-		   size_t extrahandlesize);
+		   isc_tlsctx_t			*ctx,
+		   isc_tlsctx_client_session_cache_t *client_sess_cache,
+		   unsigned int timeout, size_t extrahandlesize);
 
 isc_result_t
 isc_nm_listenhttp(isc_nm_t *mgr, isc_sockaddr_t *iface, int backlog,

lib/isc/netmgr/http.c

@@ -1424,8 +1424,9 @@ error:
 void
 isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 		   const char *uri, bool post, isc_nm_cb_t cb, void *cbarg,
-		   isc_tlsctx_t *tlsctx, unsigned int timeout,
-		   size_t extrahandlesize) {
+		   isc_tlsctx_t *tlsctx,
+		   isc_tlsctx_client_session_cache_t *client_sess_cache,
+		   unsigned int timeout, size_t extrahandlesize) {
 	isc_sockaddr_t local_interface;
 	isc_nmsocket_t *sock = NULL;
 
@@ -1487,7 +1488,7 @@ isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 
 	if (tlsctx != NULL) {
 		isc_nm_tlsconnect(mgr, local, peer, transport_connect_cb, sock,
-				  tlsctx, timeout, 0);
+				  tlsctx, client_sess_cache, timeout, 0);
 	} else {
 		isc_nm_tcpconnect(mgr, local, peer, transport_connect_cb, sock,
 				  timeout, 0);

lib/isc/netmgr/netmgr-int.h

@@ -971,6 +971,8 @@ struct isc_nmsocket {
 		isc_tlsctx_t **listener_tls_ctx; /*%< A context reference per
 						    worker */
 		size_t n_listener_tls_ctx;
+		isc_tlsctx_client_session_cache_t *client_sess_cache;
+		bool client_session_saved;
 		isc_nmsocket_t *tlslistener;
 		isc_nmsocket_t *tlssocket;
 		atomic_bool result_updated;
@@ -2175,3 +2177,6 @@ isc__nmsocket_writetimeout_cb(void *data, isc_result_t eresult);
 		isc_error_fatal(__FILE__, __LINE__, "%s failed: %s\n", #func, \
 				uv_strerror(ret));                            \
 	}
+
+void
+isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls);

lib/isc/netmgr/netmgr.c

@@ -3729,6 +3729,27 @@ isc_nm_verify_tls_peer_result_string(const isc_nmhandle_t *handle) {
 	return (NULL);
 }
 
+void
+isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls) {
+	const int log_level = ISC_LOG_DEBUG(1);
+	char client_sabuf[ISC_SOCKADDR_FORMATSIZE];
+	char local_sabuf[ISC_SOCKADDR_FORMATSIZE];
+
+	REQUIRE(tls != NULL);
+
+	if (!isc_log_wouldlog(isc_lctx, log_level)) {
+		return;
+	};
+
+	isc_sockaddr_format(&sock->peer, client_sabuf, sizeof(client_sabuf));
+	isc_sockaddr_format(&sock->iface, local_sabuf, sizeof(local_sabuf));
+	isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR,
+		      log_level, "TLS %s session %s for %s on %s",
+		      SSL_is_server(tls) ? "server" : "client",
+		      SSL_session_reused(tls) ? "resumed" : "created",
+		      client_sabuf, local_sabuf);
+}
+
 #ifdef NETMGR_TRACE
 /*
  * Dump all active sockets in netmgr. We output to stderr

lib/isc/netmgr/tlsstream.c

@@ -86,6 +86,12 @@ tls_cleanup_listener_tlsctx(isc_nmsocket_t *listener);
 static isc_tlsctx_t *
 tls_get_listener_tlsctx(isc_nmsocket_t *listener, const int tid);
 
+static void
+tls_keep_client_tls_session(isc_nmsocket_t *sock);
+
+static void
+tls_try_shutdown(isc_tls_t *tls, const bool quite);
+
 /*
  * The socket is closing, outerhandle has been detached, listener is
  * inactive, or the netmgr is closing: any operation on it should abort
@@ -128,6 +134,10 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 	tlssock = send_req->tlssock;
 	send_req->tlssock = NULL;
 
+	if (finish) {
+		tls_try_shutdown(tlssock->tlsstream.tls, true);
+	}
+
 	if (send_req->cb != NULL) {
 		INSIST(VALID_NMHANDLE(tlssock->statichandle));
 		send_req->cb(send_req->handle, eresult, send_req->cbarg);
@@ -241,10 +251,9 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		return (0);
 	}
 
-	if (finish && (SSL_get_shutdown(sock->tlsstream.tls) &
-		       SSL_SENT_SHUTDOWN) != SSL_SENT_SHUTDOWN)
-	{
-		(void)SSL_shutdown(sock->tlsstream.tls);
+	if (finish) {
+		tls_try_shutdown(sock->tlsstream.tls, false);
+		tls_keep_client_tls_session(sock);
 	}
 
 	pending = BIO_pending(sock->tlsstream.bio_out);
@@ -293,22 +302,21 @@ tls_process_outgoing(isc_nmsocket_t *sock, bool finish,
 		     isc__nm_uvreq_t *send_data) {
 	int pending;
 
+	bool received_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
+				   SSL_RECEIVED_SHUTDOWN) != 0);
+	bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
+			       SSL_SENT_SHUTDOWN) != 0);
+
+	if (received_shutdown && !sent_shutdown) {
+		finish = true;
+	}
+
 	/* Data from TLS to network */
 	if (send_data != NULL) {
 		pending = tls_send_outgoing(sock, finish, send_data->handle,
 					    send_data->cb.send,
 					    send_data->cbarg);
 	} else {
-		bool received_shutdown =
-			((SSL_get_shutdown(sock->tlsstream.tls) &
-			  SSL_RECEIVED_SHUTDOWN) != 0);
-		bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) &
-				       SSL_SENT_SHUTDOWN) != 0);
-
-		if (received_shutdown && !sent_shutdown) {
-			finish = true;
-			(void)SSL_shutdown(sock->tlsstream.tls);
-		}
 		pending = tls_send_outgoing(sock, finish, NULL, NULL, NULL);
 	}
 
@@ -331,6 +339,7 @@ tls_try_handshake(isc_nmsocket_t *sock) {
 		isc_result_t result = ISC_R_SUCCESS;
 		INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 1);
 		INSIST(sock->statichandle == NULL);
+		isc__nmsocket_log_tls_session_reuse(sock, sock->tlsstream.tls);
 		tlshandle = isc__nmhandle_get(sock, &sock->peer, &sock->iface);
 		if (sock->tlsstream.server) {
 			sock->listener->accept_cb(tlshandle, result,
@@ -889,7 +898,8 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg);
 
 void
 isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
-		  isc_nm_cb_t cb, void *cbarg, SSL_CTX *ctx,
+		  isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx,
+		  isc_tlsctx_client_session_cache_t *client_sess_cache,
 		  unsigned int timeout, size_t extrahandlesize) {
 	isc_nmsocket_t *nsock = NULL;
 #if defined(NETMGR_TRACE) && defined(NETMGR_TRACE_VERBOSE)
@@ -907,6 +917,13 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer,
 	nsock->connect_cbarg = cbarg;
 	nsock->connect_timeout = timeout;
 	isc_tlsctx_attach(ctx, &nsock->tlsstream.ctx);
+	atomic_init(&nsock->client, true);
+	if (client_sess_cache != NULL) {
+		INSIST(isc_tlsctx_client_session_cache_getctx(
+			       client_sess_cache) == ctx);
+		isc_tlsctx_client_session_cache_attach(
+			client_sess_cache, &nsock->tlsstream.client_sess_cache);
+	}
 
 	isc_nm_tcpconnect(mgr, local, peer, tcp_connected, nsock,
 			  nsock->connect_timeout, 0);
@@ -949,6 +966,12 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	isc_nmhandle_attach(handle, &tlssock->outerhandle);
 	atomic_store(&tlssock->active, true);
 
+	if (tlssock->tlsstream.client_sess_cache != NULL) {
+		isc_tlsctx_client_session_cache_reuse_sockaddr(
+			tlssock->tlsstream.client_sess_cache, &tlssock->peer,
+			tlssock->tlsstream.tls);
+	}
+
 	/*
 	 * Hold a reference to tlssock in the TCP socket: it will
 	 * detached in isc__nm_tls_cleanup_data().
@@ -1026,15 +1049,26 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) {
 	} else if (sock->type == isc_nm_tlslistener) {
 		tls_cleanup_listener_tlsctx(sock);
 	} else if (sock->type == isc_nm_tlssocket) {
-		if (sock->tlsstream.ctx != NULL) {
-			isc_tlsctx_free(&sock->tlsstream.ctx);
-		}
 		if (sock->tlsstream.tls != NULL) {
+			/*
+			 * Let's shutdown the TLS session properly so that the
+			 * session will remain resumable, if required.
+			 */
+			tls_try_shutdown(sock->tlsstream.tls, true);
+			tls_keep_client_tls_session(sock);
 			isc_tls_free(&sock->tlsstream.tls);
 			/* These are destroyed when we free SSL */
 			sock->tlsstream.bio_out = NULL;
 			sock->tlsstream.bio_in = NULL;
 		}
+		if (sock->tlsstream.ctx != NULL) {
+			isc_tlsctx_free(&sock->tlsstream.ctx);
+		}
+		if (sock->tlsstream.client_sess_cache != NULL) {
+			INSIST(atomic_load(&sock->client));
+			isc_tlsctx_client_session_cache_detach(
+				&sock->tlsstream.client_sess_cache);
+		}
 	} else if (sock->type == isc_nm_tcpsocket &&
 		   sock->tlsstream.tlssocket != NULL) {
 		/*
@@ -1164,3 +1198,30 @@ isc__nm_async_tls_set_tlsctx(isc_nmsocket_t *listener, isc_tlsctx_t *tlsctx,
 	isc_tlsctx_free(&listener->tlsstream.listener_tls_ctx[tid]);
 	isc_tlsctx_attach(tlsctx, &listener->tlsstream.listener_tls_ctx[tid]);
 }
+
+static void
+tls_keep_client_tls_session(isc_nmsocket_t *sock) {
+	/*
+	 * Ensure that the isc_tls_t is being accessed from
+	 * within the worker thread the socket is bound to.
+	 */
+	REQUIRE(sock->tid == isc_nm_tid());
+	if (sock->tlsstream.client_sess_cache != NULL &&
+	    sock->tlsstream.client_session_saved == false)
+	{
+		INSIST(atomic_load(&sock->client));
+		isc_tlsctx_client_session_cache_keep_sockaddr(
+			sock->tlsstream.client_sess_cache, &sock->peer,
+			sock->tlsstream.tls);
+		sock->tlsstream.client_session_saved = true;
+	}
+}
+
+static void
+tls_try_shutdown(isc_tls_t *tls, const bool force) {
+	if (force) {
+		(void)SSL_set_shutdown(tls, SSL_SENT_SHUTDOWN);
+	} else if ((SSL_get_shutdown(tls) & SSL_SENT_SHUTDOWN) == 0) {
+		(void)SSL_shutdown(tls);
+	}
+}

tests/isc/doh_test.c

@@ -79,6 +79,7 @@ static atomic_bool slowdown = false;
 static atomic_bool use_TLS = false;
 static isc_tlsctx_t *server_tlsctx = NULL;
 static isc_tlsctx_t *client_tlsctx = NULL;
+static isc_tlsctx_client_session_cache_t *client_sess_cache = NULL;
 
 static isc_quota_t listener_quota;
 static atomic_bool check_listener_quota = false;
@@ -176,7 +177,8 @@ connect_send_request(isc_nm_t *mgr, const char *uri, bool post,
 	}
 
 	isc_nm_httpconnect(mgr, NULL, &tcp_listen_addr, uri, post,
-			   connect_send_cb, data, ctx, timeout, 0);
+			   connect_send_cb, data, ctx, client_sess_cache,
+			   timeout, 0);
 }
 
 static int
@@ -321,6 +323,9 @@ setup_test(void **state) {
 	client_tlsctx = NULL;
 	isc_tlsctx_createclient(&client_tlsctx);
 	isc_tlsctx_enable_http2client_alpn(client_tlsctx);
+	client_sess_cache = isc_tlsctx_client_session_cache_new(
+		mctx, client_tlsctx,
+		ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE);
 
 	isc_quota_init(&listener_quota, 0);
 	atomic_store(&check_listener_quota, false);
@@ -350,6 +355,8 @@ teardown_test(void **state) {
 		isc_tlsctx_free(&client_tlsctx);
 	}
 
+	isc_tlsctx_client_session_cache_detach(&client_sess_cache);
+
 	isc_quota_destroy(&listener_quota);
 
 	isc_nm_http_endpoints_detach(&endpoints);
@@ -644,7 +651,7 @@ doh_timeout_recovery(void **state) {
 			ISC_NM_HTTP_DEFAULT_PATH);
 	isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url,
 			   atomic_load(&POST), timeout_request_cb, NULL, ctx,
-			   T_SOFT, 0);
+			   client_sess_cache, T_SOFT, 0);
 
 	/*
 	 * Sleep until sends reaches 5.
@@ -932,7 +939,7 @@ doh_recv_two(void **state) {
 
 	isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url,
 			   atomic_load(&POST), doh_connect_send_two_requests_cb,
-			   NULL, ctx, 5000, 0);
+			   NULL, ctx, client_sess_cache, 5000, 0);
 
 	while (atomic_load(&nsends) > 0) {
 		if (atomic_load(&was_error)) {

tests/isc/netmgr_test.c

@@ -28,6 +28,7 @@
 #include <isc/refcount.h>
 #include <isc/sockaddr.h>
 #include <isc/thread.h>
+#include <isc/tls.h>
 #include <isc/util.h>
 
 #include "uv_wrap.h"
@@ -62,6 +63,7 @@ static isc_sockaddr_t tcp_listen_addr;
 static isc_sockaddr_t tcp_connect_addr;
 static isc_tlsctx_t *tcp_listen_tlsctx = NULL;
 static isc_tlsctx_t *tcp_connect_tlsctx = NULL;
+static isc_tlsctx_client_session_cache_t *tcp_tlsctx_client_sess_cache = NULL;
 
 static uint64_t send_magic = 0;
 static uint64_t stop_magic = 0;
@@ -333,6 +335,10 @@ setup_test(void **state __attribute__((unused))) {
 
 	isc_tlsctx_enable_dot_client_alpn(tcp_connect_tlsctx);
 
+	tcp_tlsctx_client_sess_cache = isc_tlsctx_client_session_cache_new(
+		mctx, tcp_connect_tlsctx,
+		ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE);
+
 	return (0);
 }
 
@@ -361,6 +367,8 @@ teardown_test(void **state __attribute__((unused))) {
 	isc_refcount_destroy(&active_ssends);
 	isc_refcount_destroy(&active_sreads);
 
+	isc_tlsctx_client_session_cache_detach(&tcp_tlsctx_client_sess_cache);
+
 	return (0);
 }
 
@@ -1156,7 +1164,8 @@ stream_connect(isc_nm_cb_t cb, void *cbarg, unsigned int timeout,
 	if (stream_use_TLS) {
 		isc_nm_tlsconnect(connect_nm, &tcp_connect_addr,
 				  &tcp_listen_addr, cb, cbarg,
-				  tcp_connect_tlsctx, timeout, extrahandlesize);
+				  tcp_connect_tlsctx,
+				  tcp_tlsctx_client_sess_cache, timeout, 0);
 		return;
 	}
 #endif
@@ -2059,7 +2068,7 @@ static void
 tls_connect(isc_nm_t *nm) {
 	isc_nm_tlsconnect(nm, &tcp_connect_addr, &tcp_listen_addr,
 			  connect_connect_cb, NULL, tcp_connect_tlsctx,
-			  T_CONNECT, 0);
+			  tcp_tlsctx_client_sess_cache, T_CONNECT, 0);
 }
 
 ISC_RUN_TEST_IMPL(tls_noop) {