From 0a4a76ff7aaeab30e5b619231321af6f9b18c21e Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Fri, 22 Apr 2022 15:59:11 +0300 Subject: [PATCH] TLS stream/DoH: implement TLS client session resumption This commit extends TLS stream code and DoH code with TLS client session resumption support implemented on top of the TLS client session cache. (cherry picked from commit 90bc13a5d52a7997a5cb977d62489d034b344b73) --- bin/dig/dighost.c | 2 +- bin/tests/test_client.c | 2 +- lib/isc/include/isc/netmgr.h | 6 ++- lib/isc/netmgr/http.c | 7 +-- lib/isc/netmgr/netmgr-int.h | 5 ++ lib/isc/netmgr/netmgr.c | 21 ++++++++ lib/isc/netmgr/tlsstream.c | 97 +++++++++++++++++++++++++++++------- tests/isc/doh_test.c | 13 +++-- tests/isc/netmgr_test.c | 13 ++++- 9 files changed, 136 insertions(+), 30 deletions(-) diff --git a/bin/dig/dighost.c b/bin/dig/dighost.c index bb9e13dcdb..a4490de8ae 100644 --- a/bin/dig/dighost.c +++ b/bin/dig/dighost.c @@ -3028,7 +3028,7 @@ start_tcp(dig_query_t *query) { isc_nm_httpconnect(netmgr, &localaddr, &query->sockaddr, uri, !query->lookup->https_get, tcp_connected, connectquery, tlsctx, - local_timeout, 0); + sess_cache, 0, local_timeout); #endif } else { isc_nm_tcpdnsconnect(netmgr, &localaddr, diff --git a/bin/tests/test_client.c b/bin/tests/test_client.c index ee62a6dfb9..10b2103193 100644 --- a/bin/tests/test_client.c +++ b/bin/tests/test_client.c @@ -428,7 +428,7 @@ run(void) { } isc_nm_httpconnect(netmgr, &sockaddr_local, &sockaddr_remote, req_url, is_post, connect_cb, NULL, tls_ctx, - timeout, 0); + NULL, timeout, 0); } break; #endif default: diff --git a/lib/isc/include/isc/netmgr.h b/lib/isc/include/isc/netmgr.h index f2ead2688d..c21e832f33 100644 --- a/lib/isc/include/isc/netmgr.h +++ b/lib/isc/include/isc/netmgr.h @@ -559,13 +559,15 @@ isc_nm_listentls(isc_nm_t *mgr, isc_sockaddr_t *iface, void isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, unsigned int timeout, size_t extrahandlesize); void isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, const char *uri, bool POST, isc_nm_cb_t cb, void *cbarg, - isc_tlsctx_t *ctx, unsigned int timeout, - size_t extrahandlesize); + isc_tlsctx_t *ctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, + unsigned int timeout, size_t extrahandlesize); isc_result_t isc_nm_listenhttp(isc_nm_t *mgr, isc_sockaddr_t *iface, int backlog, diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index e66d37e8ac..b227947cca 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -1424,8 +1424,9 @@ error: void isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, const char *uri, bool post, isc_nm_cb_t cb, void *cbarg, - isc_tlsctx_t *tlsctx, unsigned int timeout, - size_t extrahandlesize) { + isc_tlsctx_t *tlsctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, + unsigned int timeout, size_t extrahandlesize) { isc_sockaddr_t local_interface; isc_nmsocket_t *sock = NULL; @@ -1487,7 +1488,7 @@ isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, if (tlsctx != NULL) { isc_nm_tlsconnect(mgr, local, peer, transport_connect_cb, sock, - tlsctx, timeout, 0); + tlsctx, client_sess_cache, timeout, 0); } else { isc_nm_tcpconnect(mgr, local, peer, transport_connect_cb, sock, timeout, 0); diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h index 55eff3ce24..68fd43f339 100644 --- a/lib/isc/netmgr/netmgr-int.h +++ b/lib/isc/netmgr/netmgr-int.h @@ -971,6 +971,8 @@ struct isc_nmsocket { isc_tlsctx_t **listener_tls_ctx; /*%< A context reference per worker */ size_t n_listener_tls_ctx; + isc_tlsctx_client_session_cache_t *client_sess_cache; + bool client_session_saved; isc_nmsocket_t *tlslistener; isc_nmsocket_t *tlssocket; atomic_bool result_updated; @@ -2175,3 +2177,6 @@ isc__nmsocket_writetimeout_cb(void *data, isc_result_t eresult); isc_error_fatal(__FILE__, __LINE__, "%s failed: %s\n", #func, \ uv_strerror(ret)); \ } + +void +isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls); diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c index 6494e99743..0661621b99 100644 --- a/lib/isc/netmgr/netmgr.c +++ b/lib/isc/netmgr/netmgr.c @@ -3729,6 +3729,27 @@ isc_nm_verify_tls_peer_result_string(const isc_nmhandle_t *handle) { return (NULL); } +void +isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls) { + const int log_level = ISC_LOG_DEBUG(1); + char client_sabuf[ISC_SOCKADDR_FORMATSIZE]; + char local_sabuf[ISC_SOCKADDR_FORMATSIZE]; + + REQUIRE(tls != NULL); + + if (!isc_log_wouldlog(isc_lctx, log_level)) { + return; + }; + + isc_sockaddr_format(&sock->peer, client_sabuf, sizeof(client_sabuf)); + isc_sockaddr_format(&sock->iface, local_sabuf, sizeof(local_sabuf)); + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR, + log_level, "TLS %s session %s for %s on %s", + SSL_is_server(tls) ? "server" : "client", + SSL_session_reused(tls) ? "resumed" : "created", + client_sabuf, local_sabuf); +} + #ifdef NETMGR_TRACE /* * Dump all active sockets in netmgr. We output to stderr diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c index 4fd909d228..ff6aff8829 100644 --- a/lib/isc/netmgr/tlsstream.c +++ b/lib/isc/netmgr/tlsstream.c @@ -86,6 +86,12 @@ tls_cleanup_listener_tlsctx(isc_nmsocket_t *listener); static isc_tlsctx_t * tls_get_listener_tlsctx(isc_nmsocket_t *listener, const int tid); +static void +tls_keep_client_tls_session(isc_nmsocket_t *sock); + +static void +tls_try_shutdown(isc_tls_t *tls, const bool quite); + /* * The socket is closing, outerhandle has been detached, listener is * inactive, or the netmgr is closing: any operation on it should abort @@ -128,6 +134,10 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) { tlssock = send_req->tlssock; send_req->tlssock = NULL; + if (finish) { + tls_try_shutdown(tlssock->tlsstream.tls, true); + } + if (send_req->cb != NULL) { INSIST(VALID_NMHANDLE(tlssock->statichandle)); send_req->cb(send_req->handle, eresult, send_req->cbarg); @@ -241,10 +251,9 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle, return (0); } - if (finish && (SSL_get_shutdown(sock->tlsstream.tls) & - SSL_SENT_SHUTDOWN) != SSL_SENT_SHUTDOWN) - { - (void)SSL_shutdown(sock->tlsstream.tls); + if (finish) { + tls_try_shutdown(sock->tlsstream.tls, false); + tls_keep_client_tls_session(sock); } pending = BIO_pending(sock->tlsstream.bio_out); @@ -293,22 +302,21 @@ tls_process_outgoing(isc_nmsocket_t *sock, bool finish, isc__nm_uvreq_t *send_data) { int pending; + bool received_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) & + SSL_RECEIVED_SHUTDOWN) != 0); + bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) & + SSL_SENT_SHUTDOWN) != 0); + + if (received_shutdown && !sent_shutdown) { + finish = true; + } + /* Data from TLS to network */ if (send_data != NULL) { pending = tls_send_outgoing(sock, finish, send_data->handle, send_data->cb.send, send_data->cbarg); } else { - bool received_shutdown = - ((SSL_get_shutdown(sock->tlsstream.tls) & - SSL_RECEIVED_SHUTDOWN) != 0); - bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) & - SSL_SENT_SHUTDOWN) != 0); - - if (received_shutdown && !sent_shutdown) { - finish = true; - (void)SSL_shutdown(sock->tlsstream.tls); - } pending = tls_send_outgoing(sock, finish, NULL, NULL, NULL); } @@ -331,6 +339,7 @@ tls_try_handshake(isc_nmsocket_t *sock) { isc_result_t result = ISC_R_SUCCESS; INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 1); INSIST(sock->statichandle == NULL); + isc__nmsocket_log_tls_session_reuse(sock, sock->tlsstream.tls); tlshandle = isc__nmhandle_get(sock, &sock->peer, &sock->iface); if (sock->tlsstream.server) { sock->listener->accept_cb(tlshandle, result, @@ -889,7 +898,8 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg); void isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, - isc_nm_cb_t cb, void *cbarg, SSL_CTX *ctx, + isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, unsigned int timeout, size_t extrahandlesize) { isc_nmsocket_t *nsock = NULL; #if defined(NETMGR_TRACE) && defined(NETMGR_TRACE_VERBOSE) @@ -907,6 +917,13 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, nsock->connect_cbarg = cbarg; nsock->connect_timeout = timeout; isc_tlsctx_attach(ctx, &nsock->tlsstream.ctx); + atomic_init(&nsock->client, true); + if (client_sess_cache != NULL) { + INSIST(isc_tlsctx_client_session_cache_getctx( + client_sess_cache) == ctx); + isc_tlsctx_client_session_cache_attach( + client_sess_cache, &nsock->tlsstream.client_sess_cache); + } isc_nm_tcpconnect(mgr, local, peer, tcp_connected, nsock, nsock->connect_timeout, 0); @@ -949,6 +966,12 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { isc_nmhandle_attach(handle, &tlssock->outerhandle); atomic_store(&tlssock->active, true); + if (tlssock->tlsstream.client_sess_cache != NULL) { + isc_tlsctx_client_session_cache_reuse_sockaddr( + tlssock->tlsstream.client_sess_cache, &tlssock->peer, + tlssock->tlsstream.tls); + } + /* * Hold a reference to tlssock in the TCP socket: it will * detached in isc__nm_tls_cleanup_data(). @@ -1026,15 +1049,26 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) { } else if (sock->type == isc_nm_tlslistener) { tls_cleanup_listener_tlsctx(sock); } else if (sock->type == isc_nm_tlssocket) { - if (sock->tlsstream.ctx != NULL) { - isc_tlsctx_free(&sock->tlsstream.ctx); - } if (sock->tlsstream.tls != NULL) { + /* + * Let's shutdown the TLS session properly so that the + * session will remain resumable, if required. + */ + tls_try_shutdown(sock->tlsstream.tls, true); + tls_keep_client_tls_session(sock); isc_tls_free(&sock->tlsstream.tls); /* These are destroyed when we free SSL */ sock->tlsstream.bio_out = NULL; sock->tlsstream.bio_in = NULL; } + if (sock->tlsstream.ctx != NULL) { + isc_tlsctx_free(&sock->tlsstream.ctx); + } + if (sock->tlsstream.client_sess_cache != NULL) { + INSIST(atomic_load(&sock->client)); + isc_tlsctx_client_session_cache_detach( + &sock->tlsstream.client_sess_cache); + } } else if (sock->type == isc_nm_tcpsocket && sock->tlsstream.tlssocket != NULL) { /* @@ -1164,3 +1198,30 @@ isc__nm_async_tls_set_tlsctx(isc_nmsocket_t *listener, isc_tlsctx_t *tlsctx, isc_tlsctx_free(&listener->tlsstream.listener_tls_ctx[tid]); isc_tlsctx_attach(tlsctx, &listener->tlsstream.listener_tls_ctx[tid]); } + +static void +tls_keep_client_tls_session(isc_nmsocket_t *sock) { + /* + * Ensure that the isc_tls_t is being accessed from + * within the worker thread the socket is bound to. + */ + REQUIRE(sock->tid == isc_nm_tid()); + if (sock->tlsstream.client_sess_cache != NULL && + sock->tlsstream.client_session_saved == false) + { + INSIST(atomic_load(&sock->client)); + isc_tlsctx_client_session_cache_keep_sockaddr( + sock->tlsstream.client_sess_cache, &sock->peer, + sock->tlsstream.tls); + sock->tlsstream.client_session_saved = true; + } +} + +static void +tls_try_shutdown(isc_tls_t *tls, const bool force) { + if (force) { + (void)SSL_set_shutdown(tls, SSL_SENT_SHUTDOWN); + } else if ((SSL_get_shutdown(tls) & SSL_SENT_SHUTDOWN) == 0) { + (void)SSL_shutdown(tls); + } +} diff --git a/tests/isc/doh_test.c b/tests/isc/doh_test.c index 6fa2d21588..ce3854cfca 100644 --- a/tests/isc/doh_test.c +++ b/tests/isc/doh_test.c @@ -79,6 +79,7 @@ static atomic_bool slowdown = false; static atomic_bool use_TLS = false; static isc_tlsctx_t *server_tlsctx = NULL; static isc_tlsctx_t *client_tlsctx = NULL; +static isc_tlsctx_client_session_cache_t *client_sess_cache = NULL; static isc_quota_t listener_quota; static atomic_bool check_listener_quota = false; @@ -176,7 +177,8 @@ connect_send_request(isc_nm_t *mgr, const char *uri, bool post, } isc_nm_httpconnect(mgr, NULL, &tcp_listen_addr, uri, post, - connect_send_cb, data, ctx, timeout, 0); + connect_send_cb, data, ctx, client_sess_cache, + timeout, 0); } static int @@ -321,6 +323,9 @@ setup_test(void **state) { client_tlsctx = NULL; isc_tlsctx_createclient(&client_tlsctx); isc_tlsctx_enable_http2client_alpn(client_tlsctx); + client_sess_cache = isc_tlsctx_client_session_cache_new( + mctx, client_tlsctx, + ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE); isc_quota_init(&listener_quota, 0); atomic_store(&check_listener_quota, false); @@ -350,6 +355,8 @@ teardown_test(void **state) { isc_tlsctx_free(&client_tlsctx); } + isc_tlsctx_client_session_cache_detach(&client_sess_cache); + isc_quota_destroy(&listener_quota); isc_nm_http_endpoints_detach(&endpoints); @@ -644,7 +651,7 @@ doh_timeout_recovery(void **state) { ISC_NM_HTTP_DEFAULT_PATH); isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url, atomic_load(&POST), timeout_request_cb, NULL, ctx, - T_SOFT, 0); + client_sess_cache, T_SOFT, 0); /* * Sleep until sends reaches 5. @@ -932,7 +939,7 @@ doh_recv_two(void **state) { isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url, atomic_load(&POST), doh_connect_send_two_requests_cb, - NULL, ctx, 5000, 0); + NULL, ctx, client_sess_cache, 5000, 0); while (atomic_load(&nsends) > 0) { if (atomic_load(&was_error)) { diff --git a/tests/isc/netmgr_test.c b/tests/isc/netmgr_test.c index 38b147b43f..07d5f6fc4d 100644 --- a/tests/isc/netmgr_test.c +++ b/tests/isc/netmgr_test.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "uv_wrap.h" @@ -62,6 +63,7 @@ static isc_sockaddr_t tcp_listen_addr; static isc_sockaddr_t tcp_connect_addr; static isc_tlsctx_t *tcp_listen_tlsctx = NULL; static isc_tlsctx_t *tcp_connect_tlsctx = NULL; +static isc_tlsctx_client_session_cache_t *tcp_tlsctx_client_sess_cache = NULL; static uint64_t send_magic = 0; static uint64_t stop_magic = 0; @@ -333,6 +335,10 @@ setup_test(void **state __attribute__((unused))) { isc_tlsctx_enable_dot_client_alpn(tcp_connect_tlsctx); + tcp_tlsctx_client_sess_cache = isc_tlsctx_client_session_cache_new( + mctx, tcp_connect_tlsctx, + ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE); + return (0); } @@ -361,6 +367,8 @@ teardown_test(void **state __attribute__((unused))) { isc_refcount_destroy(&active_ssends); isc_refcount_destroy(&active_sreads); + isc_tlsctx_client_session_cache_detach(&tcp_tlsctx_client_sess_cache); + return (0); } @@ -1156,7 +1164,8 @@ stream_connect(isc_nm_cb_t cb, void *cbarg, unsigned int timeout, if (stream_use_TLS) { isc_nm_tlsconnect(connect_nm, &tcp_connect_addr, &tcp_listen_addr, cb, cbarg, - tcp_connect_tlsctx, timeout, extrahandlesize); + tcp_connect_tlsctx, + tcp_tlsctx_client_sess_cache, timeout, 0); return; } #endif @@ -2059,7 +2068,7 @@ static void tls_connect(isc_nm_t *nm) { isc_nm_tlsconnect(nm, &tcp_connect_addr, &tcp_listen_addr, connect_connect_cb, NULL, tcp_connect_tlsctx, - T_CONNECT, 0); + tcp_tlsctx_client_sess_cache, T_CONNECT, 0); } ISC_RUN_TEST_IMPL(tls_noop) {