From 90bc13a5d52a7997a5cb977d62489d034b344b73 Mon Sep 17 00:00:00 2001 From: Artem Boldariev Date: Fri, 22 Apr 2022 15:59:11 +0300 Subject: [PATCH] TLS stream/DoH: implement TLS client session resumption This commit extends TLS stream code and DoH code with TLS client session resumption support implemented on top of the TLS client session cache. --- bin/dig/dighost.c | 2 +- bin/tests/test_client.c | 2 +- lib/isc/include/isc/netmgr.h | 7 ++- lib/isc/netmgr/http.c | 6 ++- lib/isc/netmgr/netmgr-int.h | 5 ++ lib/isc/netmgr/netmgr.c | 21 ++++++++ lib/isc/netmgr/tlsstream.c | 97 +++++++++++++++++++++++++++++------- lib/isc/tests/doh_test.c | 13 +++-- lib/isc/tests/netmgr_test.c | 12 ++++- 9 files changed, 136 insertions(+), 29 deletions(-) diff --git a/bin/dig/dighost.c b/bin/dig/dighost.c index 622c90fff6..ce550fe7bf 100644 --- a/bin/dig/dighost.c +++ b/bin/dig/dighost.c @@ -3020,7 +3020,7 @@ start_tcp(dig_query_t *query) { isc_nm_httpconnect(netmgr, &localaddr, &query->sockaddr, uri, !query->lookup->https_get, tcp_connected, connectquery, tlsctx, - local_timeout); + sess_cache, local_timeout); #endif } else { isc_nm_tcpdnsconnect(netmgr, &localaddr, diff --git a/bin/tests/test_client.c b/bin/tests/test_client.c index edc6d66320..226aac7290 100644 --- a/bin/tests/test_client.c +++ b/bin/tests/test_client.c @@ -428,7 +428,7 @@ run(void) { } isc_nm_httpconnect(netmgr, &sockaddr_local, &sockaddr_remote, req_url, is_post, connect_cb, NULL, tls_ctx, - timeout); + NULL, timeout); } break; #endif default: diff --git a/lib/isc/include/isc/netmgr.h b/lib/isc/include/isc/netmgr.h index d836251015..830fc327a8 100644 --- a/lib/isc/include/isc/netmgr.h +++ b/lib/isc/include/isc/netmgr.h @@ -523,12 +523,15 @@ isc_nm_listentls(isc_nm_t *mgr, uint32_t workers, isc_sockaddr_t *iface, void isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx, - unsigned int timeout); + isc_tlsctx_client_session_cache_t *client_sess_cache, + unsigned int timeout); void isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, const char *uri, bool POST, isc_nm_cb_t cb, void *cbarg, - isc_tlsctx_t *ctx, unsigned int timeout); + isc_tlsctx_t *ctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, + unsigned int timeout); isc_result_t isc_nm_listenhttp(isc_nm_t *mgr, uint32_t workers, isc_sockaddr_t *iface, diff --git a/lib/isc/netmgr/http.c b/lib/isc/netmgr/http.c index 98b2104025..a787e2d310 100644 --- a/lib/isc/netmgr/http.c +++ b/lib/isc/netmgr/http.c @@ -1426,7 +1426,9 @@ error: void isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, const char *uri, bool post, isc_nm_cb_t cb, void *cbarg, - isc_tlsctx_t *tlsctx, unsigned int timeout) { + isc_tlsctx_t *tlsctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, + unsigned int timeout) { isc_sockaddr_t local_interface; isc_nmsocket_t *sock = NULL; @@ -1487,7 +1489,7 @@ isc_nm_httpconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, if (tlsctx != NULL) { isc_nm_tlsconnect(mgr, local, peer, transport_connect_cb, sock, - tlsctx, timeout); + tlsctx, client_sess_cache, timeout); } else { isc_nm_tcpconnect(mgr, local, peer, transport_connect_cb, sock, timeout); diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h index 93af22635f..b6a10a16f5 100644 --- a/lib/isc/netmgr/netmgr-int.h +++ b/lib/isc/netmgr/netmgr-int.h @@ -963,6 +963,8 @@ struct isc_nmsocket { isc_tlsctx_t **listener_tls_ctx; /*%< A context reference per worker */ size_t n_listener_tls_ctx; + isc_tlsctx_client_session_cache_t *client_sess_cache; + bool client_session_saved; isc_nmsocket_t *tlslistener; isc_nmsocket_t *tlssocket; atomic_bool result_updated; @@ -2145,3 +2147,6 @@ isc__nm_udp_freebind(uv_udp_t *handle, const struct sockaddr *addr, int isc__nm_tcp_freebind(uv_tcp_t *handle, const struct sockaddr *addr, unsigned int flags); + +void +isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls); diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c index 880d57ed98..b6ae1ca319 100644 --- a/lib/isc/netmgr/netmgr.c +++ b/lib/isc/netmgr/netmgr.c @@ -3361,6 +3361,27 @@ isc_nmsocket_set_tlsctx(isc_nmsocket_t *listener, isc_tlsctx_t *tlsctx) { }; } +void +isc__nmsocket_log_tls_session_reuse(isc_nmsocket_t *sock, isc_tls_t *tls) { + const int log_level = ISC_LOG_DEBUG(1); + char client_sabuf[ISC_SOCKADDR_FORMATSIZE]; + char local_sabuf[ISC_SOCKADDR_FORMATSIZE]; + + REQUIRE(tls != NULL); + + if (!isc_log_wouldlog(isc_lctx, log_level)) { + return; + }; + + isc_sockaddr_format(&sock->peer, client_sabuf, sizeof(client_sabuf)); + isc_sockaddr_format(&sock->iface, local_sabuf, sizeof(local_sabuf)); + isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR, + log_level, "TLS %s session %s for %s on %s", + SSL_is_server(tls) ? "server" : "client", + SSL_session_reused(tls) ? "resumed" : "created", + client_sabuf, local_sabuf); +} + #ifdef NETMGR_TRACE /* * Dump all active sockets in netmgr. We output to stderr diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c index e725159471..547c518b02 100644 --- a/lib/isc/netmgr/tlsstream.c +++ b/lib/isc/netmgr/tlsstream.c @@ -85,6 +85,12 @@ tls_cleanup_listener_tlsctx(isc_nmsocket_t *listener); static isc_tlsctx_t * tls_get_listener_tlsctx(isc_nmsocket_t *listener, const int tid); +static void +tls_keep_client_tls_session(isc_nmsocket_t *sock); + +static void +tls_try_shutdown(isc_tls_t *tls, const bool quite); + /* * The socket is closing, outerhandle has been detached, listener is * inactive, or the netmgr is closing: any operation on it should abort @@ -127,6 +133,10 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) { tlssock = send_req->tlssock; send_req->tlssock = NULL; + if (finish) { + tls_try_shutdown(tlssock->tlsstream.tls, true); + } + if (send_req->cb != NULL) { INSIST(VALID_NMHANDLE(tlssock->statichandle)); send_req->cb(send_req->handle, eresult, send_req->cbarg); @@ -240,10 +250,9 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle, return (0); } - if (finish && (SSL_get_shutdown(sock->tlsstream.tls) & - SSL_SENT_SHUTDOWN) != SSL_SENT_SHUTDOWN) - { - (void)SSL_shutdown(sock->tlsstream.tls); + if (finish) { + tls_try_shutdown(sock->tlsstream.tls, false); + tls_keep_client_tls_session(sock); } pending = BIO_pending(sock->tlsstream.bio_out); @@ -292,22 +301,21 @@ tls_process_outgoing(isc_nmsocket_t *sock, bool finish, isc__nm_uvreq_t *send_data) { int pending; + bool received_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) & + SSL_RECEIVED_SHUTDOWN) != 0); + bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) & + SSL_SENT_SHUTDOWN) != 0); + + if (received_shutdown && !sent_shutdown) { + finish = true; + } + /* Data from TLS to network */ if (send_data != NULL) { pending = tls_send_outgoing(sock, finish, send_data->handle, send_data->cb.send, send_data->cbarg); } else { - bool received_shutdown = - ((SSL_get_shutdown(sock->tlsstream.tls) & - SSL_RECEIVED_SHUTDOWN) != 0); - bool sent_shutdown = ((SSL_get_shutdown(sock->tlsstream.tls) & - SSL_SENT_SHUTDOWN) != 0); - - if (received_shutdown && !sent_shutdown) { - finish = true; - (void)SSL_shutdown(sock->tlsstream.tls); - } pending = tls_send_outgoing(sock, finish, NULL, NULL, NULL); } @@ -330,6 +338,7 @@ tls_try_handshake(isc_nmsocket_t *sock) { isc_result_t result = ISC_R_SUCCESS; INSIST(SSL_is_init_finished(sock->tlsstream.tls) == 1); INSIST(sock->statichandle == NULL); + isc__nmsocket_log_tls_session_reuse(sock, sock->tlsstream.tls); tlshandle = isc__nmhandle_get(sock, &sock->peer, &sock->iface); if (sock->tlsstream.server) { sock->listener->accept_cb(tlshandle, result, @@ -884,7 +893,8 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg); void isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, - isc_nm_cb_t cb, void *cbarg, SSL_CTX *ctx, + isc_nm_cb_t cb, void *cbarg, isc_tlsctx_t *ctx, + isc_tlsctx_client_session_cache_t *client_sess_cache, unsigned int timeout) { isc_nmsocket_t *nsock = NULL; #if defined(NETMGR_TRACE) && defined(NETMGR_TRACE_VERBOSE) @@ -901,6 +911,13 @@ isc_nm_tlsconnect(isc_nm_t *mgr, isc_sockaddr_t *local, isc_sockaddr_t *peer, nsock->connect_cbarg = cbarg; nsock->connect_timeout = timeout; isc_tlsctx_attach(ctx, &nsock->tlsstream.ctx); + atomic_init(&nsock->client, true); + if (client_sess_cache != NULL) { + INSIST(isc_tlsctx_client_session_cache_getctx( + client_sess_cache) == ctx); + isc_tlsctx_client_session_cache_attach( + client_sess_cache, &nsock->tlsstream.client_sess_cache); + } isc_nm_tcpconnect(mgr, local, peer, tcp_connected, nsock, nsock->connect_timeout); @@ -943,6 +960,12 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) { isc_nmhandle_attach(handle, &tlssock->outerhandle); atomic_store(&tlssock->active, true); + if (tlssock->tlsstream.client_sess_cache != NULL) { + isc_tlsctx_client_session_cache_reuse_sockaddr( + tlssock->tlsstream.client_sess_cache, &tlssock->peer, + tlssock->tlsstream.tls); + } + /* * Hold a reference to tlssock in the TCP socket: it will * detached in isc__nm_tls_cleanup_data(). @@ -1020,15 +1043,26 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) { } else if (sock->type == isc_nm_tlslistener) { tls_cleanup_listener_tlsctx(sock); } else if (sock->type == isc_nm_tlssocket) { - if (sock->tlsstream.ctx != NULL) { - isc_tlsctx_free(&sock->tlsstream.ctx); - } if (sock->tlsstream.tls != NULL) { + /* + * Let's shutdown the TLS session properly so that the + * session will remain resumable, if required. + */ + tls_try_shutdown(sock->tlsstream.tls, true); + tls_keep_client_tls_session(sock); isc_tls_free(&sock->tlsstream.tls); /* These are destroyed when we free SSL */ sock->tlsstream.bio_out = NULL; sock->tlsstream.bio_in = NULL; } + if (sock->tlsstream.ctx != NULL) { + isc_tlsctx_free(&sock->tlsstream.ctx); + } + if (sock->tlsstream.client_sess_cache != NULL) { + INSIST(atomic_load(&sock->client)); + isc_tlsctx_client_session_cache_detach( + &sock->tlsstream.client_sess_cache); + } } else if (sock->type == isc_nm_tcpsocket && sock->tlsstream.tlssocket != NULL) { /* @@ -1158,3 +1192,30 @@ isc__nm_async_tls_set_tlsctx(isc_nmsocket_t *listener, isc_tlsctx_t *tlsctx, isc_tlsctx_free(&listener->tlsstream.listener_tls_ctx[tid]); isc_tlsctx_attach(tlsctx, &listener->tlsstream.listener_tls_ctx[tid]); } + +static void +tls_keep_client_tls_session(isc_nmsocket_t *sock) { + /* + * Ensure that the isc_tls_t is being accessed from + * within the worker thread the socket is bound to. + */ + REQUIRE(sock->tid == isc_nm_tid()); + if (sock->tlsstream.client_sess_cache != NULL && + sock->tlsstream.client_session_saved == false) + { + INSIST(atomic_load(&sock->client)); + isc_tlsctx_client_session_cache_keep_sockaddr( + sock->tlsstream.client_sess_cache, &sock->peer, + sock->tlsstream.tls); + sock->tlsstream.client_session_saved = true; + } +} + +static void +tls_try_shutdown(isc_tls_t *tls, const bool force) { + if (force) { + (void)SSL_set_shutdown(tls, SSL_SENT_SHUTDOWN); + } else if ((SSL_get_shutdown(tls) & SSL_SENT_SHUTDOWN) == 0) { + (void)SSL_shutdown(tls); + } +} diff --git a/lib/isc/tests/doh_test.c b/lib/isc/tests/doh_test.c index 0d3c43448f..84ffc49317 100644 --- a/lib/isc/tests/doh_test.c +++ b/lib/isc/tests/doh_test.c @@ -80,6 +80,7 @@ static atomic_bool slowdown = false; static atomic_bool use_TLS = false; static isc_tlsctx_t *server_tlsctx = NULL; static isc_tlsctx_t *client_tlsctx = NULL; +static isc_tlsctx_client_session_cache_t *client_sess_cache = NULL; static isc_quota_t listener_quota; static atomic_bool check_listener_quota = false; @@ -169,7 +170,8 @@ connect_send_request(isc_nm_t *mgr, const char *uri, bool post, } isc_nm_httpconnect(mgr, NULL, &tcp_listen_addr, uri, post, - connect_send_cb, data, ctx, timeout); + connect_send_cb, data, ctx, client_sess_cache, + timeout); } static int @@ -334,6 +336,9 @@ nm_setup(void **state) { client_tlsctx = NULL; isc_tlsctx_createclient(&client_tlsctx); isc_tlsctx_enable_http2client_alpn(client_tlsctx); + client_sess_cache = isc_tlsctx_client_session_cache_new( + test_mctx, client_tlsctx, + ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE); isc_quota_init(&listener_quota, 0); atomic_store(&check_listener_quota, false); @@ -363,6 +368,8 @@ nm_teardown(void **state) { isc_tlsctx_free(&client_tlsctx); } + isc_tlsctx_client_session_cache_detach(&client_sess_cache); + isc_quota_destroy(&listener_quota); isc_nm_http_endpoints_detach(&endpoints); @@ -666,7 +673,7 @@ doh_timeout_recovery(void **state) { ISC_NM_HTTP_DEFAULT_PATH); isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url, atomic_load(&POST), timeout_request_cb, NULL, ctx, - T_SOFT); + client_sess_cache, T_SOFT); /* * Sleep until sends reaches 5. @@ -946,7 +953,7 @@ doh_recv_two(void **state) { isc_nm_httpconnect(connect_nm, NULL, &tcp_listen_addr, req_url, atomic_load(&POST), doh_connect_send_two_requests_cb, - NULL, ctx, 5000); + NULL, ctx, client_sess_cache, 5000); while (atomic_load(&nsends) > 0) { if (atomic_load(&was_error)) { diff --git a/lib/isc/tests/netmgr_test.c b/lib/isc/tests/netmgr_test.c index 6c8e3ed53e..8cf0187cec 100644 --- a/lib/isc/tests/netmgr_test.c +++ b/lib/isc/tests/netmgr_test.c @@ -58,6 +58,7 @@ static isc_sockaddr_t tcp_listen_addr; static isc_sockaddr_t tcp_connect_addr; static isc_tlsctx_t *tcp_listen_tlsctx = NULL; static isc_tlsctx_t *tcp_connect_tlsctx = NULL; +static isc_tlsctx_client_session_cache_t *tcp_tlsctx_client_sess_cache = NULL; static uint64_t send_magic = 0; static uint64_t stop_magic = 0; @@ -226,6 +227,10 @@ _setup(void **state __attribute__((unused))) { isc_tlsctx_enable_dot_client_alpn(tcp_connect_tlsctx); + tcp_tlsctx_client_sess_cache = isc_tlsctx_client_session_cache_new( + test_mctx, tcp_connect_tlsctx, + ISC_TLSCTX_CLIENT_SESSION_CACHE_DEFAULT_SIZE); + return (0); } @@ -234,6 +239,8 @@ _teardown(void **state __attribute__((unused))) { isc_tlsctx_free(&tcp_connect_tlsctx); isc_tlsctx_free(&tcp_listen_tlsctx); + isc_tlsctx_client_session_cache_detach(&tcp_tlsctx_client_sess_cache); + isc_test_end(); return (0); @@ -1201,7 +1208,8 @@ stream_connect(isc_nm_cb_t cb, void *cbarg, unsigned int timeout) { if (stream_use_TLS) { isc_nm_tlsconnect(connect_nm, &tcp_connect_addr, &tcp_listen_addr, cb, cbarg, - tcp_connect_tlsctx, timeout); + tcp_connect_tlsctx, + tcp_tlsctx_client_sess_cache, timeout); return; } #endif @@ -2139,7 +2147,7 @@ static void tls_connect(isc_nm_t *nm) { isc_nm_tlsconnect(nm, &tcp_connect_addr, &tcp_listen_addr, connect_connect_cb, NULL, tcp_connect_tlsctx, - T_CONNECT); + tcp_tlsctx_client_sess_cache, T_CONNECT); } static void