From 05cfb27b809f55ecce5f7edef5b706b63738636e Mon Sep 17 00:00:00 2001
From: Artem Boldariev <artem@boldariev.com>
Date: Thu, 25 Aug 2022 22:37:26 +0300
Subject: [PATCH] Disable Nagle's algorithm for TLS connections by default

This commit ensures that Nagle's algorithm is disabled by default for
TLS connections on best effort basis, just like other networking
software (e.g. NGINX) does, as, in the case of TLS, we are not
interested in trading latency for throughput, rather vice versa.

We attempt to disable it as early as we can, right after TCP
connections establishment, as an attempt to speed up handshake
handling.
---
 lib/isc/netmgr/netmgr-int.h |  4 ++++
 lib/isc/netmgr/netmgr.c     |  3 +++
 lib/isc/netmgr/tlsstream.c  | 45 +++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+)

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index 1c229d314a..0ce595c113 100644
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -924,6 +924,7 @@ struct isc_nmsocket {
 			TLS_CLOSED
 		} state; /*%< The order of these is significant */
 		size_t nsending;
+		bool tcp_nodelay_value;
 	} tlsstream;
 
 	isc_nmsocket_h2_t h2;
@@ -1625,6 +1626,9 @@ isc__nmhandle_tls_get_selected_alpn(isc_nmhandle_t *handle,
 				    const unsigned char **alpn,
 				    unsigned int *alpnlen);
 
+isc_result_t
+isc__nmhandle_tls_set_tcp_nodelay(isc_nmhandle_t *handle, const bool value);
+
 void
 isc__nm_http_stoplistening(isc_nmsocket_t *sock);
 
diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index 1cd09c1c48..8577ac0197 100644
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -3030,6 +3030,9 @@ isc_nmhandle_set_tcp_nodelay(isc_nmhandle_t *handle, const bool value) {
 		result = isc__nm_socket_tcp_nodelay((uv_os_sock_t)tcp_fd,
 						    value);
 	} break;
+	case isc_nm_tlssocket:
+		result = isc__nmhandle_tls_set_tcp_nodelay(handle, value);
+		break;
 	default:
 		UNREACHABLE();
 		break;
diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index 834a386ce6..05d18e30bf 100644
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -94,6 +94,9 @@ tls_keep_client_tls_session(isc_nmsocket_t *sock);
 static void
 tls_try_shutdown(isc_tls_t *tls, const bool quite);
 
+static void
+tls_try_to_enable_tcp_nodelay(isc_nmsocket_t *tlssock);
+
 /*
  * The socket is closing, outerhandle has been detached, listener is
  * inactive, or the netmgr is closing: any operation on it should abort
@@ -739,6 +742,17 @@ error:
 	return (ISC_R_TLSERROR);
 }
 
+static void
+tls_try_to_enable_tcp_nodelay(isc_nmsocket_t *tlssock) {
+	/*
+	 * Try to enable TCP_NODELAY for TLS connections by default to speed up
+	 * the handshakes, just like other software (e.g. NGINX) does.
+	 */
+	isc_result_t result = isc_nmhandle_set_tcp_nodelay(tlssock->outerhandle,
+							   true);
+	tlssock->tlsstream.tcp_nodelay_value = (result == ISC_R_SUCCESS);
+}
+
 static isc_result_t
 tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	isc_nmsocket_t *tlslistensock = (isc_nmsocket_t *)cbarg;
@@ -796,6 +810,8 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	RUNTIME_CHECK(result == ISC_R_SUCCESS);
 	/* TODO: catch failure code, detach tlssock, and log the error */
 
+	tls_try_to_enable_tcp_nodelay(tlssock);
+
 	isc__nmhandle_set_manual_timer(tlssock->outerhandle, true);
 	tls_do_bio(tlssock, NULL, NULL, false);
 	return (result);
@@ -1114,6 +1130,8 @@ tcp_connected(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
 	 */
 	handle->sock->tlsstream.tlssocket = tlssock;
 
+	tls_try_to_enable_tcp_nodelay(tlssock);
+
 	isc__nmhandle_set_manual_timer(tlssock->outerhandle, true);
 	tls_do_bio(tlssock, NULL, NULL, false);
 	return;
@@ -1417,3 +1435,30 @@ isc__nmhandle_tls_get_selected_alpn(isc_nmhandle_t *handle,
 
 	isc_tls_get_selected_alpn(sock->tlsstream.tls, alpn, alpnlen);
 }
+
+isc_result_t
+isc__nmhandle_tls_set_tcp_nodelay(isc_nmhandle_t *handle, const bool value) {
+	isc_nmsocket_t *sock = NULL;
+	isc_result_t result = ISC_R_FAILURE;
+
+	REQUIRE(VALID_NMHANDLE(handle));
+	REQUIRE(VALID_NMSOCK(handle->sock));
+	REQUIRE(handle->sock->type == isc_nm_tlssocket);
+
+	sock = handle->sock;
+	if (sock->outerhandle != NULL) {
+		INSIST(VALID_NMHANDLE(sock->outerhandle));
+
+		if (value == sock->tlsstream.tcp_nodelay_value) {
+			result = ISC_R_SUCCESS;
+		} else {
+			result = isc_nmhandle_set_tcp_nodelay(sock->outerhandle,
+							      value);
+			if (result == ISC_R_SUCCESS) {
+				sock->tlsstream.tcp_nodelay_value = value;
+			}
+		}
+	}
+
+	return (result);
+}