From 56732ac2a0c38e726c2beba4e8acf33226bf7739 Mon Sep 17 00:00:00 2001
From: Artem Boldariev <artem@boldariev.com>
Date: Mon, 5 Dec 2022 20:19:03 +0200
Subject: [PATCH] TLS: try to avoid allocating send request objects

This commit optimises TLS send request object allocation to enable
send request object reuse, somewhat reducing pressure on the memory
manager. It is especially helpful in the case when Stream DNS uses the
TLS implementation as the transport.
---
 lib/isc/netmgr/netmgr-int.h |  1 +
 lib/isc/netmgr/tlsstream.c  | 66 ++++++++++++++++++++++++++++---------
 2 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index 4fe9e1c0c5..bd8d844989 100644
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -878,6 +878,7 @@ struct isc_nmsocket {
 		} state; /*%< The order of these is significant */
 		size_t nsending;
 		bool tcp_nodelay_value;
+		isc_nmsocket_tls_send_req_t *send_req; /*%< Send req to reuse */
 	} tlsstream;
 
 #if HAVE_LIBNGHTTP2
diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index 05d18e30bf..67298cf8eb 100644
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -131,6 +131,9 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 		(isc_nmsocket_tls_send_req_t *)cbarg;
 	isc_nmsocket_t *tlssock = NULL;
 	bool finish = send_req->finish;
+	isc_nm_cb_t send_cb = NULL;
+	void *send_cbarg = NULL;
+	isc_nmhandle_t *send_handle = NULL;
 
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(VALID_NMSOCK(handle->sock));
@@ -138,34 +141,52 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 
 	tlssock = send_req->tlssock;
 	send_req->tlssock = NULL;
+	send_cb = send_req->cb;
+	send_cbarg = send_req->cbarg;
+	send_handle = send_req->handle;
+	send_req->handle = NULL;
 
 	if (finish) {
 		tls_try_shutdown(tlssock->tlsstream.tls, true);
 	}
 
-	if (send_req->cb != NULL) {
-		INSIST(VALID_NMHANDLE(tlssock->statichandle));
-		send_req->cb(send_req->handle, eresult, send_req->cbarg);
-		isc_nmhandle_detach(&send_req->handle);
-		/* The last handle has been just detached: close the underlying
-		 * socket. */
-		if (tlssock->statichandle == NULL) {
-			finish = true;
-		}
-	}
-
-	/* We are tying to avoid a memory allocation for small write
+	/*
+	 * We are tying to avoid a memory allocation for small write
 	 * requests. See the mirroring code in the tls_send_outgoing()
-	 * function. */
+	 * function. The object is attempted to be freed or put for reuse
+	 * before the call to callback because there is a chance that it
+	 * is going to be reused during the call to the callback.
+	 */
 	if (send_req->data.length > sizeof(send_req->smallbuf)) {
 		isc_mem_put(handle->sock->worker->mctx, send_req->data.base,
 			    send_req->data.length);
 	} else {
 		INSIST(&send_req->smallbuf[0] == send_req->data.base);
 	}
-	isc_mem_put(handle->sock->worker->mctx, send_req, sizeof(*send_req));
+
+	send_req->data.base = NULL;
+	send_req->data.length = 0;
+
+	/* Try to keep the object to be reused later - to avoid an allocation */
+	if (tlssock->tlsstream.send_req == NULL) {
+		tlssock->tlsstream.send_req = send_req;
+	} else {
+		isc_mem_put(handle->sock->worker->mctx, send_req,
+			    sizeof(*send_req));
+	}
 	tlssock->tlsstream.nsending--;
 
+	if (send_cb != NULL) {
+		INSIST(VALID_NMHANDLE(tlssock->statichandle));
+		send_cb(send_handle, eresult, send_cbarg);
+		isc_nmhandle_detach(&send_handle);
+		/* The last handle has been just detached: close the underlying
+		 * socket. */
+		if (tlssock->statichandle == NULL) {
+			finish = true;
+		}
+	}
+
 	if (finish && eresult == ISC_R_SUCCESS && tlssock->reading) {
 		tls_failed_read_cb(tlssock, ISC_R_EOF);
 	} else if (eresult == ISC_R_SUCCESS) {
@@ -277,7 +298,14 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		pending = TLS_BUF_SIZE;
 	}
 
-	send_req = isc_mem_get(sock->worker->mctx, sizeof(*send_req));
+	/* Try to reuse previously allocated object */
+	if (sock->tlsstream.send_req != NULL) {
+		send_req = sock->tlsstream.send_req;
+		sock->tlsstream.send_req = NULL;
+	} else {
+		send_req = isc_mem_get(sock->worker->mctx, sizeof(*send_req));
+	}
+
 	*send_req = (isc_nmsocket_tls_send_req_t){ .finish = finish,
 						   .data.length = pending };
 
@@ -1181,6 +1209,14 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) {
 			isc_tlsctx_client_session_cache_detach(
 				&sock->tlsstream.client_sess_cache);
 		}
+
+		if (sock->tlsstream.send_req != NULL) {
+			INSIST(sock->tlsstream.send_req->data.base == NULL);
+			INSIST(sock->tlsstream.send_req->data.length == 0);
+			isc_mem_put(sock->worker->mctx,
+				    sock->tlsstream.send_req,
+				    sizeof(*sock->tlsstream.send_req));
+		}
 	} else if (sock->type == isc_nm_tcpsocket &&
 		   sock->tlsstream.tlssocket != NULL)
 	{