From e639957b586a0824e32ef7db7f04cd1708fa8da7 Mon Sep 17 00:00:00 2001
From: Artem Boldariev <artem@boldariev.com>
Date: Mon, 2 Aug 2021 17:15:13 +0300
Subject: [PATCH] Optimise TLS stream for small write size (>= 512 bytes)

This commit changes TLS stream behaviour in such a way, that it is now
optimised for small writes. In the case there is a need to write less
or equal to 512 bytes, we could avoid calling the memory allocator at
the expense of possibly slight increase in memory usage. In case of
larger writes, the behviour remains unchanged.
---
 lib/isc/netmgr/netmgr-int.h |  1 +
 lib/isc/netmgr/tlsstream.c  | 25 ++++++++++++++++++-------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index c75ddddfb8..23cfe8cbf1 100644
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -754,6 +754,7 @@ typedef struct isc_nmsocket_tls_send_req {
 	void *cbarg;
 	isc_nmhandle_t *handle;
 	bool finish;
+	uint8_t smallbuf[512];
 } isc_nmsocket_tls_send_req_t;
 
 typedef enum isc_http_request_type {
diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index f5e65e930d..0ccf68b4eb 100644
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -123,8 +123,15 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 		}
 	}
 
-	isc_mem_put(handle->sock->mgr->mctx, send_req->data.base,
-		    send_req->data.length);
+	/* We are tying to avoid a memory allocation for small write
+	 * requests. See the mirroring code in the tls_send_outgoing()
+	 * function. */
+	if (ISC_UNLIKELY(send_req->data.length > sizeof(send_req->smallbuf))) {
+		isc_mem_put(handle->sock->mgr->mctx, send_req->data.base,
+			    send_req->data.length);
+	} else {
+		INSIST(&send_req->smallbuf[0] == send_req->data.base);
+	}
 	isc_mem_put(handle->sock->mgr->mctx, send_req, sizeof(*send_req));
 	tlssock->tlsstream.nsending--;
 
@@ -236,11 +243,15 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 	}
 
 	send_req = isc_mem_get(sock->mgr->mctx, sizeof(*send_req));
-	*send_req = (isc_nmsocket_tls_send_req_t){
-		.finish = finish,
-		.data.base = isc_mem_get(sock->mgr->mctx, pending),
-		.data.length = pending
-	};
+	*send_req = (isc_nmsocket_tls_send_req_t){ .finish = finish,
+						   .data.length = pending };
+
+	/* Let's try to avoid a memory allocation for small write requests */
+	if (ISC_UNLIKELY((size_t)pending > sizeof(send_req->smallbuf))) {
+		send_req->data.base = isc_mem_get(sock->mgr->mctx, pending);
+	} else {
+		send_req->data.base = &send_req->smallbuf[0];
+	}
 
 	isc__nmsocket_attach(sock, &send_req->tlssock);
 	if (cb != NULL) {