BUG/MEDIUM: ktls: defer enabling TLS ULP on a socket until connected

The Linux tls module requires a socket to be in TCP_ESTABLISHED state
before we can enable the TLS ULP on the socket, if the socket is in any
other state, then the setsockopt() call will fail, and we won't use
kTLS on that socket.
To make sure we're not doing it too early, defer it until the TLS
handshake is done, which means the TCP connection is established.

This should be backported up to 3.3.

Signed-off-by: Karol Kucharski <kkucharski@fastlogic.pl>

include/haproxy/ssl_sock-t.h

@@ -255,6 +255,7 @@ struct ssl_keylog {
 #define SSL_SOCK_F_KTLS_RECV            (1 << 3) /* kTLS receive is configure on that socket */
 #define SSL_SOCK_F_CTRL_SEND            (1 << 4) /* We want to send a kTLS control message for that socket */
 #define SSL_SOCK_F_HAS_ALPN             (1 << 5) /* An ALPN has been negotiated */
+#define SSL_SOCK_F_KTLS_ULP             (1 << 6) /* TLS ULP is enabled on that socket */
 
 struct ssl_sock_ctx {
 	struct connection *conn;

src/ssl_sock.c

@@ -439,6 +439,22 @@ static int ha_ssl_read(BIO *h, char *buf, int size)
 }
 
 #ifdef HA_USE_KTLS
+static int ktls_enable_ulp(struct ssl_sock_ctx *ctx)
+{
+	int ret = 0;
+
+	if (!(ctx->flags & SSL_SOCK_F_KTLS_ULP)) {
+		ret = setsockopt(ctx->conn->handle.fd, SOL_TCP, TCP_ULP, "tls",
+				 sizeof("tls"));
+		if (ret == 0)
+			ctx->flags |= SSL_SOCK_F_KTLS_ULP;
+		else
+			ctx->flags &= ~SSL_SOCK_F_KTLS_ENABLED;
+	}
+
+	return ret;
+}
+
 /* Returns 0 on success, -1 on failure */
 static int ktls_set_key(struct ssl_sock_ctx *ctx, void *info, size_t info_len, int is_tx)
 {
@@ -485,6 +501,9 @@ static long ha_ssl_ctrl(BIO *h, int cmd, long arg1, void *arg2)
 
 		if (!(ctx->flags & SSL_SOCK_F_KTLS_ENABLED))
 			return 0;
+
+		if (ktls_enable_ulp(ctx) == -1)
+			return 0;
 		/*
 		 * As OpenSSL doesn't export struct tls_crypto_info_all,
 		 * and it puts the size at the end of the struct,
@@ -5670,14 +5689,6 @@ static int ssl_sock_start(struct connection *conn, void *xprt_ctx)
 		if (ret < 0)
 			return ret;
 	}
-#ifdef HA_USE_KTLS
-	/*
-	 * Make the socket usable for kTLS. That does not mean that we will
-	 * use kTLS, though, just that the socket will be able to do it.
-	 */
-	if ((ctx->flags & SSL_SOCK_F_KTLS_ENABLED) && setsockopt(conn->handle.fd, SOL_TCP, TCP_ULP, "tls", sizeof("tls")) != 0)
-		ctx->flags &= ~SSL_SOCK_F_KTLS_ENABLED;
-#endif
 	tasklet_wakeup(ctx->wait_event.tasklet);
 
 	return 0;
@@ -6617,6 +6628,9 @@ static void ssl_sock_setup_ktls(struct ssl_sock_ctx *ctx)
 	if (!(ctx->flags & SSL_SOCK_F_KTLS_ENABLED))
 		return;
 
+	if (ktls_enable_ulp(ctx) == -1)
+		return;
+
 	switch (SSL_version(ctx->ssl)) {
 		case TLS_1_2_VERSION:
 			is_tls_12 = 1;