From 9e14430d463ccd92a3acd5e82676ee3ccad8b0ef Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 8 Oct 2019 21:34:06 +0000 Subject: [PATCH] Add a TOE KTLS mode and a TOE hook for allocating TLS sessions. This adds the glue to allocate TLS sessions and invokes it from the TLS enable socket option handler. This also adds some counters for active TOE sessions. The TOE KTLS mode is returned by getsockopt(TLSTX_TLS_MODE) when TOE KTLS is in use on a socket, but cannot be set via setsockopt(). To simplify various checks, a TLS session now includes an explicit 'mode' member set to the value returned by TLSTX_TLS_MODE. Various places that used to check 'sw_encrypt' against NULL to determine software vs ifnet (NIC) TLS now check 'mode' instead. Reviewed by: np, gallatin Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D21891 --- sys/kern/kern_sendfile.c | 5 +- sys/kern/uipc_ktls.c | 121 +++++++++++++++++++++++++++++++++----- sys/kern/uipc_socket.c | 4 +- sys/netinet/tcp.h | 1 + sys/netinet/tcp_offload.c | 11 ++++ sys/netinet/tcp_offload.h | 1 + sys/netinet/tcp_usrreq.c | 2 - sys/netinet/toecore.c | 9 +++ sys/netinet/toecore.h | 5 ++ sys/sys/ktls.h | 1 + 10 files changed, 140 insertions(+), 20 deletions(-) diff --git a/sys/kern/kern_sendfile.c b/sys/kern/kern_sendfile.c index 5e1c49dca99..b5c50c17b90 100644 --- a/sys/kern/kern_sendfile.c +++ b/sys/kern/kern_sendfile.c @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include @@ -295,7 +296,7 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, int error) mb_free_notready(sfio->m, sfio->npages); #ifdef KERN_TLS - } else if (sfio->tls != NULL && sfio->tls->sw_encrypt != NULL) { + } else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) { /* * I/O operation is complete, but we still need to * encrypt. We cannot do this in the interrupt thread @@ -1028,7 +1029,7 @@ prepend_header: */ free(sfio, M_TEMP); #ifdef KERN_TLS - if (tls != NULL && tls->sw_encrypt != NULL) { + if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { error = (*so->so_proto->pr_usrreqs->pru_send) (so, PRUS_NOTREADY, m, NULL, NULL, td); soref(so); diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c index 80abbfcb44a..c439edea501 100644 --- a/sys/kern/uipc_ktls.c +++ b/sys/kern/uipc_ktls.c @@ -63,6 +63,9 @@ __FBSDID("$FreeBSD$"); #include #endif #include +#ifdef TCP_OFFLOAD +#include +#endif #include #include #include @@ -161,6 +164,10 @@ SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD, 0, "Software TLS session stats"); SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD, 0, "Hardware (ifnet) TLS session stats"); +#ifdef TCP_OFFLOAD +SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD, 0, + "TOE TLS session stats"); +#endif static counter_u64_t ktls_sw_cbc; SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, @@ -199,6 +206,18 @@ SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN, &ktls_ifnet_permitted, 1, "Whether to permit hardware (ifnet) TLS sessions"); +#ifdef TCP_OFFLOAD +static counter_u64_t ktls_toe_cbc; +SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD, + &ktls_toe_cbc, + "Active number of TOE TLS sessions using AES-CBC"); + +static counter_u64_t ktls_toe_gcm; +SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD, + &ktls_toe_gcm, + "Active number of TOE TLS sessions using AES-GCM"); +#endif + static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); static void ktls_cleanup(struct ktls_session *tls); @@ -325,6 +344,10 @@ ktls_init(void *dummy __unused) ktls_ifnet_reset = counter_u64_alloc(M_WAITOK); ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK); ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK); +#ifdef TCP_OFFLOAD + ktls_toe_cbc = counter_u64_alloc(M_WAITOK); + ktls_toe_gcm = counter_u64_alloc(M_WAITOK); +#endif rm_init(&ktls_backends_lock, "ktls backends"); LIST_INIT(&ktls_backends); @@ -607,7 +630,8 @@ ktls_cleanup(struct ktls_session *tls) { counter_u64_add(ktls_offload_active, -1); - if (tls->free != NULL) { + switch (tls->mode) { + case TCP_TLS_MODE_SW: MPASS(tls->be != NULL); switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: @@ -618,7 +642,8 @@ ktls_cleanup(struct ktls_session *tls) break; } tls->free(tls); - } else if (tls->snd_tag != NULL) { + break; + case TCP_TLS_MODE_IFNET: switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_ifnet_cbc, -1); @@ -628,6 +653,19 @@ ktls_cleanup(struct ktls_session *tls) break; } m_snd_tag_rele(tls->snd_tag); + break; +#ifdef TCP_OFFLOAD + case TCP_TLS_MODE_TOE: + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_toe_cbc, -1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_toe_gcm, -1); + break; + } + break; +#endif } if (tls->params.auth_key != NULL) { explicit_bzero(tls->params.auth_key, tls->params.auth_key_len); @@ -646,6 +684,52 @@ ktls_cleanup(struct ktls_session *tls) } #if defined(INET) || defined(INET6) + +#ifdef TCP_OFFLOAD +static int +ktls_try_toe(struct socket *so, struct ktls_session *tls) +{ + struct inpcb *inp; + struct tcpcb *tp; + int error; + + inp = so->so_pcb; + INP_WLOCK(inp); + if (inp->inp_flags2 & INP_FREED) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_socket == NULL) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + tp = intotcpcb(inp); + if (tp->tod == NULL) { + INP_WUNLOCK(inp); + return (EOPNOTSUPP); + } + + error = tcp_offload_alloc_tls_session(tp, tls); + INP_WUNLOCK(inp); + if (error == 0) { + tls->mode = TCP_TLS_MODE_TOE; + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_toe_cbc, 1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_toe_gcm, 1); + break; + } + } + return (error); +} +#endif + /* * Common code used when first enabling ifnet TLS on a connection or * when allocating a new ifnet TLS session due to a routing change. @@ -744,6 +828,7 @@ ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force) error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); if (error == 0) { + tls->mode = TCP_TLS_MODE_IFNET; tls->snd_tag = mst; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: @@ -787,6 +872,7 @@ ktls_try_sw(struct socket *so, struct ktls_session *tls) rm_runlock(&ktls_backends_lock, &prio); if (be == NULL) return (EOPNOTSUPP); + tls->mode = TCP_TLS_MODE_SW; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_sw_cbc, 1); @@ -834,8 +920,12 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en) if (error) return (error); - /* Prefer ifnet TLS over software TLS. */ - error = ktls_try_ifnet(so, tls, false); + /* Prefer TOE -> ifnet TLS -> software TLS. */ +#ifdef TCP_OFFLOAD + error = ktls_try_toe(so, tls); + if (error) +#endif + error = ktls_try_ifnet(so, tls, false); if (error) error = ktls_try_sw(so, tls); @@ -852,7 +942,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *en) SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls; - if (tls->sw_encrypt == NULL) + if (tls->mode != TCP_TLS_MODE_SW) so->so_snd.sb_flags |= SB_TLS_IFNET; SOCKBUF_UNLOCK(&so->so_snd); sbunlock(&so->so_snd); @@ -875,10 +965,8 @@ ktls_get_tx_mode(struct socket *so) tls = so->so_snd.sb_tls_info; if (tls == NULL) mode = TCP_TLS_MODE_NONE; - else if (tls->sw_encrypt != NULL) - mode = TCP_TLS_MODE_SW; else - mode = TCP_TLS_MODE_IFNET; + mode = tls->mode; SOCKBUF_UNLOCK(&so->so_snd); return (mode); } @@ -893,7 +981,13 @@ ktls_set_tx_mode(struct socket *so, int mode) struct inpcb *inp; int error; - MPASS(mode == TCP_TLS_MODE_SW || mode == TCP_TLS_MODE_IFNET); + switch (mode) { + case TCP_TLS_MODE_SW: + case TCP_TLS_MODE_IFNET: + break; + default: + return (EINVAL); + } inp = so->so_pcb; INP_WLOCK_ASSERT(inp); @@ -904,8 +998,7 @@ ktls_set_tx_mode(struct socket *so, int mode) return (0); } - if ((tls->sw_encrypt != NULL && mode == TCP_TLS_MODE_SW) || - (tls->sw_encrypt == NULL && mode == TCP_TLS_MODE_IFNET)) { + if (tls->mode == mode) { SOCKBUF_UNLOCK(&so->so_snd); return (0); } @@ -952,7 +1045,7 @@ ktls_set_tx_mode(struct socket *so, int mode) SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls_new; - if (tls_new->sw_encrypt == NULL) + if (tls_new->mode != TCP_TLS_MODE_SW) so->so_snd.sb_flags |= SB_TLS_IFNET; SOCKBUF_UNLOCK(&so->so_snd); sbunlock(&so->so_snd); @@ -1238,7 +1331,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt, * When using ifnet TLS, unencrypted TLS records are * sent down the stack to the NIC. */ - if (tls->sw_encrypt != NULL) { + if (tls->mode == TCP_TLS_MODE_SW) { m->m_flags |= M_NOTREADY; pgs->nrdy = pgs->npgs; *enq_cnt += pgs->npgs; @@ -1278,7 +1371,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count) pgs = m->m_ext.ext_pgs; - KASSERT(pgs->tls->sw_encrypt != NULL, ("ifnet TLS mbuf")); + KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf")); pgs->enc_cnt = page_count; pgs->mbuf = m; diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index dbb6ab4d261..0ebb2fcfe38 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -1491,7 +1491,7 @@ sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, tls_pruflag = 0; tls = ktls_hold(so->so_snd.sb_tls_info); if (tls != NULL) { - if (tls->sw_encrypt != NULL) + if (tls->mode == TCP_TLS_MODE_SW) tls_pruflag = PRUS_NOTREADY; if (control != NULL) { @@ -1659,7 +1659,7 @@ restart: } #ifdef KERN_TLS - if (tls != NULL && tls->sw_encrypt != NULL) { + if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { /* * Note that error is intentionally * ignored. diff --git a/sys/netinet/tcp.h b/sys/netinet/tcp.h index 37ba3bb5574..125cacb2835 100644 --- a/sys/netinet/tcp.h +++ b/sys/netinet/tcp.h @@ -357,6 +357,7 @@ struct tcp_function_set { #define TCP_TLS_MODE_NONE 0 #define TCP_TLS_MODE_SW 1 #define TCP_TLS_MODE_IFNET 2 +#define TCP_TLS_MODE_TOE 3 /* * TCP Control message types diff --git a/sys/netinet/tcp_offload.c b/sys/netinet/tcp_offload.c index 7e0bab70e03..60fb78e435b 100644 --- a/sys/netinet/tcp_offload.c +++ b/sys/netinet/tcp_offload.c @@ -178,6 +178,17 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info *ti) tod->tod_tcp_info(tod, tp, ti); } +int +tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls) +{ + struct toedev *tod = tp->tod; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); + + return (tod->tod_alloc_tls_session(tod, tp, tls)); +} + void tcp_offload_detach(struct tcpcb *tp) { diff --git a/sys/netinet/tcp_offload.h b/sys/netinet/tcp_offload.h index f755ce7ed20..b89a367cd62 100644 --- a/sys/netinet/tcp_offload.h +++ b/sys/netinet/tcp_offload.h @@ -46,6 +46,7 @@ int tcp_offload_output(struct tcpcb *); void tcp_offload_rcvd(struct tcpcb *); void tcp_offload_ctloutput(struct tcpcb *, int, int); void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *); +int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *); void tcp_offload_detach(struct tcpcb *); #endif diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c index 4812912d7d7..4d5c0db9fc4 100644 --- a/sys/netinet/tcp_usrreq.c +++ b/sys/netinet/tcp_usrreq.c @@ -1936,8 +1936,6 @@ unlock_and_done: error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui)); if (error) return (error); - if (ui != TCP_TLS_MODE_SW && ui != TCP_TLS_MODE_IFNET) - return (EINVAL); INP_WLOCK_RECHECK(inp); error = ktls_set_tx_mode(so, ui); diff --git a/sys/netinet/toecore.c b/sys/netinet/toecore.c index 5df1968fb3b..b0b9e9a2f66 100644 --- a/sys/netinet/toecore.c +++ b/sys/netinet/toecore.c @@ -191,6 +191,14 @@ toedev_tcp_info(struct toedev *tod __unused, struct tcpcb *tp __unused, return; } +static int +toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, + struct ktls_session *tls __unused) +{ + + return (EINVAL); +} + /* * Inform one or more TOE devices about a listening socket. */ @@ -281,6 +289,7 @@ init_toedev(struct toedev *tod) tod->tod_offload_socket = toedev_offload_socket; tod->tod_ctloutput = toedev_ctloutput; tod->tod_tcp_info = toedev_tcp_info; + tod->tod_alloc_tls_session = toedev_alloc_tls_session; } /* diff --git a/sys/netinet/toecore.h b/sys/netinet/toecore.h index f8bda614b35..2180d0fb19b 100644 --- a/sys/netinet/toecore.h +++ b/sys/netinet/toecore.h @@ -41,6 +41,7 @@ struct tcpopt; struct tcphdr; struct in_conninfo; struct tcp_info; +struct ktls_session; struct toedev { TAILQ_ENTRY(toedev) link; /* glue for toedev_list */ @@ -108,6 +109,10 @@ struct toedev { /* Update software state */ void (*tod_tcp_info)(struct toedev *, struct tcpcb *, struct tcp_info *); + + /* Create a TLS session */ + int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *, + struct ktls_session *); }; typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h index 62f694b3dae..f11cd8c287e 100644 --- a/sys/sys/ktls.h +++ b/sys/sys/ktls.h @@ -156,6 +156,7 @@ struct ktls_session { struct tls_session_params params; u_int wq_index; volatile u_int refcount; + int mode; struct task reset_tag_task; struct inpcb *inp;