diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 27655ec2fe5..edcc6e3f524 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -1291,11 +1291,28 @@ int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int, u_int); /* t4_keyctx.c */ struct auth_hash; union authctx; +#ifdef KERN_TLS +struct ktls_session; +struct tls_key_req; +struct tls_keyctx; +#endif void t4_aes_getdeckey(void *, const void *, unsigned int); void t4_copy_partial_hash(int, union authctx *, void *); void t4_init_gmac_hash(const char *, int, char *); void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *); +#ifdef KERN_TLS +u_int t4_tls_key_info_size(const struct ktls_session *); +int t4_tls_proto_ver(const struct ktls_session *); +int t4_tls_cipher_mode(const struct ktls_session *); +int t4_tls_auth_mode(const struct ktls_session *); +int t4_tls_hmac_ctrl(const struct ktls_session *); +void t4_tls_key_ctx(const struct ktls_session *, int, struct tls_keyctx *); +int t4_alloc_tls_keyid(struct adapter *); +void t4_free_tls_keyid(struct adapter *, int); +void t4_write_tlskey_wr(const struct ktls_session *, int, int, int, int, + struct tls_key_req *); +#endif #ifdef DEV_NETMAP /* t4_netmap.c */ diff --git a/sys/dev/cxgbe/crypto/t4_crypto.h b/sys/dev/cxgbe/crypto/t4_crypto.h index 161bb2aea52..83ea0fecc6c 100644 --- a/sys/dev/cxgbe/crypto/t4_crypto.h +++ b/sys/dev/cxgbe/crypto/t4_crypto.h @@ -191,4 +191,59 @@ struct phys_sge_pairs { #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 +/* + * TODO: Should coalesce ccr's key context with the TLS key context. + * Lookaside requests use the TX context header. + */ + +struct tls_key_req { + /* FW_ULPTX_WR */ + __be32 wr_hi; + __be32 wr_mid; + __be32 ftid; + __u8 reneg_to_write_rx; + __u8 protocol; + __be16 mfs; + /* master command */ + __be32 cmd; + __be32 len16; /* command length */ + __be32 dlen; /* data length in 32-byte units */ + __be32 kaddr; + /* sub-command */ + __be32 sc_more; + __be32 sc_len; +}__packed; + +struct tls_keyctx { + union key_ctx { + struct tx_keyctx_hdr { + __u8 ctxlen; + __u8 r2; + __be16 dualck_to_txvalid; + __u8 txsalt[4]; + __be64 r5; + } txhdr; + struct rx_keyctx_hdr { + __u8 flitcnt_hmacctrl; + __u8 protover_ciphmode; + __u8 authmode_to_rxvalid; + __u8 ivpresent_to_rxmk_size; + __u8 rxsalt[4]; + __be64 ivinsert_to_authinsrt; + } rxhdr; + } u; + struct keys { + __u8 edkey[32]; + __u8 ipad[64]; + __u8 opad[64]; + } keys; +}; + +#define SALT_SIZE 4 + +#define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_keyctx), 32) + +#define TLS_KEY_WR_SZ \ + roundup2(sizeof(struct tls_key_req) + TLS_KEY_CONTEXT_SZ, 16) + #endif /* !__T4_CRYPTO_H__ */ diff --git a/sys/dev/cxgbe/crypto/t4_kern_tls.c b/sys/dev/cxgbe/crypto/t4_kern_tls.c index 99d0d33cf12..a20c3045b5b 100644 --- a/sys/dev/cxgbe/crypto/t4_kern_tls.c +++ b/sys/dev/cxgbe/crypto/t4_kern_tls.c @@ -60,101 +60,13 @@ __FBSDID("$FreeBSD$"); #if defined(INET) || defined(INET6) -#define SALT_SIZE 4 - -#define GCM_TAG_SIZE 16 #define TLS_HEADER_LENGTH 5 -#define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_keyctx), 32) - struct tls_scmd { __be32 seqno_numivs; __be32 ivgen_hdrlen; }; -struct tls_key_req { - /* FW_ULPTX_WR */ - __be32 wr_hi; - __be32 wr_mid; - __be32 ftid; - __u8 reneg_to_write_rx; - __u8 protocol; - __be16 mfs; - /* master command */ - __be32 cmd; - __be32 len16; /* command length */ - __be32 dlen; /* data length in 32-byte units */ - __be32 kaddr; - /* sub-command */ - __be32 sc_more; - __be32 sc_len; -}__packed; - -struct tls_keyctx { - struct tx_keyctx_hdr { - __u8 ctxlen; - __u8 r2; - __be16 dualck_to_txvalid; - __u8 txsalt[4]; - __be64 r5; - } txhdr; - struct keys { - __u8 edkey[32]; - __u8 ipad[64]; - __u8 opad[64]; - } keys; -}; - -#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 -#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 -#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) -#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ - M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) -#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ - V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) - -#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 -#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 -#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) -#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ - M_TLS_KEYCTX_TX_WR_SALT_PRESENT) -#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ - V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) - -#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 -#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf -#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) -#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ - M_TLS_KEYCTX_TX_WR_TXCK_SIZE) - -#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 -#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf -#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) -#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ - M_TLS_KEYCTX_TX_WR_TXMK_SIZE) - -#define S_TLS_KEYCTX_TX_WR_TXVALID 0 -#define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 -#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXVALID) -#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) -#define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) - -/* Key Context Programming Operation type */ -#define KEY_WRITE_RX 0x1 -#define KEY_WRITE_TX 0x2 -#define KEY_DELETE_RX 0x4 -#define KEY_DELETE_TX 0x8 - struct tlspcb { struct m_snd_tag com; struct vi_info *vi; /* virtual interface */ @@ -187,15 +99,6 @@ struct tlspcb { struct sge_wrq *ctrlq; struct clip_entry *ce; /* CLIP table entry used by this tid */ - unsigned char auth_mode; - unsigned char hmac_ctrl; - unsigned char mac_first; - unsigned char iv_size; - - unsigned int frag_size; - unsigned int cipher_secret_size; - int proto_ver; - bool open_pending; }; @@ -208,27 +111,6 @@ mst_to_tls(struct m_snd_tag *t) return (__containerof(t, struct tlspcb, com)); } -/* XXX: There are similar versions of these two in tom/t4_tls.c. */ -static int -get_new_keyid(struct tlspcb *tlsp) -{ - vmem_addr_t addr; - - if (vmem_alloc(tlsp->sc->key_map, TLS_KEY_CONTEXT_SZ, - M_NOWAIT | M_FIRSTFIT, &addr) != 0) - return (-1); - - return (addr); -} - -static void -free_keyid(struct tlspcb *tlsp, int keyid) -{ - - CTR3(KTR_CXGBE, "%s: tid %d key addr %#x", __func__, tlsp->tid, keyid); - vmem_free(tlsp->sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); -} - static struct tlspcb * alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags) { @@ -250,50 +132,6 @@ alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags) return (tlsp); } -static void -init_ktls_key_params(struct tlspcb *tlsp, const struct ktls_session *tls) -{ - int mac_key_size; - - if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) - tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_1; - else - tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_2; - tlsp->cipher_secret_size = tls->params.cipher_key_len; - tlsp->tx_key_info_size = sizeof(struct tx_keyctx_hdr) + - tlsp->cipher_secret_size; - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { - tlsp->auth_mode = SCMD_AUTH_MODE_GHASH; - tlsp->enc_mode = SCMD_CIPH_MODE_AES_GCM; - tlsp->iv_size = 4; - tlsp->mac_first = 0; - tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NOP; - tlsp->tx_key_info_size += GMAC_BLOCK_LEN; - } else { - switch (tls->params.auth_algorithm) { - case CRYPTO_SHA1_HMAC: - mac_key_size = roundup2(SHA1_HASH_LEN, 16); - tlsp->auth_mode = SCMD_AUTH_MODE_SHA1; - break; - case CRYPTO_SHA2_256_HMAC: - mac_key_size = SHA2_256_HASH_LEN; - tlsp->auth_mode = SCMD_AUTH_MODE_SHA256; - break; - case CRYPTO_SHA2_384_HMAC: - mac_key_size = SHA2_512_HASH_LEN; - tlsp->auth_mode = SCMD_AUTH_MODE_SHA512_384; - break; - } - tlsp->enc_mode = SCMD_CIPH_MODE_AES_CBC; - tlsp->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */ - tlsp->mac_first = 1; - tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC; - tlsp->tx_key_info_size += mac_key_size * 2; - } - - tlsp->frag_size = tls->params.max_frame_len; -} - static int ktls_act_open_cpl_size(bool isipv6) { @@ -539,7 +377,7 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct inpcb *inp; struct tcpcb *tp; struct sge_txq *txq; - int atid, error, keyid; + int atid, error, explicit_iv_size, keyid, mac_first; tls = params->tls.tls; @@ -569,6 +407,8 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, default: return (EPROTONOSUPPORT); } + explicit_iv_size = AES_BLOCK_LEN; + mac_first = 1; break; case CRYPTO_AES_NIST_GCM_16: if (tls->params.iv_len != SALT_SIZE) @@ -581,6 +421,8 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, default: return (EINVAL); } + explicit_iv_size = 8; + mac_first = 0; break; default: return (EPROTONOSUPPORT); @@ -600,7 +442,7 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, if (sc->tlst.inline_keys) keyid = -1; else - keyid = get_new_keyid(tlsp); + keyid = t4_alloc_tls_keyid(sc); if (keyid < 0) { CTR2(KTR_CXGBE, "%s: atid %d using immediate key ctx", __func__, atid); @@ -673,21 +515,22 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, if (error) goto failed; - init_ktls_key_params(tlsp, tls); - error = ktls_setup_keys(tlsp, tls, txq); if (error) goto failed; + tlsp->enc_mode = t4_tls_cipher_mode(tls); + tlsp->tx_key_info_size = t4_tls_key_info_size(tls); + /* The SCMD fields used when encrypting a full TLS record. */ tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) | - V_SCMD_PROTO_VERSION(tlsp->proto_ver) | + V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) | V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | - V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) | + V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | V_SCMD_CIPH_MODE(tlsp->enc_mode) | - V_SCMD_AUTH_MODE(tlsp->auth_mode) | - V_SCMD_HMAC_CTRL(tlsp->hmac_ctrl) | - V_SCMD_IV_SIZE(tlsp->iv_size) | V_SCMD_NUM_IVS(1)); + V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) | + V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) | + V_SCMD_IV_SIZE(explicit_iv_size / 2) | V_SCMD_NUM_IVS(1)); tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | V_SCMD_TLS_FRAG_ENABLE(0); @@ -702,7 +545,7 @@ cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) | V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | - V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) | + V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(0); @@ -741,14 +584,11 @@ static int ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls, struct sge_txq *txq) { - struct auth_hash *axf; - int error, keyid, kwrlen, kctxlen, len; struct tls_key_req *kwr; struct tls_keyctx *kctx; - void *items[1], *key; - struct tx_keyctx_hdr *khdr; - unsigned int ck_size, mk_size, partial_digest_len; + void *items[1]; struct mbuf *m; + int error; /* * Store the salt and keys in the key context. For @@ -757,80 +597,12 @@ ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls, * storing the key in DDR, a work request is used to store a * copy of the key context in DDR. */ - kctx = &tlsp->keyctx; - khdr = &kctx->txhdr; - - switch (tlsp->cipher_secret_size) { - case 128 / 8: - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - break; - case 192 / 8: - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; - break; - case 256 / 8: - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; - break; - default: - panic("bad key size"); - } - axf = NULL; - partial_digest_len = 0; - if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; - else { - switch (tlsp->auth_mode) { - case SCMD_AUTH_MODE_SHA1: - axf = &auth_hash_hmac_sha1; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; - partial_digest_len = SHA1_HASH_LEN; - break; - case SCMD_AUTH_MODE_SHA256: - axf = &auth_hash_hmac_sha2_256; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; - partial_digest_len = SHA2_256_HASH_LEN; - break; - case SCMD_AUTH_MODE_SHA512_384: - axf = &auth_hash_hmac_sha2_384; - mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; - partial_digest_len = SHA2_512_HASH_LEN; - break; - default: - panic("bad auth mode"); - } - } - - khdr->ctxlen = (tlsp->tx_key_info_size >> 4); - khdr->dualck_to_txvalid = V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | - V_TLS_KEYCTX_TX_WR_TXCK_SIZE(ck_size) | - V_TLS_KEYCTX_TX_WR_TXMK_SIZE(mk_size) | - V_TLS_KEYCTX_TX_WR_TXVALID(1); - if (tlsp->enc_mode != SCMD_CIPH_MODE_AES_GCM) - khdr->dualck_to_txvalid |= V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1); - khdr->dualck_to_txvalid = htobe16(khdr->dualck_to_txvalid); - key = kctx->keys.edkey; - memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len); - if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { - memcpy(khdr->txsalt, tls->params.iv, SALT_SIZE); - t4_init_gmac_hash(tls->params.cipher_key, - tls->params.cipher_key_len, - (char *)key + tls->params.cipher_key_len); - } else { - t4_init_hmac_digest(axf, partial_digest_len, - tls->params.auth_key, tls->params.auth_key_len, - (char *)key + tls->params.cipher_key_len); - } - + t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx); if (tlsp->inline_key) return (0); - keyid = tlsp->tx_key_addr; - /* Populate key work request. */ - kwrlen = sizeof(*kwr); - kctxlen = roundup2(sizeof(*kctx), 32); - len = kwrlen + kctxlen; - - m = alloc_wr_mbuf(len, M_NOWAIT); + m = alloc_wr_mbuf(TLS_KEY_WR_SZ, M_NOWAIT); if (m == NULL) { CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__, tlsp->tid); @@ -839,27 +611,9 @@ ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls, m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com); m->m_pkthdr.csum_flags |= CSUM_SND_TAG; kwr = mtod(m, void *); - memset(kwr, 0, len); - - kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | - F_FW_WR_ATOMIC); - kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16))); - kwr->protocol = tlsp->proto_ver; - kwr->mfs = htons(tlsp->frag_size); - kwr->reneg_to_write_rx = KEY_WRITE_TX; - - /* master command */ - kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | - V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); - kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); - kwr->len16 = htobe32((tlsp->tid << 8) | - DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); - kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); - - /* sub command */ - kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); - kwr->sc_len = htobe32(kctxlen); + memset(kwr, 0, TLS_KEY_WR_SZ); + t4_write_tlskey_wr(tls, KTLS_TX, tlsp->tid, 0, tlsp->tx_key_addr, kwr); kctx = (struct tls_keyctx *)(kwr + 1); memcpy(kctx, &tlsp->keyctx, sizeof(*kctx)); @@ -2029,7 +1783,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, if (plen == tlen) { iv = out; if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { - memcpy(iv, tlsp->keyctx.txhdr.txsalt, SALT_SIZE); + memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE); memcpy(iv + 4, hdr + 1, 8); *(uint32_t *)(iv + 12) = htobe32(2 + offset / AES_BLOCK_LEN); @@ -2335,7 +2089,7 @@ cxgbe_tls_tag_free(struct m_snd_tag *mst) if (tlsp->ce) t4_release_clip_entry(sc, tlsp->ce); if (tlsp->tx_key_addr >= 0) - free_keyid(tlsp, tlsp->tx_key_addr); + t4_free_tls_keyid(sc, tlsp->tx_key_addr); zfree(tlsp, M_CXGBE); } diff --git a/sys/dev/cxgbe/crypto/t4_keyctx.c b/sys/dev/cxgbe/crypto/t4_keyctx.c index 91bd5f28321..b64eb4ff23d 100644 --- a/sys/dev/cxgbe/crypto/t4_keyctx.c +++ b/sys/dev/cxgbe/crypto/t4_keyctx.c @@ -25,10 +25,13 @@ * SUCH DAMAGE. */ +#include "opt_kern_tls.h" + #include __FBSDID("$FreeBSD$"); #include +#include #include #include @@ -61,6 +64,234 @@ __FBSDID("$FreeBSD$"); * +-------------------------------+ - */ +/* Fields in the key context header. */ +#define S_TLS_KEYCTX_TX_WR_DUALCK 12 +#define M_TLS_KEYCTX_TX_WR_DUALCK 0x1 +#define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK) +#define G_TLS_KEYCTX_TX_WR_DUALCK(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK) +#define F_TLS_KEYCTX_TX_WR_DUALCK V_TLS_KEYCTX_TX_WR_DUALCK(1U) + +#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 +#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) +#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ + M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) +#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ + V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 +#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) +#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ + M_TLS_KEYCTX_TX_WR_SALT_PRESENT) +#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ + V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 +#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf +#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) +#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_TXCK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 +#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf +#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) +#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_TXMK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_TXVALID 0 +#define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 +#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_TXVALID) +#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) +#define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) + +#define S_TLS_KEYCTX_TX_WR_FLITCNT 3 +#define M_TLS_KEYCTX_TX_WR_FLITCNT 0x1f +#define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT) +#define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT) + +#define S_TLS_KEYCTX_TX_WR_HMACCTRL 0 +#define M_TLS_KEYCTX_TX_WR_HMACCTRL 0x7 +#define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL) +#define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL) + +#define S_TLS_KEYCTX_TX_WR_PROTOVER 4 +#define M_TLS_KEYCTX_TX_WR_PROTOVER 0xf +#define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER) +#define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER) + +#define S_TLS_KEYCTX_TX_WR_CIPHMODE 0 +#define M_TLS_KEYCTX_TX_WR_CIPHMODE 0xf +#define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE) +#define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE) + +#define S_TLS_KEYCTX_TX_WR_AUTHMODE 4 +#define M_TLS_KEYCTX_TX_WR_AUTHMODE 0xf +#define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE) +#define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE) + +#define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3 +#define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1 +#define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) +#define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \ + M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) +#define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \ + V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U) + +#define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1 +#define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3 +#define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) +#define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \ + M_TLS_KEYCTX_TX_WR_SEQNUMCTRL) + +#define S_TLS_KEYCTX_TX_WR_RXVALID 0 +#define M_TLS_KEYCTX_TX_WR_RXVALID 0x1 +#define V_TLS_KEYCTX_TX_WR_RXVALID(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXVALID) +#define G_TLS_KEYCTX_TX_WR_RXVALID(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID) +#define F_TLS_KEYCTX_TX_WR_RXVALID V_TLS_KEYCTX_TX_WR_RXVALID(1U) + +#define S_TLS_KEYCTX_TX_WR_IVPRESENT 7 +#define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT) +#define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \ + M_TLS_KEYCTX_TX_WR_IVPRESENT) +#define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6 +#define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1 +#define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) +#define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \ + M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) +#define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \ + V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U) + +#define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3 +#define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7 +#define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE) +#define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_RXCK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0 +#define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7 +#define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE) +#define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \ + M_TLS_KEYCTX_TX_WR_RXMK_SIZE) + +#define S_TLS_KEYCTX_TX_WR_IVINSERT 55 +#define M_TLS_KEYCTX_TX_WR_IVINSERT 0x1ffULL +#define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT) +#define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT) + +#define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47 +#define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL +#define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST) +#define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \ + M_TLS_KEYCTX_TX_WR_AADSTRTOFST) + +#define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39 +#define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL +#define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST) +#define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \ + M_TLS_KEYCTX_TX_WR_AADSTOPOFST) + +#define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30 +#define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL +#define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) +#define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \ + M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) + +#define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23 +#define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f +#define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) +#define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \ + M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) + +#define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14 +#define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff +#define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) +#define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \ + M_TLS_KEYCTX_TX_WR_AUTHSRTOFST) + +#define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7 +#define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f +#define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) +#define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \ + M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) + +#define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0 +#define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f +#define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ + ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT) +#define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ + (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \ + M_TLS_KEYCTX_TX_WR_AUTHINSRT) + +/* Key Context Programming Operation type */ +#define KEY_WRITE_RX 0x1 +#define KEY_WRITE_TX 0x2 +#define KEY_DELETE_RX 0x4 +#define KEY_DELETE_TX 0x8 + +#define S_KEY_CLR_LOC 4 +#define M_KEY_CLR_LOC 0xf +#define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC) +#define G_KEY_CLR_LOC(x) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC) +#define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U) + +#define S_KEY_GET_LOC 0 +#define M_KEY_GET_LOC 0xf +#define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC) +#define G_KEY_GET_LOC(x) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC) + /* * Generate the initial GMAC hash state for a AES-GCM key. * @@ -172,3 +403,278 @@ t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits) MPASS(dkey == dec_key); explicit_bzero(ek, sizeof(ek)); } + +#ifdef KERN_TLS +/* + * - keyid management + * - request to program key? + */ +u_int +t4_tls_key_info_size(const struct ktls_session *tls) +{ + u_int key_info_size, mac_key_size; + + key_info_size = sizeof(struct tx_keyctx_hdr) + + tls->params.cipher_key_len; + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { + key_info_size += GMAC_BLOCK_LEN; + } else { + switch (tls->params.auth_algorithm) { + case CRYPTO_SHA1_HMAC: + mac_key_size = SHA1_HASH_LEN; + break; + case CRYPTO_SHA2_256_HMAC: + mac_key_size = SHA2_256_HASH_LEN; + break; + case CRYPTO_SHA2_384_HMAC: + mac_key_size = SHA2_512_HASH_LEN; + break; + default: + __assert_unreachable(); + } + key_info_size += roundup2(mac_key_size, 16) * 2; + } + return (key_info_size); +} + +int +t4_tls_proto_ver(const struct ktls_session *tls) +{ + if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) + return (SCMD_PROTO_VERSION_TLS_1_1); + else + return (SCMD_PROTO_VERSION_TLS_1_2); +} + +int +t4_tls_cipher_mode(const struct ktls_session *tls) +{ + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + return (SCMD_CIPH_MODE_AES_CBC); + case CRYPTO_AES_NIST_GCM_16: + return (SCMD_CIPH_MODE_AES_GCM); + default: + return (SCMD_CIPH_MODE_NOP); + } +} + +int +t4_tls_auth_mode(const struct ktls_session *tls) +{ + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + switch (tls->params.auth_algorithm) { + case CRYPTO_SHA1_HMAC: + return (SCMD_AUTH_MODE_SHA1); + case CRYPTO_SHA2_256_HMAC: + return (SCMD_AUTH_MODE_SHA256); + case CRYPTO_SHA2_384_HMAC: + return (SCMD_AUTH_MODE_SHA512_384); + default: + return (SCMD_AUTH_MODE_NOP); + } + case CRYPTO_AES_NIST_GCM_16: + return (SCMD_AUTH_MODE_GHASH); + default: + return (SCMD_AUTH_MODE_NOP); + } +} + +int +t4_tls_hmac_ctrl(const struct ktls_session *tls) +{ + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + return (SCMD_HMAC_CTRL_NO_TRUNC); + case CRYPTO_AES_NIST_GCM_16: + return (SCMD_HMAC_CTRL_NOP); + default: + return (SCMD_HMAC_CTRL_NOP); + } +} + +static int +tls_cipher_key_size(const struct ktls_session *tls) +{ + switch (tls->params.cipher_key_len) { + case 128 / 8: + return (CHCR_KEYCTX_CIPHER_KEY_SIZE_128); + case 192 / 8: + return (CHCR_KEYCTX_CIPHER_KEY_SIZE_192); + case 256 / 8: + return (CHCR_KEYCTX_CIPHER_KEY_SIZE_256); + default: + __assert_unreachable(); + } +} + +static int +tls_mac_key_size(const struct ktls_session *tls) +{ + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) + return (CHCR_KEYCTX_MAC_KEY_SIZE_512); + else { + switch (tls->params.auth_algorithm) { + case CRYPTO_SHA1_HMAC: + return (CHCR_KEYCTX_MAC_KEY_SIZE_160); + case CRYPTO_SHA2_256_HMAC: + return (CHCR_KEYCTX_MAC_KEY_SIZE_256); + case CRYPTO_SHA2_384_HMAC: + return (CHCR_KEYCTX_MAC_KEY_SIZE_512); + default: + __assert_unreachable(); + } + } +} + +void +t4_tls_key_ctx(const struct ktls_session *tls, int direction, + struct tls_keyctx *kctx) +{ + struct auth_hash *axf; + u_int mac_key_size; + char *hash; + + /* Key context header. */ + if (direction == KTLS_TX) { + kctx->u.txhdr.ctxlen = t4_tls_key_info_size(tls) / 16; + kctx->u.txhdr.dualck_to_txvalid = + V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | + V_TLS_KEYCTX_TX_WR_TXCK_SIZE(tls_cipher_key_size(tls)) | + V_TLS_KEYCTX_TX_WR_TXMK_SIZE(tls_mac_key_size(tls)) | + V_TLS_KEYCTX_TX_WR_TXVALID(1); + if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) + kctx->u.txhdr.dualck_to_txvalid |= + V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1); + kctx->u.txhdr.dualck_to_txvalid = + htobe16(kctx->u.txhdr.dualck_to_txvalid); + } else { + kctx->u.rxhdr.flitcnt_hmacctrl = + V_TLS_KEYCTX_TX_WR_FLITCNT(t4_tls_key_info_size(tls) / 16) | + V_TLS_KEYCTX_TX_WR_HMACCTRL(t4_tls_hmac_ctrl(tls)); + + kctx->u.rxhdr.protover_ciphmode = + V_TLS_KEYCTX_TX_WR_PROTOVER(t4_tls_proto_ver(tls)) | + V_TLS_KEYCTX_TX_WR_CIPHMODE(t4_tls_cipher_mode(tls)); + + kctx->u.rxhdr.authmode_to_rxvalid = + V_TLS_KEYCTX_TX_WR_AUTHMODE(t4_tls_auth_mode(tls)) | + V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | + V_TLS_KEYCTX_TX_WR_RXVALID(1); + + kctx->u.rxhdr.ivpresent_to_rxmk_size = + V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | + V_TLS_KEYCTX_TX_WR_RXCK_SIZE(tls_cipher_key_size(tls)) | + V_TLS_KEYCTX_TX_WR_RXMK_SIZE(tls_mac_key_size(tls)); + + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { + kctx->u.rxhdr.ivinsert_to_authinsrt = + htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | + V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | + V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | + V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); + } else { + kctx->u.rxhdr.authmode_to_rxvalid |= + V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1); + kctx->u.rxhdr.ivpresent_to_rxmk_size |= + V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1); + kctx->u.rxhdr.ivinsert_to_authinsrt = + htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | + V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | + V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | + V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | + V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | + V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); + } + } + + /* Key. */ + if (direction == KTLS_RX && + tls->params.cipher_algorithm == CRYPTO_AES_CBC) + t4_aes_getdeckey(kctx->keys.edkey, tls->params.cipher_key, + tls->params.cipher_key_len * 8); + else + memcpy(kctx->keys.edkey, tls->params.cipher_key, + tls->params.cipher_key_len); + + /* Auth state and implicit IV (salt). */ + hash = kctx->keys.edkey + tls->params.cipher_key_len; + if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { + _Static_assert(offsetof(struct tx_keyctx_hdr, txsalt) == + offsetof(struct rx_keyctx_hdr, rxsalt), + "salt offset mismatch"); + memcpy(kctx->u.txhdr.txsalt, tls->params.iv, SALT_SIZE); + t4_init_gmac_hash(tls->params.cipher_key, + tls->params.cipher_key_len, hash); + } else { + switch (tls->params.auth_algorithm) { + case CRYPTO_SHA1_HMAC: + axf = &auth_hash_hmac_sha1; + mac_key_size = SHA1_HASH_LEN; + break; + case CRYPTO_SHA2_256_HMAC: + axf = &auth_hash_hmac_sha2_256; + mac_key_size = SHA2_256_HASH_LEN; + break; + case CRYPTO_SHA2_384_HMAC: + axf = &auth_hash_hmac_sha2_384; + mac_key_size = SHA2_512_HASH_LEN; + break; + default: + __assert_unreachable(); + } + t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, + tls->params.auth_key_len, hash); + } +} + +int +t4_alloc_tls_keyid(struct adapter *sc) +{ + vmem_addr_t addr; + + if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, + &addr) != 0) + return (-1); + + return (addr); +} + +void +t4_free_tls_keyid(struct adapter *sc, int keyid) +{ + vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); +} + +void +t4_write_tlskey_wr(const struct ktls_session *tls, int direction, int tid, + int flags, int keyid, struct tls_key_req *kwr) +{ + kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_ATOMIC | flags); + kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(TLS_KEY_WR_SZ, 16)) | + V_FW_WR_FLOWID(tid)); + kwr->protocol = t4_tls_proto_ver(tls); + kwr->mfs = htobe16(tls->params.max_frame_len); + kwr->reneg_to_write_rx = V_KEY_GET_LOC(direction == KTLS_TX ? + KEY_WRITE_TX : KEY_WRITE_RX); + + /* master command */ + kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | + V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); + kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(TLS_KEY_CONTEXT_SZ >> 5)); + kwr->len16 = htobe32((tid << 8) | + DIV_ROUND_UP(TLS_KEY_WR_SZ - sizeof(struct work_request_hdr), 16)); + kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); + + /* sub command */ + kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); + kwr->sc_len = htobe32(TLS_KEY_CONTEXT_SZ); +} +#endif diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c index be47dbac7ae..95fe96b9024 100644 --- a/sys/dev/cxgbe/tom/t4_tls.c +++ b/sys/dev/cxgbe/tom/t4_tls.c @@ -147,274 +147,19 @@ tls_content_type(unsigned char content_type) return CPL_TX_TLS_SFO_TYPE_DATA; } -static int -tls_key_info_size(struct ktls_session *tls) -{ - u_int key_info_size, mac_key_size; - - key_info_size = sizeof(struct tx_keyctx_hdr) + - tls->params.cipher_key_len; - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { - key_info_size += GMAC_BLOCK_LEN; - } else { - switch (tls->params.auth_algorithm) { - case CRYPTO_SHA1_HMAC: - mac_key_size = SHA1_HASH_LEN; - break; - case CRYPTO_SHA2_256_HMAC: - mac_key_size = SHA2_256_HASH_LEN; - break; - case CRYPTO_SHA2_384_HMAC: - mac_key_size = SHA2_512_HASH_LEN; - break; - default: - __assert_unreachable(); - } - key_info_size += roundup2(mac_key_size, 16) * 2; - } - return (key_info_size); -} - -static int -tls_proto_ver(struct ktls_session *tls) -{ - if (tls->params.tls_vminor == TLS_MINOR_VER_ONE) - return (SCMD_PROTO_VERSION_TLS_1_1); - else - return (SCMD_PROTO_VERSION_TLS_1_2); -} - -static int -tls_cipher_mode(struct ktls_session *tls) -{ - switch (tls->params.cipher_algorithm) { - case CRYPTO_AES_CBC: - return (SCMD_CIPH_MODE_AES_CBC); - case CRYPTO_AES_NIST_GCM_16: - return (SCMD_CIPH_MODE_AES_GCM); - default: - return (SCMD_CIPH_MODE_NOP); - } -} - -static int -tls_auth_mode(struct ktls_session *tls) -{ - switch (tls->params.cipher_algorithm) { - case CRYPTO_AES_CBC: - switch (tls->params.auth_algorithm) { - case CRYPTO_SHA1_HMAC: - return (SCMD_AUTH_MODE_SHA1); - case CRYPTO_SHA2_256_HMAC: - return (SCMD_AUTH_MODE_SHA256); - case CRYPTO_SHA2_384_HMAC: - return (SCMD_AUTH_MODE_SHA512_384); - default: - return (SCMD_AUTH_MODE_NOP); - } - case CRYPTO_AES_NIST_GCM_16: - return (SCMD_AUTH_MODE_GHASH); - default: - return (SCMD_AUTH_MODE_NOP); - } -} - -static int -tls_hmac_ctrl(struct ktls_session *tls) -{ - switch (tls->params.cipher_algorithm) { - case CRYPTO_AES_CBC: - return (SCMD_HMAC_CTRL_NO_TRUNC); - case CRYPTO_AES_NIST_GCM_16: - return (SCMD_HMAC_CTRL_NOP); - default: - return (SCMD_HMAC_CTRL_NOP); - } -} - -static int -tls_cipher_key_size(struct ktls_session *tls) -{ - switch (tls->params.cipher_key_len) { - case 128 / 8: - return (CHCR_KEYCTX_CIPHER_KEY_SIZE_128); - case 192 / 8: - return (CHCR_KEYCTX_CIPHER_KEY_SIZE_192); - case 256 / 8: - return (CHCR_KEYCTX_CIPHER_KEY_SIZE_256); - default: - __assert_unreachable(); - } -} - -static int -tls_mac_key_size(struct ktls_session *tls) -{ - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) - /* - * XXX: This used to use 128 (SHA_NOP) for TOE, - * but NIC TLS has always used 512. - */ - return (CHCR_KEYCTX_MAC_KEY_SIZE_512); - else { - switch (tls->params.auth_algorithm) { - case CRYPTO_SHA1_HMAC: - return (CHCR_KEYCTX_MAC_KEY_SIZE_160); - case CRYPTO_SHA2_256_HMAC: - return (CHCR_KEYCTX_MAC_KEY_SIZE_256); - case CRYPTO_SHA2_384_HMAC: - return (CHCR_KEYCTX_MAC_KEY_SIZE_512); - default: - __assert_unreachable(); - } - } -} - -static void -prepare_tls_keys(char *key, char *salt, struct ktls_session *tls, - int direction) -{ - struct auth_hash *axf; - u_int mac_key_size; - char *hash; - - if (direction == KTLS_RX && - tls->params.cipher_algorithm == CRYPTO_AES_CBC) - t4_aes_getdeckey(key, tls->params.cipher_key, - tls->params.cipher_key_len * 8); - else - memcpy(key, tls->params.cipher_key, - tls->params.cipher_key_len); - hash = key + tls->params.cipher_key_len; - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { - memcpy(salt, tls->params.iv, SALT_SIZE); - t4_init_gmac_hash(tls->params.cipher_key, - tls->params.cipher_key_len, hash); - } else { - switch (tls->params.auth_algorithm) { - case CRYPTO_SHA1_HMAC: - axf = &auth_hash_hmac_sha1; - mac_key_size = SHA1_HASH_LEN; - break; - case CRYPTO_SHA2_256_HMAC: - axf = &auth_hash_hmac_sha2_256; - mac_key_size = SHA2_256_HASH_LEN; - break; - case CRYPTO_SHA2_384_HMAC: - axf = &auth_hash_hmac_sha2_384; - mac_key_size = SHA2_512_HASH_LEN; - break; - default: - __assert_unreachable(); - } - t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key, - tls->params.auth_key_len, hash); - } -} - -/* Rx key */ -static void -prepare_rxkey_wr(struct tls_keyctx *kwr, struct ktls_session *tls) -{ - - kwr->u.rxhdr.flitcnt_hmacctrl = - ((tls_key_info_size(tls) / 16) << 3) | tls_hmac_ctrl(tls); - - kwr->u.rxhdr.protover_ciphmode = - V_TLS_KEYCTX_TX_WR_PROTOVER(tls_proto_ver(tls)) | - V_TLS_KEYCTX_TX_WR_CIPHMODE(tls_cipher_mode(tls)); - - kwr->u.rxhdr.authmode_to_rxvalid = - V_TLS_KEYCTX_TX_WR_AUTHMODE(tls_auth_mode(tls)) | - V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) | - V_TLS_KEYCTX_TX_WR_RXVALID(1); - - kwr->u.rxhdr.ivpresent_to_rxmk_size = - V_TLS_KEYCTX_TX_WR_IVPRESENT(0) | - V_TLS_KEYCTX_TX_WR_RXCK_SIZE(tls_cipher_key_size(tls)) | - V_TLS_KEYCTX_TX_WR_RXMK_SIZE(tls_mac_key_size(tls)); - - if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) { - kwr->u.rxhdr.ivinsert_to_authinsrt = - htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | - V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | - V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | - V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) | - V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) | - V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) | - V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | - V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL)); - } else { - kwr->u.rxhdr.authmode_to_rxvalid |= - V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1); - kwr->u.rxhdr.ivpresent_to_rxmk_size |= - V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1); - kwr->u.rxhdr.ivinsert_to_authinsrt = - htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) | - V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) | - V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) | - V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) | - V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) | - V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) | - V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) | - V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL)); - } - - prepare_tls_keys(kwr->keys.edkey, kwr->u.rxhdr.rxsalt, tls, KTLS_RX); -} - -/* Tx key */ -static void -prepare_txkey_wr(struct tls_keyctx *kwr, struct ktls_session *tls) -{ - - kwr->u.txhdr.ctxlen = tls_key_info_size(tls) / 16; - kwr->u.txhdr.dualck_to_txvalid = - V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) | - V_TLS_KEYCTX_TX_WR_TXCK_SIZE(tls_cipher_key_size(tls)) | - V_TLS_KEYCTX_TX_WR_TXMK_SIZE(tls_mac_key_size(tls)) | - V_TLS_KEYCTX_TX_WR_TXVALID(1); - if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) - kwr->u.txhdr.dualck_to_txvalid |= - V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1); - kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid); - - prepare_tls_keys(kwr->keys.edkey, kwr->u.txhdr.txsalt, tls, KTLS_TX); -} - /* TLS Key memory management */ -static int -get_new_keyid(struct toepcb *toep) -{ - struct adapter *sc = td_adapter(toep->td); - vmem_addr_t addr; - - if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT, - &addr) != 0) - return (-1); - - return (addr); -} - -static void -free_keyid(struct toepcb *toep, int keyid) -{ - struct adapter *sc = td_adapter(toep->td); - - vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ); -} - static void clear_tls_keyid(struct toepcb *toep) { struct tls_ofld_info *tls_ofld = &toep->tls; + struct adapter *sc = td_adapter(toep->td); if (tls_ofld->rx_key_addr >= 0) { - free_keyid(toep, tls_ofld->rx_key_addr); + t4_free_tls_keyid(sc, tls_ofld->rx_key_addr); tls_ofld->rx_key_addr = -1; } if (tls_ofld->tx_key_addr >= 0) { - free_keyid(toep, tls_ofld->tx_key_addr); + t4_free_tls_keyid(sc, tls_ofld->tx_key_addr); tls_ofld->tx_key_addr = -1; } } @@ -435,64 +180,45 @@ tls_program_key_id(struct toepcb *toep, struct ktls_session *tls, struct tls_ofld_info *tls_ofld = &toep->tls; struct adapter *sc = td_adapter(toep->td); struct ofld_tx_sdesc *txsd; - int kwrlen, kctxlen, keyid, len; + int keyid; struct wrqe *wr; struct tls_key_req *kwr; struct tls_keyctx *kctx; +#ifdef INVARIANTS + int kwrlen, kctxlen, len; + kwrlen = sizeof(*kwr); kctxlen = roundup2(sizeof(*kctx), 32); len = roundup2(kwrlen + kctxlen, 16); - + MPASS(TLS_KEY_WR_SZ == len); +#endif if (toep->txsd_avail == 0) return (EAGAIN); - if ((keyid = get_new_keyid(toep)) < 0) { + if ((keyid = t4_alloc_tls_keyid(sc)) < 0) { return (ENOSPC); } - wr = alloc_wrqe(len, &toep->ofld_txq->wrq); + wr = alloc_wrqe(TLS_KEY_WR_SZ, &toep->ofld_txq->wrq); if (wr == NULL) { - free_keyid(toep, keyid); + t4_free_tls_keyid(sc, keyid); return (ENOMEM); } kwr = wrtod(wr); - memset(kwr, 0, kwrlen); - - kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL | - F_FW_WR_ATOMIC); - kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) | - V_FW_WR_FLOWID(toep->tid)); - kwr->protocol = tls_proto_ver(tls); - kwr->mfs = htons(tls->params.max_frame_len); - kwr->reneg_to_write_rx = V_KEY_GET_LOC(direction == KTLS_TX ? - KEY_WRITE_TX : KEY_WRITE_RX); - - /* master command */ - kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | - V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1)); - kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5)); - kwr->len16 = htobe32((toep->tid << 8) | - DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16)); - kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5)); - - /* sub command */ - kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); - kwr->sc_len = htobe32(kctxlen); + memset(kwr, 0, TLS_KEY_WR_SZ); + t4_write_tlskey_wr(tls, direction, toep->tid, F_FW_WR_COMPL, keyid, + kwr); kctx = (struct tls_keyctx *)(kwr + 1); - memset(kctx, 0, kctxlen); - - if (direction == KTLS_TX) { + if (direction == KTLS_TX) tls_ofld->tx_key_addr = keyid; - prepare_txkey_wr(kctx, tls); - } else { + else tls_ofld->rx_key_addr = keyid; - prepare_rxkey_wr(kctx, tls); - } + t4_tls_key_ctx(tls, direction, kctx); txsd = &toep->txsd[toep->txsd_pidx]; - txsd->tx_credits = DIV_ROUND_UP(len, 16); + txsd->tx_credits = DIV_ROUND_UP(TLS_KEY_WR_SZ, 16); txsd->plen = 0; toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) @@ -661,12 +387,12 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) if (direction == KTLS_TX) { toep->tls.scmd0.seqno_numivs = (V_SCMD_SEQ_NO_CTRL(3) | - V_SCMD_PROTO_VERSION(tls_proto_ver(tls)) | + V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) | V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | - V_SCMD_CIPH_MODE(tls_cipher_mode(tls)) | - V_SCMD_AUTH_MODE(tls_auth_mode(tls)) | - V_SCMD_HMAC_CTRL(tls_hmac_ctrl(tls)) | + V_SCMD_CIPH_MODE(t4_tls_cipher_mode(tls)) | + V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) | + V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) | V_SCMD_IV_SIZE(explicit_iv_size / 2)); toep->tls.scmd0.ivgen_hdrlen = @@ -686,7 +412,7 @@ tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction) toep->tls.pdus_per_ulp = 1; toep->tls.adjusted_plen = toep->tls.expn_per_ulp + tls->params.max_frame_len; - toep->tls.tx_key_info_size = tls_key_info_size(tls); + toep->tls.tx_key_info_size = t4_tls_key_info_size(tls); } else { /* Stop timer on handshake completion */ tls_stop_handshake_timer(toep); diff --git a/sys/dev/cxgbe/tom/t4_tls.h b/sys/dev/cxgbe/tom/t4_tls.h index 2770526704a..e5166bf6d9b 100644 --- a/sys/dev/cxgbe/tom/t4_tls.h +++ b/sys/dev/cxgbe/tom/t4_tls.h @@ -33,26 +33,8 @@ #ifndef __T4_TLS_H__ #define __T4_TLS_H__ -#define SALT_SIZE 4 - -/* Key Context Programming Operation type */ -#define KEY_WRITE_RX 0x1 -#define KEY_WRITE_TX 0x2 -#define KEY_DELETE_RX 0x4 -#define KEY_DELETE_TX 0x8 - -#define S_KEY_CLR_LOC 4 -#define M_KEY_CLR_LOC 0xf -#define V_KEY_CLR_LOC(x) ((x) << S_KEY_CLR_LOC) -#define G_KEY_CLR_LOC(x) (((x) >> S_KEY_CLR_LOC) & M_KEY_CLR_LOC) -#define F_KEY_CLR_LOC V_KEY_CLR_LOC(1U) - -#define S_KEY_GET_LOC 0 -#define M_KEY_GET_LOC 0xf -#define V_KEY_GET_LOC(x) ((x) << S_KEY_GET_LOC) -#define G_KEY_GET_LOC(x) (((x) >> S_KEY_GET_LOC) & M_KEY_GET_LOC) - #ifdef _KERNEL + /* Timeouts for handshake timer in seconds. */ #define TLS_SRV_HELLO_DONE 9 #define TLS_SRV_HELLO_RD_TM 5 @@ -70,8 +52,6 @@ #define TP_TX_PG_SZ 65536 #define FC_TP_PLEN_MAX 17408 -#define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_keyctx), 32) - enum { TLS_SFO_WR_CONTEXTLOC_DSGL, TLS_SFO_WR_CONTEXTLOC_IMMEDIATE, @@ -108,258 +88,6 @@ struct tls_ofld_info { struct callout handshake_timer; }; -struct tls_key_req { - __be32 wr_hi; - __be32 wr_mid; - __be32 ftid; - __u8 reneg_to_write_rx; - __u8 protocol; - __be16 mfs; - /* master command */ - __be32 cmd; - __be32 len16; /* command length */ - __be32 dlen; /* data length in 32-byte units */ - __be32 kaddr; - /* sub-command */ - __be32 sc_more; - __be32 sc_len; -}__packed; - -struct tls_keyctx { - union key_ctx { - struct tx_keyctx_hdr { - __u8 ctxlen; - __u8 r2; - __be16 dualck_to_txvalid; - __u8 txsalt[4]; - __be64 r5; - } txhdr; - struct rx_keyctx_hdr { - __u8 flitcnt_hmacctrl; - __u8 protover_ciphmode; - __u8 authmode_to_rxvalid; - __u8 ivpresent_to_rxmk_size; - __u8 rxsalt[4]; - __be64 ivinsert_to_authinsrt; - } rxhdr; - } u; - struct keys { - __u8 edkey[32]; - __u8 ipad[64]; - __u8 opad[64]; - } keys; -}; - -#define S_TLS_KEYCTX_TX_WR_DUALCK 12 -#define M_TLS_KEYCTX_TX_WR_DUALCK 0x1 -#define V_TLS_KEYCTX_TX_WR_DUALCK(x) ((x) << S_TLS_KEYCTX_TX_WR_DUALCK) -#define G_TLS_KEYCTX_TX_WR_DUALCK(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_DUALCK) & M_TLS_KEYCTX_TX_WR_DUALCK) -#define F_TLS_KEYCTX_TX_WR_DUALCK V_TLS_KEYCTX_TX_WR_DUALCK(1U) - -#define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11 -#define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1 -#define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) -#define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \ - M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) -#define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \ - V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U) - -#define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10 -#define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1 -#define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT) -#define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \ - M_TLS_KEYCTX_TX_WR_SALT_PRESENT) -#define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \ - V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U) - -#define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6 -#define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf -#define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE) -#define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \ - M_TLS_KEYCTX_TX_WR_TXCK_SIZE) - -#define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2 -#define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf -#define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE) -#define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \ - M_TLS_KEYCTX_TX_WR_TXMK_SIZE) - -#define S_TLS_KEYCTX_TX_WR_TXVALID 0 -#define M_TLS_KEYCTX_TX_WR_TXVALID 0x1 -#define V_TLS_KEYCTX_TX_WR_TXVALID(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_TXVALID) -#define G_TLS_KEYCTX_TX_WR_TXVALID(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID) -#define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U) - -#define S_TLS_KEYCTX_TX_WR_FLITCNT 3 -#define M_TLS_KEYCTX_TX_WR_FLITCNT 0x1f -#define V_TLS_KEYCTX_TX_WR_FLITCNT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_FLITCNT) -#define G_TLS_KEYCTX_TX_WR_FLITCNT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_FLITCNT) & M_TLS_KEYCTX_TX_WR_FLITCNT) - -#define S_TLS_KEYCTX_TX_WR_HMACCTRL 0 -#define M_TLS_KEYCTX_TX_WR_HMACCTRL 0x7 -#define V_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_HMACCTRL) -#define G_TLS_KEYCTX_TX_WR_HMACCTRL(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_HMACCTRL) & M_TLS_KEYCTX_TX_WR_HMACCTRL) - -#define S_TLS_KEYCTX_TX_WR_PROTOVER 4 -#define M_TLS_KEYCTX_TX_WR_PROTOVER 0xf -#define V_TLS_KEYCTX_TX_WR_PROTOVER(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_PROTOVER) -#define G_TLS_KEYCTX_TX_WR_PROTOVER(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_PROTOVER) & M_TLS_KEYCTX_TX_WR_PROTOVER) - -#define S_TLS_KEYCTX_TX_WR_CIPHMODE 0 -#define M_TLS_KEYCTX_TX_WR_CIPHMODE 0xf -#define V_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_CIPHMODE) -#define G_TLS_KEYCTX_TX_WR_CIPHMODE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_CIPHMODE) & M_TLS_KEYCTX_TX_WR_CIPHMODE) - -#define S_TLS_KEYCTX_TX_WR_AUTHMODE 4 -#define M_TLS_KEYCTX_TX_WR_AUTHMODE 0xf -#define V_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_AUTHMODE) -#define G_TLS_KEYCTX_TX_WR_AUTHMODE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_AUTHMODE) & M_TLS_KEYCTX_TX_WR_AUTHMODE) - -#define S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 3 -#define M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL 0x1 -#define V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) -#define G_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) & \ - M_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL) -#define F_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL \ - V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1U) - -#define S_TLS_KEYCTX_TX_WR_SEQNUMCTRL 1 -#define M_TLS_KEYCTX_TX_WR_SEQNUMCTRL 0x3 -#define V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) -#define G_TLS_KEYCTX_TX_WR_SEQNUMCTRL(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_SEQNUMCTRL) & \ - M_TLS_KEYCTX_TX_WR_SEQNUMCTRL) - -#define S_TLS_KEYCTX_TX_WR_RXVALID 0 -#define M_TLS_KEYCTX_TX_WR_RXVALID 0x1 -#define V_TLS_KEYCTX_TX_WR_RXVALID(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_RXVALID) -#define G_TLS_KEYCTX_TX_WR_RXVALID(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_RXVALID) & M_TLS_KEYCTX_TX_WR_RXVALID) -#define F_TLS_KEYCTX_TX_WR_RXVALID V_TLS_KEYCTX_TX_WR_RXVALID(1U) - -#define S_TLS_KEYCTX_TX_WR_IVPRESENT 7 -#define M_TLS_KEYCTX_TX_WR_IVPRESENT 0x1 -#define V_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_IVPRESENT) -#define G_TLS_KEYCTX_TX_WR_IVPRESENT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_IVPRESENT) & \ - M_TLS_KEYCTX_TX_WR_IVPRESENT) -#define F_TLS_KEYCTX_TX_WR_IVPRESENT V_TLS_KEYCTX_TX_WR_IVPRESENT(1U) - -#define S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 6 -#define M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT 0x1 -#define V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) -#define G_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) & \ - M_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT) -#define F_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT \ - V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1U) - -#define S_TLS_KEYCTX_TX_WR_RXCK_SIZE 3 -#define M_TLS_KEYCTX_TX_WR_RXCK_SIZE 0x7 -#define V_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_RXCK_SIZE) -#define G_TLS_KEYCTX_TX_WR_RXCK_SIZE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_RXCK_SIZE) & \ - M_TLS_KEYCTX_TX_WR_RXCK_SIZE) - -#define S_TLS_KEYCTX_TX_WR_RXMK_SIZE 0 -#define M_TLS_KEYCTX_TX_WR_RXMK_SIZE 0x7 -#define V_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_RXMK_SIZE) -#define G_TLS_KEYCTX_TX_WR_RXMK_SIZE(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_RXMK_SIZE) & \ - M_TLS_KEYCTX_TX_WR_RXMK_SIZE) - -#define S_TLS_KEYCTX_TX_WR_IVINSERT 55 -#define M_TLS_KEYCTX_TX_WR_IVINSERT 0x1ffULL -#define V_TLS_KEYCTX_TX_WR_IVINSERT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_IVINSERT) -#define G_TLS_KEYCTX_TX_WR_IVINSERT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_IVINSERT) & M_TLS_KEYCTX_TX_WR_IVINSERT) - -#define S_TLS_KEYCTX_TX_WR_AADSTRTOFST 47 -#define M_TLS_KEYCTX_TX_WR_AADSTRTOFST 0xffULL -#define V_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_AADSTRTOFST) -#define G_TLS_KEYCTX_TX_WR_AADSTRTOFST(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_AADSTRTOFST) & \ - M_TLS_KEYCTX_TX_WR_AADSTRTOFST) - -#define S_TLS_KEYCTX_TX_WR_AADSTOPOFST 39 -#define M_TLS_KEYCTX_TX_WR_AADSTOPOFST 0xffULL -#define V_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_AADSTOPOFST) -#define G_TLS_KEYCTX_TX_WR_AADSTOPOFST(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_AADSTOPOFST) & \ - M_TLS_KEYCTX_TX_WR_AADSTOPOFST) - -#define S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 30 -#define M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST 0x1ffULL -#define V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) -#define G_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) & \ - M_TLS_KEYCTX_TX_WR_CIPHERSRTOFST) - -#define S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 23 -#define M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST 0x7f -#define V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) -#define G_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) & \ - M_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST) - -#define S_TLS_KEYCTX_TX_WR_AUTHSRTOFST 14 -#define M_TLS_KEYCTX_TX_WR_AUTHSRTOFST 0x1ff -#define V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) -#define G_TLS_KEYCTX_TX_WR_AUTHSRTOFST(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSRTOFST) & \ - M_TLS_KEYCTX_TX_WR_AUTHSRTOFST) - -#define S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 7 -#define M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST 0x7f -#define V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) -#define G_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) & \ - M_TLS_KEYCTX_TX_WR_AUTHSTOPOFST) - -#define S_TLS_KEYCTX_TX_WR_AUTHINSRT 0 -#define M_TLS_KEYCTX_TX_WR_AUTHINSRT 0x7f -#define V_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ - ((x) << S_TLS_KEYCTX_TX_WR_AUTHINSRT) -#define G_TLS_KEYCTX_TX_WR_AUTHINSRT(x) \ - (((x) >> S_TLS_KEYCTX_TX_WR_AUTHINSRT) & \ - M_TLS_KEYCTX_TX_WR_AUTHINSRT) - struct tls_hdr { __u8 type; __be16 version;