armv8crypto: add AES-XTS support

A straightforward(ish) port from aesni(4). This implementation does not
perform loop unrolling on the input blocks, so this is left as a future
performance improvement.

Submitted by:	Greg V <greg AT unrelenting.technology>
Looks good:	jhb, jmg
Tested by:	mhorne
Differential Revision:	https://reviews.freebsd.org/D21017

sys/crypto/armv8/armv8_crypto.c

@@ -114,7 +114,7 @@ armv8_crypto_probe(device_t dev)
 		break;
 	}
 
-	device_set_desc_copy(dev, "AES-CBC");
+	device_set_desc_copy(dev, "AES-CBC,AES-XTS");
 
 	/* TODO: Check more fields as we support more features */
 
@@ -204,6 +204,17 @@ armv8_crypto_probesession(device_t dev,
 				return (EINVAL);
 			}
 			break;
+		case CRYPTO_AES_XTS:
+			if (csp->csp_ivlen != AES_XTS_IV_LEN)
+				return (EINVAL);
+			switch (csp->csp_cipher_klen * 8) {
+			case 256:
+			case 512:
+				break;
+			default:
+				return (EINVAL);
+			}
+			break;
 		default:
 			return (EINVAL);
 		}
@@ -211,16 +222,19 @@ armv8_crypto_probesession(device_t dev,
 	default:
 		return (EINVAL);
 	}
-	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);		
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
 }
 
 static void
 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
-    const struct crypto_session_params *csp)
+    const struct crypto_session_params *csp, const uint8_t *key, int keylen)
 {
 	int i;
 
-	switch (csp->csp_cipher_klen * 8) {
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+		keylen /= 2;
+
+	switch (keylen * 8) {
 	case 128:
 		ses->rounds = AES128_ROUNDS;
 		break;
@@ -231,16 +245,19 @@ armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
 		ses->rounds = AES256_ROUNDS;
 		break;
 	default:
-		panic("invalid CBC key length");
+		panic("invalid AES key length");
 	}
 
-	rijndaelKeySetupEnc(ses->enc_schedule, csp->csp_cipher_key,
-	    csp->csp_cipher_klen * 8);
-	rijndaelKeySetupDec(ses->dec_schedule, csp->csp_cipher_key,
-	    csp->csp_cipher_klen * 8);
+	rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8);
+	rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8);
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+		rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen * 8);
+
 	for (i = 0; i < nitems(ses->enc_schedule); i++) {
 		ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
 		ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
+		if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+			ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]);
 	}
 }
 
@@ -259,7 +276,8 @@ armv8_crypto_newsession(device_t dev, crypto_session_t cses,
 	}
 
 	ses = crypto_get_driver_session(cses);
-	armv8_crypto_cipher_setup(ses, csp);
+	armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
+	    csp->csp_cipher_klen);
 	rw_wunlock(&sc->lock);
 	return (0);
 }
@@ -333,7 +351,8 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 	}
 
 	if (crp->crp_cipher_key != NULL) {
-		panic("armv8: new cipher key");
+		armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
+		    csp->csp_cipher_klen);
 	}
 
 	crypto_read_iv(crp, iv);
@@ -348,6 +367,16 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 			armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
 			    crp->crp_payload_length, buf, iv);
 		break;
+	case CRYPTO_AES_XTS:
+		if (encflag)
+			armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule,
+			    ses->xts_schedule, crp->crp_payload_length, buf,
+			    buf, iv);
+		else
+			armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule,
+			    ses->xts_schedule, crp->crp_payload_length, buf,
+			    buf, iv);
+		break;
 	}
 
 	if (allocated)

sys/crypto/armv8/armv8_crypto.h

@@ -40,6 +40,7 @@
 struct armv8_crypto_session {
 	uint32_t enc_schedule[AES_SCHED_LEN/4];
 	uint32_t dec_schedule[AES_SCHED_LEN/4];
+	uint32_t xts_schedule[AES_SCHED_LEN/4];
 	int algo;
 	int rounds;
 };
@@ -49,4 +50,9 @@ void armv8_aes_encrypt_cbc(int, const void *, size_t, const uint8_t *,
 void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *,
     const uint8_t[static AES_BLOCK_LEN]);
 
+void armv8_aes_encrypt_xts(int, const void *, const void *, size_t,
+    const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
+void armv8_aes_decrypt_xts(int, const void *, const void *, size_t,
+    const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
+
 #endif /* _ARMV8_CRYPTO_H_ */

sys/crypto/armv8/armv8_crypto_wrap.c

@@ -126,3 +126,95 @@ armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
 		buf += AES_BLOCK_LEN;
 	}
 }
+
+#define	AES_XTS_BLOCKSIZE	16
+#define	AES_XTS_IVSIZE		8
+#define	AES_XTS_ALPHA		0x87	/* GF(2^128) generator polynomial */
+
+static inline int32x4_t
+xts_crank_lfsr(int32x4_t inp)
+{
+	const int32x4_t alphamask = {AES_XTS_ALPHA, 1, 1, 1};
+	int32x4_t xtweak, ret;
+
+	/* set up xor mask */
+	xtweak = vextq_s32(inp, inp, 3);
+	xtweak = vshrq_n_s32(xtweak, 31);
+	xtweak &= alphamask;
+
+	/* next term */
+	ret = vshlq_n_s32(inp, 1);
+	ret ^= xtweak;
+
+	return ret;
+}
+
+static void
+armv8_aes_crypt_xts_block(int rounds, const uint8x16_t *key_schedule,
+    uint8x16_t *tweak, const uint8_t *from, uint8_t *to, int do_encrypt)
+{
+	uint8x16_t block;
+
+	block = vld1q_u8(from) ^ *tweak;
+
+	if (do_encrypt)
+		block = armv8_aes_enc(rounds - 1, key_schedule, block);
+	else
+		block = armv8_aes_dec(rounds - 1, key_schedule, block);
+
+	vst1q_u8(to, block ^ *tweak);
+
+	*tweak = vreinterpretq_u8_s32(xts_crank_lfsr(vreinterpretq_s32_u8(*tweak)));
+}
+
+static void
+armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule,
+    const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from,
+    uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt)
+{
+	uint8x16_t tweakreg;
+	uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16);
+	size_t i, cnt;
+
+	/*
+	 * Prepare tweak as E_k2(IV). IV is specified as LE representation
+	 * of a 64-bit block number which we allow to be passed in directly.
+	 */
+#if BYTE_ORDER == LITTLE_ENDIAN
+	bcopy(iv, tweak, AES_XTS_IVSIZE);
+	/* Last 64 bits of IV are always zero. */
+	bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
+#else
+#error Only LITTLE_ENDIAN architectures are supported.
+#endif
+	tweakreg = vld1q_u8(tweak);
+	tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg);
+
+	cnt = len / AES_XTS_BLOCKSIZE;
+	for (i = 0; i < cnt; i++) {
+		armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg,
+		    from, to, do_encrypt);
+		from += AES_XTS_BLOCKSIZE;
+		to += AES_XTS_BLOCKSIZE;
+	}
+}
+
+void
+armv8_aes_encrypt_xts(int rounds, const void *data_schedule,
+    const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+    const uint8_t iv[static AES_BLOCK_LEN])
+{
+
+	armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
+	    iv, 1);
+}
+
+void
+armv8_aes_decrypt_xts(int rounds, const void *data_schedule,
+    const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
+    const uint8_t iv[static AES_BLOCK_LEN])
+{
+
+	armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
+	    iv, 0);
+}