diff --git a/crypto/fipsmodule/aes/asm/vpaes-armv7.pl b/crypto/fipsmodule/aes/asm/vpaes-armv7.pl index d147c9687..f9fadda99 100644 --- a/crypto/fipsmodule/aes/asm/vpaes-armv7.pl +++ b/crypto/fipsmodule/aes/asm/vpaes-armv7.pl @@ -1281,6 +1281,65 @@ vpaes_decrypt_key_to_bsaes: ___ } +{ +# Register-passed parameters. +my ($inp, $out, $len, $key) = map("r$_", 0..3); +# Temporaries. _vpaes_encrypt_core already uses r8..r11, so overlap $ivec and +# $tmp. $ctr is r7 because it must be preserved across calls. +my ($ctr, $ivec, $tmp) = map("r$_", 7..9); + +# void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, +# const AES_KEY *key, const uint8_t ivec[16]); +$code .= <<___; +.globl vpaes_ctr32_encrypt_blocks +.type vpaes_ctr32_encrypt_blocks,%function +.align 4 +vpaes_ctr32_encrypt_blocks: + mov ip, sp + stmdb sp!, {r7-r11, lr} + @ This function uses q4-q7 (d8-d15), which are callee-saved. + vstmdb sp!, {d8-d15} + + cmp $len, #0 + @ $ivec is passed on the stack. + ldr $ivec, [ip] + beq .Lctr32_done + + @ _vpaes_encrypt_core expects the key in r2, so swap $len and $key. + mov $tmp, $key + mov $key, $len + mov $len, $tmp +___ +my ($len, $key) = ($key, $len); +$code .= <<___; + + @ Load the IV and counter portion. + ldr $ctr, [$ivec, #12] + vld1.8 {q7}, [$ivec] + + bl _vpaes_preheat + rev $ctr, $ctr @ The counter is big-endian. + +.Lctr32_loop: + vmov q0, q7 + vld1.8 {q6}, [$inp]! @ Load input ahead of time + bl _vpaes_encrypt_core + veor q0, q0, q6 @ XOR input and result + vst1.8 {q0}, [$out]! + subs $len, $len, #1 + @ Update the counter. + add $ctr, $ctr, #1 + rev $tmp, $ctr + vmov.32 q7#hi[1], $tmp + bne .Lctr32_loop + +.Lctr32_done: + vldmia sp!, {d8-d15} + ldmia sp!, {r7-r11, pc} @ return +.size vpaes_ctr32_encrypt_blocks,.-vpaes_ctr32_encrypt_blocks +___ +} + foreach (split("\n",$code)) { s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo; print $_,"\n"; diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index 8471a808a..99d509a98 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h @@ -51,6 +51,7 @@ OPENSSL_INLINE int hwaes_capable(void) { return CRYPTO_is_ARMv8_AES_capable(); } #if defined(OPENSSL_ARM) #define BSAES #define VPAES +#define VPAES_CTR32 OPENSSL_INLINE int bsaes_capable(void) { return CRYPTO_is_NEON_capable(); } OPENSSL_INLINE int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); } #endif diff --git a/crypto/fipsmodule/cipher/e_aes.c b/crypto/fipsmodule/cipher/e_aes.c index 685d2db72..72910edb7 100644 --- a/crypto/fipsmodule/cipher/e_aes.c +++ b/crypto/fipsmodule/cipher/e_aes.c @@ -68,6 +68,48 @@ OPENSSL_MSVC_PRAGMA(warning(push)) OPENSSL_MSVC_PRAGMA(warning(disable: 4702)) // Unreachable code. +#if defined(BSAES) +static void vpaes_ctr32_encrypt_blocks_with_bsaes(const uint8_t *in, + uint8_t *out, size_t blocks, + const AES_KEY *key, + const uint8_t ivec[16]) { + // |bsaes_ctr32_encrypt_blocks| is faster than |vpaes_ctr32_encrypt_blocks|, + // but it takes at least one full 8-block batch to amortize the conversion. + if (blocks < 8) { + vpaes_ctr32_encrypt_blocks(in, out, blocks, key, ivec); + return; + } + + size_t bsaes_blocks = blocks; + if (bsaes_blocks % 8 < 6) { + // |bsaes_ctr32_encrypt_blocks| internally works in 8-block batches. If the + // final batch is too small (under six blocks), it is faster to loop over + // |vpaes_encrypt|. Round |bsaes_blocks| down to a multiple of 8. + bsaes_blocks -= bsaes_blocks % 8; + } + + AES_KEY bsaes; + vpaes_encrypt_key_to_bsaes(&bsaes, key); + bsaes_ctr32_encrypt_blocks(in, out, bsaes_blocks, &bsaes, ivec); + OPENSSL_cleanse(&bsaes, sizeof(bsaes)); + + in += 16 * bsaes_blocks; + out += 16 * bsaes_blocks; + blocks -= bsaes_blocks; + + union { + uint32_t u32[4]; + uint8_t u8[16]; + } new_ivec; + memcpy(new_ivec.u8, ivec, 16); + uint32_t ctr = CRYPTO_bswap4(new_ivec.u32[3]) + bsaes_blocks; + new_ivec.u32[3] = CRYPTO_bswap4(ctr); + + // Finish any remaining blocks with |vpaes_ctr32_encrypt_blocks|. + vpaes_ctr32_encrypt_blocks(in, out, blocks, key, new_ivec.u8); +} +#endif // BSAES + typedef struct { union { double align; @@ -110,6 +152,7 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, dat->stream.cbc = aes_hw_cbc_encrypt; } } else if (bsaes_capable() && mode == EVP_CIPH_CBC_MODE) { + assert(vpaes_capable()); ret = vpaes_set_decrypt_key(key, ctx->key_len * 8, &dat->ks.ks); if (ret == 0) { vpaes_decrypt_key_to_bsaes(&dat->ks.ks, &dat->ks.ks); @@ -145,11 +188,6 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, } else if (mode == EVP_CIPH_CTR_MODE) { dat->stream.ctr = aes_hw_ctr32_encrypt_blocks; } - } else if (bsaes_capable() && mode == EVP_CIPH_CTR_MODE) { - ret = aes_nohw_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); - // If |dat->stream.ctr| is provided, |dat->block| is never used. - dat->block = NULL; - dat->stream.ctr = bsaes_ctr32_encrypt_blocks; } else if (vpaes_capable()) { ret = vpaes_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = vpaes_encrypt; @@ -159,11 +197,14 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, dat->stream.cbc = vpaes_cbc_encrypt; } #endif -#if defined(VPAES_CTR32) if (mode == EVP_CIPH_CTR_MODE) { +#if defined(BSAES) + assert(bsaes_capable()); + dat->stream.ctr = vpaes_ctr32_encrypt_blocks_with_bsaes; +#elif defined(VPAES_CTR32) dat->stream.ctr = vpaes_ctr32_encrypt_blocks; - } #endif + } } else { ret = aes_nohw_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = aes_nohw_encrypt; @@ -252,17 +293,6 @@ ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_KEY *gcm_key, return aes_hw_ctr32_encrypt_blocks; } - if (bsaes_capable()) { - aes_nohw_set_encrypt_key(key, key_bytes * 8, aes_key); - if (gcm_key != NULL) { - CRYPTO_gcm128_init_key(gcm_key, aes_key, aes_nohw_encrypt, 0); - } - if (out_block) { - *out_block = aes_nohw_encrypt; - } - return bsaes_ctr32_encrypt_blocks; - } - if (vpaes_capable()) { vpaes_set_encrypt_key(key, key_bytes * 8, aes_key); if (out_block) { @@ -271,7 +301,10 @@ ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_KEY *gcm_key, if (gcm_key != NULL) { CRYPTO_gcm128_init_key(gcm_key, aes_key, vpaes_encrypt, 0); } -#if defined(VPAES_CTR32) +#if defined(BSAES) + assert(bsaes_capable()); + return vpaes_ctr32_encrypt_blocks_with_bsaes; +#elif defined(VPAES_CTR32) return vpaes_ctr32_encrypt_blocks; #else return NULL;