diff --git a/crypto/fipsmodule/aes/aes.c b/crypto/fipsmodule/aes/aes.c index f654cb1e7..a9c20d202 100644 --- a/crypto/fipsmodule/aes/aes.c +++ b/crypto/fipsmodule/aes/aes.c @@ -534,8 +534,8 @@ static const uint32_t rcon[] = { // for 128-bit blocks, Rijndael never uses more than 10 rcon values }; -static int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits, - AES_KEY *aeskey) { +int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits, + AES_KEY *aeskey) { uint32_t *rk; int i = 0; uint32_t temp; @@ -630,8 +630,8 @@ static int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits, return 0; } -static int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits, - AES_KEY *aeskey) { +int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits, + AES_KEY *aeskey) { uint32_t *rk; int i, j, status; uint32_t temp; @@ -679,8 +679,7 @@ static int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits, return 0; } -static void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, - const AES_KEY *key) { +void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { const uint32_t *rk; uint32_t s0, s1, s2, s3, t0, t1, t2, t3; int r; @@ -741,8 +740,7 @@ static void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, PUTU32(out + 12, s3); } -static void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, - const AES_KEY *key) { +void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { const uint32_t *rk; uint32_t s0, s1, s2, s3, t0, t1, t2, t3; int r; @@ -808,17 +806,7 @@ static void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, PUTU32(out + 12, s3); } -#else // NO_ASM || (!X86 && !X86_64 && !ARM) - -// If not implemented in C, these functions will be provided by assembly code. -void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); -void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); -int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits, - AES_KEY *aeskey); -int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits, - AES_KEY *aeskey); - -#endif +#endif // NO_ASM || (!X86 && !X86_64 && !ARM) // Be aware that on x86(-64), the |aes_nohw_*| functions are incompatible with // the aes_hw_* functions. The latter set |AES_KEY.rounds| to one less than the diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index 5f9ee3126..a91ea708e 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h @@ -30,19 +30,34 @@ extern "C" { #define HWAES #define HWAES_ECB -static int hwaes_capable(void) { +OPENSSL_INLINE int hwaes_capable(void) { return (OPENSSL_ia32cap_get()[1] & (1 << (57 - 32))) != 0; } + +#define VPAES +OPENSSL_INLINE char vpaes_capable(void) { + return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0; +} + +#if defined(OPENSSL_X86_64) +#define BSAES +OPENSSL_INLINE char bsaes_capable(void) { return vpaes_capable(); } +#endif // X86_64 + #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) #define HWAES -static int hwaes_capable(void) { - return CRYPTO_is_ARMv8_AES_capable(); -} +OPENSSL_INLINE int hwaes_capable(void) { return CRYPTO_is_ARMv8_AES_capable(); } + +#if defined(OPENSSL_ARM) && __ARM_MAX_ARCH__ >= 7 +#define BSAES +OPENSSL_INLINE char bsaes_capable(void) { return CRYPTO_is_NEON_capable(); } +#endif + #elif defined(OPENSSL_PPC64LE) #define HWAES -static int hwaes_capable(void) { +OPENSSL_INLINE int hwaes_capable(void) { return CRYPTO_is_PPC64LE_vcrypto_capable(); } #endif @@ -67,36 +82,37 @@ void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, // If HWAES isn't defined then we provide dummy functions for each of the hwaes // functions. -static int hwaes_capable(void) { return 0; } +OPENSSL_INLINE int hwaes_capable(void) { return 0; } -static int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits, - AES_KEY *key) { +OPENSSL_INLINE int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits, + AES_KEY *key) { abort(); } -static int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits, - AES_KEY *key) { +OPENSSL_INLINE int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits, + AES_KEY *key) { abort(); } -static void aes_hw_encrypt(const uint8_t *in, uint8_t *out, - const AES_KEY *key) { +OPENSSL_INLINE void aes_hw_encrypt(const uint8_t *in, uint8_t *out, + const AES_KEY *key) { abort(); } -static void aes_hw_decrypt(const uint8_t *in, uint8_t *out, - const AES_KEY *key) { +OPENSSL_INLINE void aes_hw_decrypt(const uint8_t *in, uint8_t *out, + const AES_KEY *key) { abort(); } -static void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, uint8_t *ivec, int enc) { +OPENSSL_INLINE void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, + size_t length, const AES_KEY *key, + uint8_t *ivec, int enc) { abort(); } -static void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, - size_t len, const AES_KEY *key, - const uint8_t ivec[16]) { +OPENSSL_INLINE void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, + size_t len, const AES_KEY *key, + const uint8_t ivec[16]) { abort(); } @@ -106,8 +122,90 @@ static void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, #if defined(HWAES_ECB) void aes_hw_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length, const AES_KEY *key, const int enc); +#endif // HWAES_ECB + + +#if defined(BSAES) +// On platforms where BSAES gets defined (just above), then these functions are +// provided by asm. +void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, + const AES_KEY *key, uint8_t ivec[16], int enc); +void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, const uint8_t ivec[16]); +#else +OPENSSL_INLINE char bsaes_capable(void) { return 0; } + +// On other platforms, bsaes_capable() will always return false and so the +// following will never be called. +OPENSSL_INLINE void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, + size_t length, const AES_KEY *key, + uint8_t ivec[16], int enc) { + abort(); +} + +OPENSSL_INLINE void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, + size_t len, const AES_KEY *key, + const uint8_t ivec[16]) { + abort(); +} +#endif // !BSAES + + +#if defined(VPAES) +// On platforms where VPAES gets defined (just above), then these functions are +// provided by asm. +int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); +int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); + +void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); +void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); + +void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, + const AES_KEY *key, uint8_t *ivec, int enc); +#else +OPENSSL_INLINE char vpaes_capable(void) { return 0; } + +// On other platforms, vpaes_capable() will always return false and so the +// following will never be called. +OPENSSL_INLINE int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, + AES_KEY *key) { + abort(); +} +OPENSSL_INLINE int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, + AES_KEY *key) { + abort(); +} +OPENSSL_INLINE void vpaes_encrypt(const uint8_t *in, uint8_t *out, + const AES_KEY *key) { + abort(); +} +OPENSSL_INLINE void vpaes_decrypt(const uint8_t *in, uint8_t *out, + const AES_KEY *key) { + abort(); +} +OPENSSL_INLINE void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, + size_t length, const AES_KEY *key, + uint8_t *ivec, int enc) { + abort(); +} +#endif // !VPAES + + +void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); +void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); +int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits, + AES_KEY *aeskey); +int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits, + AES_KEY *aeskey); + +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) +#define AES_NOHW_CBC +void aes_nohw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t *ivec, const int enc); #endif + #if defined(__cplusplus) } // extern C #endif diff --git a/crypto/fipsmodule/aes/mode_wrappers.c b/crypto/fipsmodule/aes/mode_wrappers.c index 0140c70fa..36c657e43 100644 --- a/crypto/fipsmodule/aes/mode_wrappers.c +++ b/crypto/fipsmodule/aes/mode_wrappers.c @@ -72,11 +72,6 @@ void AES_ecb_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key, } } -#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) -void aes_nohw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t *ivec, const int enc); -#endif - void AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, const AES_KEY *key, uint8_t *ivec, const int enc) { if (hwaes_capable()) { @@ -84,8 +79,7 @@ void AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, return; } -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) +#if defined(AES_NOHW_CBC) aes_nohw_cbc_encrypt(in, out, len, key, ivec, enc); #else if (enc) { diff --git a/crypto/fipsmodule/cipher/e_aes.c b/crypto/fipsmodule/cipher/e_aes.c index 2ccec4420..f7c145b9d 100644 --- a/crypto/fipsmodule/cipher/e_aes.c +++ b/crypto/fipsmodule/cipher/e_aes.c @@ -98,97 +98,6 @@ typedef struct { ctr128_f ctr; } EVP_AES_GCM_CTX; -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) -#define VPAES -static char vpaes_capable(void) { - return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0; -} - -#if defined(OPENSSL_X86_64) -#define BSAES -static char bsaes_capable(void) { - return vpaes_capable(); -} -#endif - -#elif !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)) - -#if defined(OPENSSL_ARM) && __ARM_MAX_ARCH__ >= 7 -#define BSAES -static char bsaes_capable(void) { - return CRYPTO_is_NEON_capable(); -} -#endif - -#endif - - -#if defined(BSAES) -// On platforms where BSAES gets defined (just above), then these functions are -// provided by asm. -void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, uint8_t ivec[16], int enc); -void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, const uint8_t ivec[16]); -#else -static char bsaes_capable(void) { - return 0; -} - -// On other platforms, bsaes_capable() will always return false and so the -// following will never be called. -static void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, uint8_t ivec[16], int enc) { - abort(); -} - -static void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, - size_t len, const AES_KEY *key, - const uint8_t ivec[16]) { - abort(); -} -#endif - -#if defined(VPAES) -// On platforms where VPAES gets defined (just above), then these functions are -// provided by asm. -int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); -int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); - -void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); -void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); - -void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, uint8_t *ivec, int enc); -#else -static char vpaes_capable(void) { - return 0; -} - -// On other platforms, vpaes_capable() will always return false and so the -// following will never be called. -static int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, - AES_KEY *key) { - abort(); -} -static int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, - AES_KEY *key) { - abort(); -} -static void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { - abort(); -} -static void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { - abort(); -} -static void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, uint8_t *ivec, int enc) { - abort(); -} -#endif - static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, const uint8_t *iv, int enc) { int ret, mode; diff --git a/util/fipstools/delocate/delocate.go b/util/fipstools/delocate/delocate.go index 473428533..ce1a8b2d0 100644 --- a/util/fipstools/delocate/delocate.go +++ b/util/fipstools/delocate/delocate.go @@ -1216,6 +1216,9 @@ func transform(w stringWriter, inputs []inputFile) error { // .file directive. Zero is not a valid number. maxObservedFileNumber := 0 + // OPENSSL_ia32cap_get will be synthesized by this script. + symbols["OPENSSL_ia32cap_get"] = struct{}{} + for _, input := range inputs { forEachPath(input.ast.up, func(node *node32) { symbol := input.contents[node.begin:node.end] @@ -1394,6 +1397,8 @@ func transform(w stringWriter, inputs []inputFile) error { } w.WriteString(".type OPENSSL_ia32cap_get, @function\n") + w.WriteString(".globl OPENSSL_ia32cap_get\n") + w.WriteString(localTargetName("OPENSSL_ia32cap_get") + ":\n") w.WriteString("OPENSSL_ia32cap_get:\n") w.WriteString("\tleaq OPENSSL_ia32cap_P(%rip), %rax\n") w.WriteString("\tret\n") diff --git a/util/fipstools/delocate/testdata/x86_64-BSS/out.s b/util/fipstools/delocate/testdata/x86_64-BSS/out.s index 5c576d956..e8cd56ead 100644 --- a/util/fipstools/delocate/testdata/x86_64-BSS/out.s +++ b/util/fipstools/delocate/testdata/x86_64-BSS/out.s @@ -66,6 +66,8 @@ z_bss_get: leaq .Lz_local_target(%rip), %rax ret .type OPENSSL_ia32cap_get, @function +.globl OPENSSL_ia32cap_get +.LOPENSSL_ia32cap_get_local_target: OPENSSL_ia32cap_get: leaq OPENSSL_ia32cap_P(%rip), %rax ret diff --git a/util/fipstools/delocate/testdata/x86_64-Basic/out.s b/util/fipstools/delocate/testdata/x86_64-Basic/out.s index b76fd3fa2..23e97c8ba 100644 --- a/util/fipstools/delocate/testdata/x86_64-Basic/out.s +++ b/util/fipstools/delocate/testdata/x86_64-Basic/out.s @@ -59,6 +59,8 @@ foo: .loc 2 2 0 BORINGSSL_bcm_text_end: .type OPENSSL_ia32cap_get, @function +.globl OPENSSL_ia32cap_get +.LOPENSSL_ia32cap_get_local_target: OPENSSL_ia32cap_get: leaq OPENSSL_ia32cap_P(%rip), %rax ret diff --git a/util/fipstools/delocate/testdata/x86_64-GOTRewrite/out.s b/util/fipstools/delocate/testdata/x86_64-GOTRewrite/out.s index 3d421e566..273cfe940 100644 --- a/util/fipstools/delocate/testdata/x86_64-GOTRewrite/out.s +++ b/util/fipstools/delocate/testdata/x86_64-GOTRewrite/out.s @@ -148,7 +148,7 @@ foo: # WAS movq foobar_bss_get@GOTPCREL(%rip), %r11 leaq foobar_bss_get(%rip), %r11 # WAS movq OPENSSL_ia32cap_get@GOTPCREL(%rip), %r11 - leaq OPENSSL_ia32cap_get(%rip), %r11 + leaq .LOPENSSL_ia32cap_get_local_target(%rip), %r11 # Transforming moves run the transform in-place after the load. # WAS vpbroadcastq stderr@GOTPCREL(%rip), %xmm0 @@ -186,6 +186,8 @@ stderr_GOTPCREL_external: .long stderr@GOTPCREL .long 0 .type OPENSSL_ia32cap_get, @function +.globl OPENSSL_ia32cap_get +.LOPENSSL_ia32cap_get_local_target: OPENSSL_ia32cap_get: leaq OPENSSL_ia32cap_P(%rip), %rax ret diff --git a/util/fipstools/delocate/testdata/x86_64-LabelRewrite/out.s b/util/fipstools/delocate/testdata/x86_64-LabelRewrite/out.s index 4a0185373..8cc27de83 100644 --- a/util/fipstools/delocate/testdata/x86_64-LabelRewrite/out.s +++ b/util/fipstools/delocate/testdata/x86_64-LabelRewrite/out.s @@ -39,7 +39,7 @@ bar: # Synthesized symbols are treated as local ones. # WAS call OPENSSL_ia32cap_get@PLT - call OPENSSL_ia32cap_get + call .LOPENSSL_ia32cap_get_local_target # References to local labels are left as-is in the first file. .Llocal_label: @@ -90,6 +90,8 @@ BORINGSSL_bcm_text_end: bcm_redirector_memcpy: jmp memcpy@PLT .type OPENSSL_ia32cap_get, @function +.globl OPENSSL_ia32cap_get +.LOPENSSL_ia32cap_get_local_target: OPENSSL_ia32cap_get: leaq OPENSSL_ia32cap_P(%rip), %rax ret diff --git a/util/fipstools/delocate/testdata/x86_64-Sections/out.s b/util/fipstools/delocate/testdata/x86_64-Sections/out.s index ba427ade8..ab3e3d9d7 100644 --- a/util/fipstools/delocate/testdata/x86_64-Sections/out.s +++ b/util/fipstools/delocate/testdata/x86_64-Sections/out.s @@ -48,6 +48,8 @@ foo: .loc 1 2 0 BORINGSSL_bcm_text_end: .type OPENSSL_ia32cap_get, @function +.globl OPENSSL_ia32cap_get +.LOPENSSL_ia32cap_get_local_target: OPENSSL_ia32cap_get: leaq OPENSSL_ia32cap_P(%rip), %rax ret