From 10c24cb433e54f874574fec480c0f60eaf598d31 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Mon, 27 Nov 2023 14:42:36 -0800 Subject: [PATCH] sha: Move AArch64/X86-64 dispatching to C. Take a step towards removing all dispatching logic from assembly for AArch64 and X86-64. Change-Id: I1c965012e81837ff228c810d54e730c525cad54f Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/64208 Reviewed-by: Bob Beck Commit-Queue: David Benjamin Reviewed-by: David Benjamin --- crypto/fipsmodule/sha/asm/sha1-armv8.pl | 26 ++-- crypto/fipsmodule/sha/asm/sha1-x86_64.pl | 71 ++++------ crypto/fipsmodule/sha/asm/sha512-armv8.pl | 42 ++---- crypto/fipsmodule/sha/asm/sha512-x86_64.pl | 69 +++------- crypto/fipsmodule/sha/internal.h | 143 ++++++++++++++++++++- crypto/fipsmodule/sha/sha1.c | 37 +++++- crypto/fipsmodule/sha/sha256.c | 38 +++++- crypto/fipsmodule/sha/sha512.c | 31 ++++- crypto/fipsmodule/sha/sha_test.cc | 95 +++++++++++--- crypto/internal.h | 44 +++++++ 10 files changed, 416 insertions(+), 180 deletions(-) diff --git a/crypto/fipsmodule/sha/asm/sha1-armv8.pl b/crypto/fipsmodule/sha/asm/sha1-armv8.pl index 0a7a2fcd9..2eccfb732 100644 --- a/crypto/fipsmodule/sha/asm/sha1-armv8.pl +++ b/crypto/fipsmodule/sha/asm/sha1-armv8.pl @@ -175,22 +175,12 @@ $code.=<<___; .text -.extern OPENSSL_armcap_P -.hidden OPENSSL_armcap_P -.globl sha1_block_data_order -.type sha1_block_data_order,%function +.globl sha1_block_data_order_nohw +.type sha1_block_data_order_nohw,%function .align 6 -sha1_block_data_order: +sha1_block_data_order_nohw: // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET -#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10 - adrp x16,:pg_hi21_nc:OPENSSL_armcap_P -#else - adrp x16,:pg_hi21:OPENSSL_armcap_P -#endif - ldr w16,[x16,:lo12:OPENSSL_armcap_P] - tst w16,#ARMV8_SHA1 - b.ne .Lv8_entry stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -239,7 +229,7 @@ $code.=<<___; ldp x27,x28,[sp,#80] ldr x29,[sp],#96 ret -.size sha1_block_data_order,.-sha1_block_data_order +.size sha1_block_data_order_nohw,.-sha1_block_data_order_nohw ___ {{{ my ($ABCD,$E,$E0,$E1)=map("v$_.16b",(0..3)); @@ -249,12 +239,12 @@ my ($W0,$W1)=("v20.4s","v21.4s"); my $ABCD_SAVE="v22.16b"; $code.=<<___; -.type sha1_block_armv8,%function +.globl sha1_block_data_order_hw +.type sha1_block_data_order_hw,%function .align 6 -sha1_block_armv8: +sha1_block_data_order_hw: // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. AARCH64_VALID_CALL_TARGET -.Lv8_entry: stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -318,7 +308,7 @@ $code.=<<___; ldr x29,[sp],#16 ret -.size sha1_block_armv8,.-sha1_block_armv8 +.size sha1_block_data_order_hw,.-sha1_block_data_order_hw .section .rodata .align 6 .Lconst: diff --git a/crypto/fipsmodule/sha/asm/sha1-x86_64.pl b/crypto/fipsmodule/sha/asm/sha1-x86_64.pl index c2c8921af..886f5cf54 100755 --- a/crypto/fipsmodule/sha/asm/sha1-x86_64.pl +++ b/crypto/fipsmodule/sha/asm/sha1-x86_64.pl @@ -237,42 +237,13 @@ push(@xi,shift(@xi)); $code.=<<___; .text -.extern OPENSSL_ia32cap_P -.globl sha1_block_data_order -.type sha1_block_data_order,\@function,3 +.globl sha1_block_data_order_nohw +.type sha1_block_data_order_nohw,\@function,3 .align 16 -sha1_block_data_order: +sha1_block_data_order_nohw: .cfi_startproc _CET_ENDBR - leaq OPENSSL_ia32cap_P(%rip),%r10 - mov 0(%r10),%r9d - mov 4(%r10),%r8d - mov 8(%r10),%r10d - test \$`1<<9`,%r8d # check SSSE3 bit - jz .Lialu -___ -$code.=<<___ if ($shaext); - test \$`1<<29`,%r10d # check SHA bit - jnz _shaext_shortcut -___ -$code.=<<___ if ($avx>1); - and \$`1<<3|1<<5|1<<8`,%r10d # check AVX2+BMI1+BMI2 - cmp \$`1<<3|1<<5|1<<8`,%r10d - je _avx2_shortcut -___ -$code.=<<___ if ($avx); - and \$`1<<28`,%r8d # mask AVX bit - and \$`1<<30`,%r9d # mask "Intel CPU" bit - or %r9d,%r8d - cmp \$`1<<28|1<<30`,%r8d - je _avx_shortcut -___ -$code.=<<___; - jmp _ssse3_shortcut - -.align 16 -.Lialu: mov %rsp,%rax .cfi_def_cfa_register %rax push %rbx @@ -341,7 +312,7 @@ $code.=<<___; .Lepilogue: ret .cfi_endproc -.size sha1_block_data_order,.-sha1_block_data_order +.size sha1_block_data_order_nohw,.-sha1_block_data_order_nohw ___ if ($shaext) {{{ ###################################################################### @@ -352,11 +323,12 @@ my ($ABCD,$E,$E_,$BSWAP,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(0..3,8,9)); my @MSG=map("%xmm$_",(4..7)); $code.=<<___; -.type sha1_block_data_order_shaext,\@function,3 +.globl sha1_block_data_order_hw +.type sha1_block_data_order_hw,\@function,3 .align 32 -sha1_block_data_order_shaext: -_shaext_shortcut: +sha1_block_data_order_hw: .cfi_startproc + _CET_ENDBR ___ $code.=<<___ if ($win64); lea `-8-4*16`(%rsp),%rsp @@ -457,7 +429,7 @@ ___ $code.=<<___; ret .cfi_endproc -.size sha1_block_data_order_shaext,.-sha1_block_data_order_shaext +.size sha1_block_data_order_hw,.-sha1_block_data_order_hw ___ }}} {{{ @@ -487,11 +459,12 @@ ___ } $code.=<<___; +.globl sha1_block_data_order_ssse3 .type sha1_block_data_order_ssse3,\@function,3 .align 16 sha1_block_data_order_ssse3: -_ssse3_shortcut: .cfi_startproc + _CET_ENDBR mov %rsp,$fp # frame pointer .cfi_def_cfa_register $fp push %rbx @@ -961,11 +934,12 @@ my $_rol=sub { &shld(@_[0],@_) }; my $_ror=sub { &shrd(@_[0],@_) }; $code.=<<___; +.globl sha1_block_data_order_avx .type sha1_block_data_order_avx,\@function,3 .align 16 sha1_block_data_order_avx: -_avx_shortcut: .cfi_startproc + _CET_ENDBR mov %rsp,$fp .cfi_def_cfa_register $fp push %rbx @@ -1340,11 +1314,12 @@ my $rx=0; my $frame="%r13"; $code.=<<___; +.globl sha1_block_data_order_avx2 .type sha1_block_data_order_avx2,\@function,3 .align 16 sha1_block_data_order_avx2: -_avx2_shortcut: .cfi_startproc + _CET_ENDBR mov %rsp,$fp .cfi_def_cfa_register $fp push %rbx @@ -2019,14 +1994,14 @@ ssse3_handler: .section .pdata .align 4 - .rva .LSEH_begin_sha1_block_data_order - .rva .LSEH_end_sha1_block_data_order - .rva .LSEH_info_sha1_block_data_order + .rva .LSEH_begin_sha1_block_data_order_nohw + .rva .LSEH_end_sha1_block_data_order_nohw + .rva .LSEH_info_sha1_block_data_order_nohw ___ $code.=<<___ if ($shaext); - .rva .LSEH_begin_sha1_block_data_order_shaext - .rva .LSEH_end_sha1_block_data_order_shaext - .rva .LSEH_info_sha1_block_data_order_shaext + .rva .LSEH_begin_sha1_block_data_order_hw + .rva .LSEH_end_sha1_block_data_order_hw + .rva .LSEH_info_sha1_block_data_order_hw ___ $code.=<<___; .rva .LSEH_begin_sha1_block_data_order_ssse3 @@ -2046,12 +2021,12 @@ ___ $code.=<<___; .section .xdata .align 8 -.LSEH_info_sha1_block_data_order: +.LSEH_info_sha1_block_data_order_nohw: .byte 9,0,0,0 .rva se_handler ___ $code.=<<___ if ($shaext); -.LSEH_info_sha1_block_data_order_shaext: +.LSEH_info_sha1_block_data_order_hw: .byte 9,0,0,0 .rva shaext_handler ___ diff --git a/crypto/fipsmodule/sha/asm/sha512-armv8.pl b/crypto/fipsmodule/sha/asm/sha512-armv8.pl index c7d91540c..0ec0f5132 100644 --- a/crypto/fipsmodule/sha/asm/sha512-armv8.pl +++ b/crypto/fipsmodule/sha/asm/sha512-armv8.pl @@ -76,7 +76,7 @@ if ($flavour && $flavour ne "void") { *STDOUT=*OUT; } -$func="sha${BITS}_block_data_order"; +$func="sha${BITS}_block_data_order_nohw"; ($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30)); @@ -180,31 +180,10 @@ $code.=<<___; .text -.extern OPENSSL_armcap_P -.hidden OPENSSL_armcap_P .globl $func .type $func,%function .align 6 $func: - AARCH64_VALID_CALL_TARGET -#ifndef __KERNEL__ -#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10 - adrp x16,:pg_hi21_nc:OPENSSL_armcap_P -#else - adrp x16,:pg_hi21:OPENSSL_armcap_P -#endif - ldr w16,[x16,:lo12:OPENSSL_armcap_P] -___ -$code.=<<___ if ($SZ==4); - tst w16,#ARMV8_SHA256 - b.ne .Lv8_entry -___ -$code.=<<___ if ($SZ==8); - tst w16,#ARMV8_SHA512 - b.ne .Lv8_entry -___ -$code.=<<___; -#endif AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -356,11 +335,12 @@ my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b"); $code.=<<___; .text #ifndef __KERNEL__ -.type sha256_block_armv8,%function +.globl sha256_block_data_order_hw +.type sha256_block_data_order_hw,%function .align 6 -sha256_block_armv8: -.Lv8_entry: +sha256_block_data_order_hw: // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -425,7 +405,7 @@ $code.=<<___; ldr x29,[sp],#16 ret -.size sha256_block_armv8,.-sha256_block_armv8 +.size sha256_block_data_order_hw,.-sha256_block_data_order_hw #endif ___ } @@ -442,10 +422,12 @@ my ($AB,$CD,$EF,$GH)=map("v$_.16b",(26..29)); $code.=<<___; .text #ifndef __KERNEL__ -.type sha512_block_armv8,%function +.globl sha512_block_data_order_hw +.type sha512_block_data_order_hw,%function .align 6 -sha512_block_armv8: -.Lv8_entry: +sha512_block_data_order_hw: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -529,7 +511,7 @@ $code.=<<___; ldr x29,[sp],#16 ret -.size sha512_block_armv8,.-sha512_block_armv8 +.size sha512_block_data_order_hw,.-sha512_block_data_order_hw #endif ___ } diff --git a/crypto/fipsmodule/sha/asm/sha512-x86_64.pl b/crypto/fipsmodule/sha/asm/sha512-x86_64.pl index a3e4122ac..45bb81ccb 100755 --- a/crypto/fipsmodule/sha/asm/sha512-x86_64.pl +++ b/crypto/fipsmodule/sha/asm/sha512-x86_64.pl @@ -257,42 +257,12 @@ ___ $code=<<___; .text -.extern OPENSSL_ia32cap_P -.globl $func -.type $func,\@function,3 +.globl ${func}_nohw +.type ${func}_nohw,\@function,3 .align 16 -$func: +${func}_nohw: .cfi_startproc _CET_ENDBR -___ -$code.=<<___ if ($SZ==4 || $avx); - leaq OPENSSL_ia32cap_P(%rip),%r11 - mov 0(%r11),%r9d - mov 4(%r11),%r10d - mov 8(%r11),%r11d -___ -$code.=<<___ if ($SZ==4 && $shaext); - test \$`1<<29`,%r11d # check for SHA - jnz .Lshaext_shortcut -___ - # XOP codepath removed. -$code.=<<___ if ($avx>1); - and \$`1<<8|1<<5|1<<3`,%r11d # check for BMI2+AVX2+BMI1 - cmp \$`1<<8|1<<5|1<<3`,%r11d - je .Lavx2_shortcut -___ -$code.=<<___ if ($avx); - and \$`1<<30`,%r9d # mask "Intel CPU" bit - and \$`1<<28|1<<9`,%r10d # mask AVX and SSSE3 bits - or %r9d,%r10d - cmp \$`1<<28|1<<9|1<<30`,%r10d - je .Lavx_shortcut -___ -$code.=<<___ if ($SZ==4); - test \$`1<<9`,%r10d - jnz .Lssse3_shortcut -___ -$code.=<<___; mov %rsp,%rax # copy %rsp .cfi_def_cfa_register %rax push %rbx @@ -400,7 +370,7 @@ $code.=<<___; .Lepilogue: ret .cfi_endproc -.size $func,.-$func +.size ${func}_nohw,.-${func}_nohw ___ if ($SZ==4) { @@ -558,11 +528,12 @@ my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..2,7..10) my @MSG=map("%xmm$_",(3..6)); $code.=<<___; -.type sha256_block_data_order_shaext,\@function,3 +.globl sha256_block_data_order_hw +.type sha256_block_data_order_hw,\@function,3 .align 64 -sha256_block_data_order_shaext: +sha256_block_data_order_hw: .cfi_startproc -.Lshaext_shortcut: + _CET_ENDBR ___ $code.=<<___ if ($win64); lea `-8-5*16`(%rsp),%rsp @@ -707,7 +678,7 @@ ___ $code.=<<___; ret .cfi_endproc -.size sha256_block_data_order_shaext,.-sha256_block_data_order_shaext +.size sha256_block_data_order_hw,.-sha256_block_data_order_hw ___ }}} {{{ @@ -772,11 +743,12 @@ my @X = map("%xmm$_",(0..3)); my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9)); $code.=<<___; +.globl ${func}_ssse3 .type ${func}_ssse3,\@function,3 .align 64 ${func}_ssse3: .cfi_startproc -.Lssse3_shortcut: + _CET_ENDBR mov %rsp,%rax # copy %rsp .cfi_def_cfa_register %rax push %rbx @@ -1135,11 +1107,12 @@ if ($avx) {{ local *ror = sub { &shrd(@_[0],@_) }; $code.=<<___; +.globl ${func}_avx .type ${func}_avx,\@function,3 .align 64 ${func}_avx: .cfi_startproc -.Lavx_shortcut: + _CET_ENDBR mov %rsp,%rax # copy %rsp .cfi_def_cfa_register %rax push %rbx @@ -2005,14 +1978,14 @@ ___ $code.=<<___; .section .pdata .align 4 - .rva .LSEH_begin_$func - .rva .LSEH_end_$func - .rva .LSEH_info_$func + .rva .LSEH_begin_${func}_nohw + .rva .LSEH_end_${func}_nohw + .rva .LSEH_info_${func}_nohw ___ $code.=<<___ if ($SZ==4 && $shaext); - .rva .LSEH_begin_${func}_shaext - .rva .LSEH_end_${func}_shaext - .rva .LSEH_info_${func}_shaext + .rva .LSEH_begin_${func}_hw + .rva .LSEH_end_${func}_hw + .rva .LSEH_info_${func}_hw ___ $code.=<<___ if ($SZ==4); .rva .LSEH_begin_${func}_ssse3 @@ -2032,13 +2005,13 @@ ___ $code.=<<___; .section .xdata .align 8 -.LSEH_info_$func: +.LSEH_info_${func}_nohw: .byte 9,0,0,0 .rva se_handler .rva .Lprologue,.Lepilogue # HandlerData[] ___ $code.=<<___ if ($SZ==4 && $shaext); -.LSEH_info_${func}_shaext: +.LSEH_info_${func}_hw: .byte 9,0,0,0 .rva shaext_handler ___ diff --git a/crypto/fipsmodule/sha/internal.h b/crypto/fipsmodule/sha/internal.h index 605f1665b..7dbab6bd4 100644 --- a/crypto/fipsmodule/sha/internal.h +++ b/crypto/fipsmodule/sha/internal.h @@ -17,25 +17,156 @@ #include +#include "../../internal.h" + #if defined(__cplusplus) extern "C" { #endif +// Define SHA{n}[_{variant}]_ASM if sha{n}_block_data_order[_{variant}] is +// defined in assembly. + +#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86) || defined(OPENSSL_ARM)) -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \ - defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)) #define SHA1_ASM #define SHA256_ASM #define SHA512_ASM -void sha1_block_data_order(uint32_t *state, const uint8_t *in, + +void sha1_block_data_order(uint32_t *state, const uint8_t *data, size_t num_blocks); -void sha256_block_data_order(uint32_t *state, const uint8_t *in, +void sha256_block_data_order(uint32_t *state, const uint8_t *data, size_t num_blocks); -void sha512_block_data_order(uint64_t *state, const uint8_t *in, +void sha512_block_data_order(uint64_t *state, const uint8_t *data, size_t num_blocks); + +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) + +#define SHA1_ASM_NOHW +#define SHA256_ASM_NOHW +#define SHA512_ASM_NOHW + +#define SHA1_ASM_HW +OPENSSL_INLINE int sha1_hw_capable(void) { + return CRYPTO_is_ARMv8_SHA1_capable(); +} + +#define SHA256_ASM_HW +OPENSSL_INLINE int sha256_hw_capable(void) { + return CRYPTO_is_ARMv8_SHA256_capable(); +} + +#define SHA512_ASM_HW +OPENSSL_INLINE int sha512_hw_capable(void) { + return CRYPTO_is_ARMv8_SHA512_capable(); +} + +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) + +#define SHA1_ASM_NOHW +#define SHA256_ASM_NOHW +#define SHA512_ASM_NOHW + +#define SHA1_ASM_HW +OPENSSL_INLINE int sha1_hw_capable(void) { + return CRYPTO_is_x86_SHA_capable() && CRYPTO_is_SSSE3_capable(); +} + +#define SHA1_ASM_AVX2 +OPENSSL_INLINE int sha1_avx2_capable(void) { + // TODO: Simplify this logic, which was extracted from the assembly: + // * Does AVX2 imply SSSE3? + // * sha1_block_data_order_avx2 does not seem to use SSSE3 instructions. + return CRYPTO_is_AVX2_capable() && CRYPTO_is_BMI2_capable() && + CRYPTO_is_BMI1_capable() && CRYPTO_is_SSSE3_capable(); +} +void sha1_block_data_order_avx2(uint32_t *state, const uint8_t *data, + size_t num); + +#define SHA1_ASM_AVX +OPENSSL_INLINE int sha1_avx_capable(void) { + // TODO: Simplify this logic, which was extracted from the assembly: + // * Does AVX imply SSSE3? + // * sha1_block_data_order_avx does not seem to use SSSE3 instructions. + // Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the + // discussion in sha1-586.pl. + return CRYPTO_is_AVX_capable() && CRYPTO_is_SSSE3_capable() && + CRYPTO_is_intel_cpu(); +} +void sha1_block_data_order_avx(uint32_t *state, const uint8_t *data, + size_t num); + +#define SHA1_ASM_SSSE3 +OPENSSL_INLINE int sha1_ssse3_capable(void) { + return CRYPTO_is_SSSE3_capable(); +} +void sha1_block_data_order_ssse3(uint32_t *state, const uint8_t *data, + size_t num); + +#define SHA256_ASM_HW +OPENSSL_INLINE int sha256_hw_capable(void) { + return CRYPTO_is_x86_SHA_capable(); +} + +#define SHA256_ASM_AVX +OPENSSL_INLINE int sha256_avx_capable(void) { + // TODO: Simplify this logic, which was extracted from the assembly: + // * Does AVX imply SSSE3? + // * sha256_block_data_order_avx does not seem to use SSSE3 instructions. + // Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the + // discussion in sha1-586.pl. + return CRYPTO_is_AVX_capable() && CRYPTO_is_SSSE3_capable() && + CRYPTO_is_intel_cpu(); +} +void sha256_block_data_order_avx(uint32_t *state, const uint8_t *data, + size_t num); + +#define SHA256_ASM_SSSE3 +OPENSSL_INLINE int sha256_ssse3_capable(void) { + return CRYPTO_is_SSSE3_capable(); +} +void sha256_block_data_order_ssse3(uint32_t *state, const uint8_t *data, + size_t num); + +#define SHA512_ASM_AVX +OPENSSL_INLINE int sha512_avx_capable(void) { + // TODO: Simplify this logic, which was extracted from the assembly: + // * Does AVX imply SSSE3? + // * sha512_block_data_order_avx does not seem to use SSSE3 instructions. + // Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the + // discussion in sha1-586.pl. + return CRYPTO_is_AVX_capable() && CRYPTO_is_SSSE3_capable() && + CRYPTO_is_intel_cpu(); +} +void sha512_block_data_order_avx(uint64_t *state, const uint8_t *data, + size_t num); + #endif +#if defined(SHA1_ASM_HW) +void sha1_block_data_order_hw(uint32_t *state, const uint8_t *data, size_t num); +#endif +#if defined(SHA1_ASM_NOHW) +void sha1_block_data_order_nohw(uint32_t *state, const uint8_t *data, + size_t num); +#endif + +#if defined(SHA256_ASM_HW) +void sha256_block_data_order_hw(uint32_t *state, const uint8_t *data, + size_t num); +#endif +#if defined(SHA256_ASM_NOHW) +void sha256_block_data_order_nohw(uint32_t *state, const uint8_t *data, + size_t num); +#endif + +#if defined(SHA512_ASM_HW) +void sha512_block_data_order_hw(uint64_t *state, const uint8_t *data, + size_t num); +#endif +#if defined(SHA512_ASM_NOHW) +void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *data, + size_t num); +#endif #if defined(__cplusplus) } // extern "C" diff --git a/crypto/fipsmodule/sha/sha1.c b/crypto/fipsmodule/sha/sha1.c index 4baeed669..7b267e3df 100644 --- a/crypto/fipsmodule/sha/sha1.c +++ b/crypto/fipsmodule/sha/sha1.c @@ -232,8 +232,10 @@ void CRYPTO_fips_186_2_prf(uint8_t *out, size_t out_len, #define X(i) XX##i #if !defined(SHA1_ASM) -static void sha1_block_data_order(uint32_t *state, const uint8_t *data, - size_t num) { + +#if !defined(SHA1_ASM_NOHW) +static void sha1_block_data_order_nohw(uint32_t *state, const uint8_t *data, + size_t num) { register uint32_t A, B, C, D, E, T; uint32_t XX0, XX1, XX2, XX3, XX4, XX5, XX6, XX7, XX8, XX9, XX10, XX11, XX12, XX13, XX14, XX15; @@ -380,7 +382,38 @@ static void sha1_block_data_order(uint32_t *state, const uint8_t *data, E = state[4]; } } +#endif // !SHA1_ASM_NOHW + +static void sha1_block_data_order(uint32_t *state, const uint8_t *data, + size_t num) { +#if defined(SHA1_ASM_HW) + if (sha1_hw_capable()) { + sha1_block_data_order_hw(state, data, num); + return; + } #endif +#if defined(SHA1_ASM_AVX2) + if (sha1_avx2_capable()) { + sha1_block_data_order_avx(state, data, num); + return; + } +#endif +#if defined(SHA1_ASM_AVX) + if (sha1_avx_capable()) { + sha1_block_data_order_avx(state, data, num); + return; + } +#endif +#if defined(SHA1_ASM_SSSE3) + if (sha1_ssse3_capable()) { + sha1_block_data_order_ssse3(state, data, num); + return; + } +#endif + sha1_block_data_order_nohw(state, data, num); +} + +#endif // !SHA1_ASM #undef Xupdate #undef K_00_19 diff --git a/crypto/fipsmodule/sha/sha256.c b/crypto/fipsmodule/sha/sha256.c index 046f6e25b..0b0aca22f 100644 --- a/crypto/fipsmodule/sha/sha256.c +++ b/crypto/fipsmodule/sha/sha256.c @@ -114,7 +114,7 @@ uint8_t *SHA256(const uint8_t *data, size_t len, return out; } -#ifndef SHA256_ASM +#if !defined(SHA256_ASM) static void sha256_block_data_order(uint32_t *state, const uint8_t *in, size_t num); #endif @@ -172,7 +172,9 @@ int SHA224_Final(uint8_t out[SHA224_DIGEST_LENGTH], SHA256_CTX *ctx) { return sha256_final_impl(out, SHA224_DIGEST_LENGTH, ctx); } -#ifndef SHA256_ASM +#if !defined(SHA256_ASM) + +#if !defined(SHA256_ASM_NOHW) static const uint32_t K256[64] = { 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, @@ -221,8 +223,8 @@ static const uint32_t K256[64] = { ROUND_00_15(i, a, b, c, d, e, f, g, h); \ } while (0) -static void sha256_block_data_order(uint32_t *state, const uint8_t *data, - size_t num) { +static void sha256_block_data_order_nohw(uint32_t *state, const uint8_t *data, + size_t num) { uint32_t a, b, c, d, e, f, g, h, s0, s1, T1; uint32_t X[16]; int i; @@ -308,7 +310,33 @@ static void sha256_block_data_order(uint32_t *state, const uint8_t *data, } } -#endif // !SHA256_ASM +#endif // !defined(SHA256_ASM_NOHW) + +static void sha256_block_data_order(uint32_t *state, const uint8_t *data, + size_t num) { +#if defined(SHA256_ASM_HW) + if (sha256_hw_capable()) { + sha256_block_data_order_hw(state, data, num); + return; + } +#endif +#if defined(SHA256_ASM_AVX) + if (sha256_avx_capable()) { + sha256_block_data_order_avx(state, data, num); + return; + } +#endif +#if defined(SHA256_ASM_SSSE3) + if (sha256_ssse3_capable()) { + sha256_block_data_order_ssse3(state, data, num); + return; + } +#endif + sha256_block_data_order_nohw(state, data, num); +} + +#endif // !defined(SHA256_ASM) + void SHA256_TransformBlocks(uint32_t state[8], const uint8_t *data, size_t num_blocks) { diff --git a/crypto/fipsmodule/sha/sha512.c b/crypto/fipsmodule/sha/sha512.c index 2c7ce314d..0f4142ca2 100644 --- a/crypto/fipsmodule/sha/sha512.c +++ b/crypto/fipsmodule/sha/sha512.c @@ -279,7 +279,9 @@ static int sha512_final_impl(uint8_t *out, size_t md_len, SHA512_CTX *sha) { return 1; } -#ifndef SHA512_ASM +#if !defined(SHA512_ASM) + +#if !defined(SHA512_ASM_NOHW) static const uint64_t K512[80] = { UINT64_C(0x428a2f98d728ae22), UINT64_C(0x7137449123ef65cd), UINT64_C(0xb5c0fbcfec4d3b2f), UINT64_C(0xe9b5dba58189dbbc), @@ -341,8 +343,8 @@ static const uint64_t K512[80] = { #if defined(__i386) || defined(__i386__) || defined(_M_IX86) // This code should give better results on 32-bit CPU with less than // ~24 registers, both size and performance wise... -static void sha512_block_data_order(uint64_t *state, const uint8_t *in, - size_t num) { +static void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *in, + size_t num) { uint64_t A, E, T; uint64_t X[9 + 80], *F; int i; @@ -414,8 +416,8 @@ static void sha512_block_data_order(uint64_t *state, const uint8_t *in, ROUND_00_15(i + j, a, b, c, d, e, f, g, h); \ } while (0) -static void sha512_block_data_order(uint64_t *state, const uint8_t *in, - size_t num) { +static void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *in, + size_t num) { uint64_t a, b, c, d, e, f, g, h, s0, s1, T1; uint64_t X[16]; int i; @@ -498,6 +500,25 @@ static void sha512_block_data_order(uint64_t *state, const uint8_t *in, #endif +#endif // !SHA512_ASM_NOHW + +static void sha512_block_data_order(uint64_t *state, const uint8_t *data, + size_t num) { +#if defined(SHA512_ASM_HW) + if (sha512_hw_capable()) { + sha512_block_data_order_hw(state, data, num); + return; + } +#endif +#if defined(SHA512_ASM_AVX) + if (sha512_avx_capable()) { + sha512_block_data_order_avx(state, data, num); + return; + } +#endif + sha512_block_data_order_nohw(state, data, num); +} + #endif // !SHA512_ASM #undef Sigma0 diff --git a/crypto/fipsmodule/sha/sha_test.cc b/crypto/fipsmodule/sha/sha_test.cc index 5029bb0f4..22856f80f 100644 --- a/crypto/fipsmodule/sha/sha_test.cc +++ b/crypto/fipsmodule/sha/sha_test.cc @@ -42,41 +42,100 @@ TEST(SHATest, FIPS1862PRF) { } } -#if defined(SHA1_ASM) && defined(SUPPORTS_ABI_TEST) +#if defined(SUPPORTS_ABI_TEST) + TEST(SHATest, SHA1ABI) { SHA_CTX ctx; SHA1_Init(&ctx); static const uint8_t kBuf[SHA_CBLOCK * 8] = {0}; - CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 1); - CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 2); - CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 4); - CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 8); + for (size_t blocks : {1, 2, 4, 8}) { +#if defined(SHA1_ASM) + CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, blocks); +#endif +#if defined(SHA1_ASM_HW) + if (sha1_hw_capable()) { + CHECK_ABI(sha1_block_data_order_hw, ctx.h, kBuf, blocks); + } +#endif +#if defined(SHA1_ASM_AVX2) + if (sha1_avx2_capable()) { + CHECK_ABI(sha1_block_data_order_avx2, ctx.h, kBuf, blocks); + } +#endif +#if defined(SHA1_ASM_AVX) + if (sha1_avx_capable()) { + CHECK_ABI(sha1_block_data_order_avx, ctx.h, kBuf, blocks); + return; + } +#endif +#if defined(SHA1_ASM_SSSE3) + if (sha1_ssse3_capable()) { + CHECK_ABI(sha1_block_data_order_ssse3, ctx.h, kBuf, blocks); + return; + } +#endif +#if defined(SHA1_ASM_NOHW) + CHECK_ABI(sha1_block_data_order_nohw, ctx.h, kBuf, blocks); +#endif + } } -#endif // SHA1_ASM && SUPPORTS_ABI_TEST -#if defined(SHA256_ASM) && defined(SUPPORTS_ABI_TEST) TEST(SHATest, SHA256ABI) { SHA256_CTX ctx; SHA256_Init(&ctx); static const uint8_t kBuf[SHA256_CBLOCK * 8] = {0}; - CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 1); - CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 2); - CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 4); - CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 8); + for (size_t blocks : {1, 2, 4, 8}) { +#if defined(SHA256_ASM) + CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, blocks); +#endif +#if defined(SHA256_ASM_HW) + if (sha256_hw_capable()) { + CHECK_ABI(sha256_block_data_order_hw, ctx.h, kBuf, blocks); + } +#endif +#if defined(SHA256_ASM_AVX) + if (sha256_avx_capable()) { + CHECK_ABI(sha256_block_data_order_avx, ctx.h, kBuf, blocks); + return; + } +#endif +#if defined(SHA256_ASM_SSSE3) + if (sha256_ssse3_capable()) { + CHECK_ABI(sha256_block_data_order_ssse3, ctx.h, kBuf, blocks); + return; + } +#endif +#if defined(SHA256_ASM_NOHW) + CHECK_ABI(sha256_block_data_order_nohw, ctx.h, kBuf, blocks); +#endif + } } -#endif // SHA256_ASM && SUPPORTS_ABI_TEST -#if defined(SHA512_ASM) && defined(SUPPORTS_ABI_TEST) TEST(SHATest, SHA512ABI) { SHA512_CTX ctx; SHA512_Init(&ctx); static const uint8_t kBuf[SHA512_CBLOCK * 4] = {0}; - CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 1); - CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 2); - CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 3); - CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 4); + for (size_t blocks : {1, 2, 3, 4}) { +#if defined(SHA512_ASM) + CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, blocks); +#endif +#if defined(SHA512_ASM_HW) + if (sha512_hw_capable()) { + CHECK_ABI(sha512_block_data_order_hw, ctx.h, kBuf, blocks); + } +#endif +#if defined(SHA512_ASM_AVX) + if (sha512_avx_capable()) { + CHECK_ABI(sha512_block_data_order_avx, ctx.h, kBuf, blocks); + } +#endif +#if defined(SHA512_ASM_NOHW) + CHECK_ABI(sha512_block_data_order_nohw, ctx.h, kBuf, blocks); +#endif + } } -#endif // SHA512_ASM && SUPPORTS_ABI_TEST + +#endif // SUPPORTS_ABI_TEST diff --git a/crypto/internal.h b/crypto/internal.h index f2db41c41..e9da01065 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -1450,6 +1450,15 @@ OPENSSL_INLINE int CRYPTO_is_ADX_capable(void) { #endif } +// SHA-1 and SHA-256 are defined as a single extension. +OPENSSL_INLINE int CRYPTO_is_x86_SHA_capable(void) { +#if defined(__SHA__) + return 1; +#else + return (OPENSSL_get_ia32cap(2) & (1u << 29)) != 0; +#endif +} + #endif // OPENSSL_X86 || OPENSSL_X86_64 #if defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) @@ -1519,6 +1528,41 @@ OPENSSL_INLINE int CRYPTO_is_ARMv8_PMULL_capable(void) { #endif } +OPENSSL_INLINE int CRYPTO_is_ARMv8_SHA1_capable(void) { + // SHA-1 and SHA-2 (only) share |__ARM_FEATURE_SHA2| but otherwise + // are dealt with independently. +#if defined(OPENSSL_STATIC_ARMCAP_SHA1) || defined(__ARM_FEATURE_SHA2) + return 1; +#elif defined(OPENSSL_STATIC_ARMCAP) + return 0; +#else + return (OPENSSL_get_armcap() & ARMV8_SHA1) != 0; +#endif +} + +OPENSSL_INLINE int CRYPTO_is_ARMv8_SHA256_capable(void) { + // SHA-1 and SHA-2 (only) share |__ARM_FEATURE_SHA2| but otherwise + // are dealt with independently. +#if defined(OPENSSL_STATIC_ARMCAP_SHA256) || defined(__ARM_FEATURE_SHA2) + return 1; +#elif defined(OPENSSL_STATIC_ARMCAP) + return 0; +#else + return (OPENSSL_get_armcap() & ARMV8_SHA256) != 0; +#endif +} + +OPENSSL_INLINE int CRYPTO_is_ARMv8_SHA512_capable(void) { + // There is no |OPENSSL_STATIC_ARMCAP_SHA512|. +#if defined(__ARM_FEATURE_SHA512) + return 1; +#elif defined(OPENSSL_STATIC_ARMCAP) + return 0; +#else + return (OPENSSL_get_armcap() & ARMV8_SHA512) != 0; +#endif +} + #endif // OPENSSL_ARM || OPENSSL_AARCH64 #if defined(BORINGSSL_DISPATCH_TEST)