OERV-RUNTIME/boringssl
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sha: Move AArch64/X86-64 dispatching to C.

Browse Source 
Take a step towards removing all dispatching logic from assembly
for AArch64 and X86-64.

Change-Id: I1c965012e81837ff228c810d54e730c525cad54f
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/64208
Reviewed-by: Bob Beck <bbe@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
This commit is contained in:
Brian Smith
2023-12-12 02:39:08 +00:00
committed by Boringssl LUCI CQ
parent 96c0bb06fd
commit 10c24cb433
10 changed files with 416 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/fipsmodule/sha/asm/sha1-armv8.pl
+8 -18
View File
@@ -175,22 +175,12 @@ $code.=<<___;
.text
.extern OPENSSL_armcap_P
.hidden OPENSSL_armcap_P
.globl sha1_block_data_order
.type sha1_block_data_order,%function
.globl sha1_block_data_order_nohw
.type sha1_block_data_order_nohw,%function
.align 6
sha1_block_data_order:
sha1_block_data_order_nohw:
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
AARCH64_VALID_CALL_TARGET
#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10
adrp x16,:pg_hi21_nc:OPENSSL_armcap_P
#else
adrp x16,:pg_hi21:OPENSSL_armcap_P
#endif
ldr w16,[x16,:lo12:OPENSSL_armcap_P]
tst w16,#ARMV8_SHA1
b.ne .Lv8_entry
stp x29,x30,[sp,#-96]!
add x29,sp,#0
@@ -239,7 +229,7 @@ $code.=<<___;
ldp x27,x28,[sp,#80]
ldr x29,[sp],#96
ret
.size sha1_block_data_order,.-sha1_block_data_order
.size sha1_block_data_order_nohw,.-sha1_block_data_order_nohw
___
{{{
my ($ABCD,$E,$E0,$E1)=map("v$_.16b",(0..3));
@@ -249,12 +239,12 @@ my ($W0,$W1)=("v20.4s","v21.4s");
my $ABCD_SAVE="v22.16b";
$code.=<<___;
.type sha1_block_armv8,%function
.globl sha1_block_data_order_hw
.type sha1_block_data_order_hw,%function
.align 6
sha1_block_armv8:
sha1_block_data_order_hw:
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
AARCH64_VALID_CALL_TARGET
.Lv8_entry:
stp x29,x30,[sp,#-16]!
add x29,sp,#0
@@ -318,7 +308,7 @@ $code.=<<___;
ldr x29,[sp],#16
ret
.size sha1_block_armv8,.-sha1_block_armv8
.size sha1_block_data_order_hw,.-sha1_block_data_order_hw
.section .rodata
.align 6
.Lconst:
crypto/fipsmodule/sha/asm/sha1-x86_64.pl
+23 -48
View File
@@ -237,42 +237,13 @@ push(@xi,shift(@xi));
$code.=<<___;
.text
.extern OPENSSL_ia32cap_P
.globl sha1_block_data_order
.type sha1_block_data_order,\@function,3
.globl sha1_block_data_order_nohw
.type sha1_block_data_order_nohw,\@function,3
.align 16
sha1_block_data_order:
sha1_block_data_order_nohw:
.cfi_startproc
_CET_ENDBR
leaq OPENSSL_ia32cap_P(%rip),%r10
mov 0(%r10),%r9d
mov 4(%r10),%r8d
mov 8(%r10),%r10d
test \$`1<<9`,%r8d # check SSSE3 bit
jz .Lialu
___
$code.=<<___ if ($shaext);
test \$`1<<29`,%r10d # check SHA bit
jnz _shaext_shortcut
___
$code.=<<___ if ($avx>1);
and \$`1<<3|1<<5|1<<8`,%r10d # check AVX2+BMI1+BMI2
cmp \$`1<<3|1<<5|1<<8`,%r10d
je _avx2_shortcut
___
$code.=<<___ if ($avx);
and \$`1<<28`,%r8d # mask AVX bit
and \$`1<<30`,%r9d # mask "Intel CPU" bit
or %r9d,%r8d
cmp \$`1<<28|1<<30`,%r8d
je _avx_shortcut
___
$code.=<<___;
jmp _ssse3_shortcut
.align 16
.Lialu:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
@@ -341,7 +312,7 @@ $code.=<<___;
.Lepilogue:
ret
.cfi_endproc
.size sha1_block_data_order,.-sha1_block_data_order
.size sha1_block_data_order_nohw,.-sha1_block_data_order_nohw
___
if ($shaext) {{{
######################################################################
@@ -352,11 +323,12 @@ my ($ABCD,$E,$E_,$BSWAP,$ABCD_SAVE,$E_SAVE)=map("%xmm$_",(0..3,8,9));
my @MSG=map("%xmm$_",(4..7));
$code.=<<___;
.type sha1_block_data_order_shaext,\@function,3
.globl sha1_block_data_order_hw
.type sha1_block_data_order_hw,\@function,3
.align 32
sha1_block_data_order_shaext:
_shaext_shortcut:
sha1_block_data_order_hw:
.cfi_startproc
_CET_ENDBR
___
$code.=<<___ if ($win64);
lea `-8-4*16`(%rsp),%rsp
@@ -457,7 +429,7 @@ ___
$code.=<<___;
ret
.cfi_endproc
.size sha1_block_data_order_shaext,.-sha1_block_data_order_shaext
.size sha1_block_data_order_hw,.-sha1_block_data_order_hw
___
}}}
{{{
@@ -487,11 +459,12 @@ ___
}
$code.=<<___;
.globl sha1_block_data_order_ssse3
.type sha1_block_data_order_ssse3,\@function,3
.align 16
sha1_block_data_order_ssse3:
_ssse3_shortcut:
.cfi_startproc
_CET_ENDBR
mov %rsp,$fp # frame pointer
.cfi_def_cfa_register $fp
push %rbx
@@ -961,11 +934,12 @@ my $_rol=sub { &shld(@_[0],@_) };
my $_ror=sub { &shrd(@_[0],@_) };
$code.=<<___;
.globl sha1_block_data_order_avx
.type sha1_block_data_order_avx,\@function,3
.align 16
sha1_block_data_order_avx:
_avx_shortcut:
.cfi_startproc
_CET_ENDBR
mov %rsp,$fp
.cfi_def_cfa_register $fp
push %rbx
@@ -1340,11 +1314,12 @@ my $rx=0;
my $frame="%r13";
$code.=<<___;
.globl sha1_block_data_order_avx2
.type sha1_block_data_order_avx2,\@function,3
.align 16
sha1_block_data_order_avx2:
_avx2_shortcut:
.cfi_startproc
_CET_ENDBR
mov %rsp,$fp
.cfi_def_cfa_register $fp
push %rbx
@@ -2019,14 +1994,14 @@ ssse3_handler:
.section .pdata
.align 4
.rva .LSEH_begin_sha1_block_data_order
.rva .LSEH_end_sha1_block_data_order
.rva .LSEH_info_sha1_block_data_order
.rva .LSEH_begin_sha1_block_data_order_nohw
.rva .LSEH_end_sha1_block_data_order_nohw
.rva .LSEH_info_sha1_block_data_order_nohw
___
$code.=<<___ if ($shaext);
.rva .LSEH_begin_sha1_block_data_order_shaext
.rva .LSEH_end_sha1_block_data_order_shaext
.rva .LSEH_info_sha1_block_data_order_shaext
.rva .LSEH_begin_sha1_block_data_order_hw
.rva .LSEH_end_sha1_block_data_order_hw
.rva .LSEH_info_sha1_block_data_order_hw
___
$code.=<<___;
.rva .LSEH_begin_sha1_block_data_order_ssse3
@@ -2046,12 +2021,12 @@ ___
$code.=<<___;
.section .xdata
.align 8
.LSEH_info_sha1_block_data_order:
.LSEH_info_sha1_block_data_order_nohw:
.byte 9,0,0,0
.rva se_handler
___
$code.=<<___ if ($shaext);
.LSEH_info_sha1_block_data_order_shaext:
.LSEH_info_sha1_block_data_order_hw:
.byte 9,0,0,0
.rva shaext_handler
___
crypto/fipsmodule/sha/asm/sha512-armv8.pl
+12 -30
View File
@@ -76,7 +76,7 @@ if ($flavour && $flavour ne "void") {
*STDOUT=*OUT;
}
$func="sha${BITS}_block_data_order";
$func="sha${BITS}_block_data_order_nohw";
($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
@@ -180,31 +180,10 @@ $code.=<<___;
.text
.extern OPENSSL_armcap_P
.hidden OPENSSL_armcap_P
.globl $func
.type $func,%function
.align 6
$func:
AARCH64_VALID_CALL_TARGET
#ifndef __KERNEL__
#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10
adrp x16,:pg_hi21_nc:OPENSSL_armcap_P
#else
adrp x16,:pg_hi21:OPENSSL_armcap_P
#endif
ldr w16,[x16,:lo12:OPENSSL_armcap_P]
___
$code.=<<___ if ($SZ==4);
tst w16,#ARMV8_SHA256
b.ne .Lv8_entry
___
$code.=<<___ if ($SZ==8);
tst w16,#ARMV8_SHA512
b.ne .Lv8_entry
___
$code.=<<___;
#endif
AARCH64_SIGN_LINK_REGISTER
stp x29,x30,[sp,#-128]!
add x29,sp,#0
@@ -356,11 +335,12 @@ my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
$code.=<<___;
.text
#ifndef __KERNEL__
.type sha256_block_armv8,%function
.globl sha256_block_data_order_hw
.type sha256_block_data_order_hw,%function
.align 6
sha256_block_armv8:
.Lv8_entry:
sha256_block_data_order_hw:
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
AARCH64_VALID_CALL_TARGET
stp x29,x30,[sp,#-16]!
add x29,sp,#0
@@ -425,7 +405,7 @@ $code.=<<___;
ldr x29,[sp],#16
ret
.size sha256_block_armv8,.-sha256_block_armv8
.size sha256_block_data_order_hw,.-sha256_block_data_order_hw
#endif
___
}
@@ -442,10 +422,12 @@ my ($AB,$CD,$EF,$GH)=map("v$_.16b",(26..29));
$code.=<<___;
.text
#ifndef __KERNEL__
.type sha512_block_armv8,%function
.globl sha512_block_data_order_hw
.type sha512_block_data_order_hw,%function
.align 6
sha512_block_armv8:
.Lv8_entry:
sha512_block_data_order_hw:
// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
AARCH64_VALID_CALL_TARGET
stp x29,x30,[sp,#-16]!
add x29,sp,#0
@@ -529,7 +511,7 @@ $code.=<<___;
ldr x29,[sp],#16
ret
.size sha512_block_armv8,.-sha512_block_armv8
.size sha512_block_data_order_hw,.-sha512_block_data_order_hw
#endif
___
}
crypto/fipsmodule/sha/asm/sha512-x86_64.pl
+21 -48
View File
@@ -257,42 +257,12 @@ ___
$code=<<___;
.text
.extern OPENSSL_ia32cap_P
.globl $func
.type $func,\@function,3
.globl ${func}_nohw
.type ${func}_nohw,\@function,3
.align 16
$func:
${func}_nohw:
.cfi_startproc
_CET_ENDBR
___
$code.=<<___ if ($SZ==4 || $avx);
leaq OPENSSL_ia32cap_P(%rip),%r11
mov 0(%r11),%r9d
mov 4(%r11),%r10d
mov 8(%r11),%r11d
___
$code.=<<___ if ($SZ==4 && $shaext);
test \$`1<<29`,%r11d # check for SHA
jnz .Lshaext_shortcut
___
# XOP codepath removed.
$code.=<<___ if ($avx>1);
and \$`1<<8|1<<5|1<<3`,%r11d # check for BMI2+AVX2+BMI1
cmp \$`1<<8|1<<5|1<<3`,%r11d
je .Lavx2_shortcut
___
$code.=<<___ if ($avx);
and \$`1<<30`,%r9d # mask "Intel CPU" bit
and \$`1<<28|1<<9`,%r10d # mask AVX and SSSE3 bits
or %r9d,%r10d
cmp \$`1<<28|1<<9|1<<30`,%r10d
je .Lavx_shortcut
___
$code.=<<___ if ($SZ==4);
test \$`1<<9`,%r10d
jnz .Lssse3_shortcut
___
$code.=<<___;
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
@@ -400,7 +370,7 @@ $code.=<<___;
.Lepilogue:
ret
.cfi_endproc
.size $func,.-$func
.size ${func}_nohw,.-${func}_nohw
___
if ($SZ==4) {
@@ -558,11 +528,12 @@ my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..2,7..10)
my @MSG=map("%xmm$_",(3..6));
$code.=<<___;
.type sha256_block_data_order_shaext,\@function,3
.globl sha256_block_data_order_hw
.type sha256_block_data_order_hw,\@function,3
.align 64
sha256_block_data_order_shaext:
sha256_block_data_order_hw:
.cfi_startproc
.Lshaext_shortcut:
_CET_ENDBR
___
$code.=<<___ if ($win64);
lea `-8-5*16`(%rsp),%rsp
@@ -707,7 +678,7 @@ ___
$code.=<<___;
ret
.cfi_endproc
.size sha256_block_data_order_shaext,.-sha256_block_data_order_shaext
.size sha256_block_data_order_hw,.-sha256_block_data_order_hw
___
}}}
{{{
@@ -772,11 +743,12 @@ my @X = map("%xmm$_",(0..3));
my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9));
$code.=<<___;
.globl ${func}_ssse3
.type ${func}_ssse3,\@function,3
.align 64
${func}_ssse3:
.cfi_startproc
.Lssse3_shortcut:
_CET_ENDBR
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
@@ -1135,11 +1107,12 @@ if ($avx) {{
local *ror = sub { &shrd(@_[0],@_) };
$code.=<<___;
.globl ${func}_avx
.type ${func}_avx,\@function,3
.align 64
${func}_avx:
.cfi_startproc
.Lavx_shortcut:
_CET_ENDBR
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
@@ -2005,14 +1978,14 @@ ___
$code.=<<___;
.section .pdata
.align 4
.rva .LSEH_begin_$func
.rva .LSEH_end_$func
.rva .LSEH_info_$func
.rva .LSEH_begin_${func}_nohw
.rva .LSEH_end_${func}_nohw
.rva .LSEH_info_${func}_nohw
___
$code.=<<___ if ($SZ==4 && $shaext);
.rva .LSEH_begin_${func}_shaext
.rva .LSEH_end_${func}_shaext
.rva .LSEH_info_${func}_shaext
.rva .LSEH_begin_${func}_hw
.rva .LSEH_end_${func}_hw
.rva .LSEH_info_${func}_hw
___
$code.=<<___ if ($SZ==4);
.rva .LSEH_begin_${func}_ssse3
@@ -2032,13 +2005,13 @@ ___
$code.=<<___;
.section .xdata
.align 8
.LSEH_info_$func:
.LSEH_info_${func}_nohw:
.byte 9,0,0,0
.rva se_handler
.rva .Lprologue,.Lepilogue # HandlerData[]
___
$code.=<<___ if ($SZ==4 && $shaext);
.LSEH_info_${func}_shaext:
.LSEH_info_${func}_hw:
.byte 9,0,0,0
.rva shaext_handler
___
crypto/fipsmodule/sha/internal.h
+137 -6
View File
@@ -17,25 +17,156 @@
#include <openssl/base.h>
#include "../../internal.h"
#if defined(__cplusplus)
extern "C" {
#endif
// Define SHA{n}[_{variant}]_ASM if sha{n}_block_data_order[_{variant}] is
// defined in assembly.
#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86) || defined(OPENSSL_ARM))
#if !defined(OPENSSL_NO_ASM) && \
(defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
#define SHA1_ASM
#define SHA256_ASM
#define SHA512_ASM
void sha1_block_data_order(uint32_t *state, const uint8_t *in,
void sha1_block_data_order(uint32_t *state, const uint8_t *data,
size_t num_blocks);
void sha256_block_data_order(uint32_t *state, const uint8_t *in,
void sha256_block_data_order(uint32_t *state, const uint8_t *data,
size_t num_blocks);
void sha512_block_data_order(uint64_t *state, const uint8_t *in,
void sha512_block_data_order(uint64_t *state, const uint8_t *data,
size_t num_blocks);
#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64)
#define SHA1_ASM_NOHW
#define SHA256_ASM_NOHW
#define SHA512_ASM_NOHW
#define SHA1_ASM_HW
OPENSSL_INLINE int sha1_hw_capable(void) {
return CRYPTO_is_ARMv8_SHA1_capable();
}
#define SHA256_ASM_HW
OPENSSL_INLINE int sha256_hw_capable(void) {
return CRYPTO_is_ARMv8_SHA256_capable();
}
#define SHA512_ASM_HW
OPENSSL_INLINE int sha512_hw_capable(void) {
return CRYPTO_is_ARMv8_SHA512_capable();
}
#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64)
#define SHA1_ASM_NOHW
#define SHA256_ASM_NOHW
#define SHA512_ASM_NOHW
#define SHA1_ASM_HW
OPENSSL_INLINE int sha1_hw_capable(void) {
return CRYPTO_is_x86_SHA_capable() && CRYPTO_is_SSSE3_capable();
}
#define SHA1_ASM_AVX2
OPENSSL_INLINE int sha1_avx2_capable(void) {
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX2 imply SSSE3?
// * sha1_block_data_order_avx2 does not seem to use SSSE3 instructions.
return CRYPTO_is_AVX2_capable() && CRYPTO_is_BMI2_capable() &&
CRYPTO_is_BMI1_capable() && CRYPTO_is_SSSE3_capable();
}
void sha1_block_data_order_avx2(uint32_t *state, const uint8_t *data,
size_t num);
#define SHA1_ASM_AVX
OPENSSL_INLINE int sha1_avx_capable(void) {
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX imply SSSE3?
// * sha1_block_data_order_avx does not seem to use SSSE3 instructions.
// Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
// discussion in sha1-586.pl.
return CRYPTO_is_AVX_capable() && CRYPTO_is_SSSE3_capable() &&
CRYPTO_is_intel_cpu();
}
void sha1_block_data_order_avx(uint32_t *state, const uint8_t *data,
size_t num);
#define SHA1_ASM_SSSE3
OPENSSL_INLINE int sha1_ssse3_capable(void) {
return CRYPTO_is_SSSE3_capable();
}
void sha1_block_data_order_ssse3(uint32_t *state, const uint8_t *data,
size_t num);
#define SHA256_ASM_HW
OPENSSL_INLINE int sha256_hw_capable(void) {
return CRYPTO_is_x86_SHA_capable();
}
#define SHA256_ASM_AVX
OPENSSL_INLINE int sha256_avx_capable(void) {
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX imply SSSE3?
// * sha256_block_data_order_avx does not seem to use SSSE3 instructions.
// Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
// discussion in sha1-586.pl.
return CRYPTO_is_AVX_capable() && CRYPTO_is_SSSE3_capable() &&
CRYPTO_is_intel_cpu();
}
void sha256_block_data_order_avx(uint32_t *state, const uint8_t *data,
size_t num);
#define SHA256_ASM_SSSE3
OPENSSL_INLINE int sha256_ssse3_capable(void) {
return CRYPTO_is_SSSE3_capable();
}
void sha256_block_data_order_ssse3(uint32_t *state, const uint8_t *data,
size_t num);
#define SHA512_ASM_AVX
OPENSSL_INLINE int sha512_avx_capable(void) {
// TODO: Simplify this logic, which was extracted from the assembly:
// * Does AVX imply SSSE3?
// * sha512_block_data_order_avx does not seem to use SSSE3 instructions.
// Pre-Zen AMD CPUs had slow SHLD/SHRD; Zen added the SHA extension; see the
// discussion in sha1-586.pl.
return CRYPTO_is_AVX_capable() && CRYPTO_is_SSSE3_capable() &&
CRYPTO_is_intel_cpu();
}
void sha512_block_data_order_avx(uint64_t *state, const uint8_t *data,
size_t num);
#endif
#if defined(SHA1_ASM_HW)
void sha1_block_data_order_hw(uint32_t *state, const uint8_t *data, size_t num);
#endif
#if defined(SHA1_ASM_NOHW)
void sha1_block_data_order_nohw(uint32_t *state, const uint8_t *data,
size_t num);
#endif
#if defined(SHA256_ASM_HW)
void sha256_block_data_order_hw(uint32_t *state, const uint8_t *data,
size_t num);
#endif
#if defined(SHA256_ASM_NOHW)
void sha256_block_data_order_nohw(uint32_t *state, const uint8_t *data,
size_t num);
#endif
#if defined(SHA512_ASM_HW)
void sha512_block_data_order_hw(uint64_t *state, const uint8_t *data,
size_t num);
#endif
#if defined(SHA512_ASM_NOHW)
void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *data,
size_t num);
#endif
#if defined(__cplusplus)
} // extern "C"
crypto/fipsmodule/sha/sha1.c
+35 -2
View File
@@ -232,8 +232,10 @@ void CRYPTO_fips_186_2_prf(uint8_t *out, size_t out_len,
#define X(i) XX##i
#if !defined(SHA1_ASM)
static void sha1_block_data_order(uint32_t *state, const uint8_t *data,
size_t num) {
#if !defined(SHA1_ASM_NOHW)
static void sha1_block_data_order_nohw(uint32_t *state, const uint8_t *data,
size_t num) {
register uint32_t A, B, C, D, E, T;
uint32_t XX0, XX1, XX2, XX3, XX4, XX5, XX6, XX7, XX8, XX9, XX10,
XX11, XX12, XX13, XX14, XX15;
@@ -380,7 +382,38 @@ static void sha1_block_data_order(uint32_t *state, const uint8_t *data,
E = state[4];
}
}
#endif // !SHA1_ASM_NOHW
static void sha1_block_data_order(uint32_t *state, const uint8_t *data,
size_t num) {
#if defined(SHA1_ASM_HW)
if (sha1_hw_capable()) {
sha1_block_data_order_hw(state, data, num);
return;
}
#endif
#if defined(SHA1_ASM_AVX2)
if (sha1_avx2_capable()) {
sha1_block_data_order_avx(state, data, num);
return;
}
#endif
#if defined(SHA1_ASM_AVX)
if (sha1_avx_capable()) {
sha1_block_data_order_avx(state, data, num);
return;
}
#endif
#if defined(SHA1_ASM_SSSE3)
if (sha1_ssse3_capable()) {
sha1_block_data_order_ssse3(state, data, num);
return;
}
#endif
sha1_block_data_order_nohw(state, data, num);
}
#endif // !SHA1_ASM
#undef Xupdate
#undef K_00_19
crypto/fipsmodule/sha/sha256.c
+33 -5
View File
@@ -114,7 +114,7 @@ uint8_t *SHA256(const uint8_t *data, size_t len,
return out;
}
#ifndef SHA256_ASM
#if !defined(SHA256_ASM)
static void sha256_block_data_order(uint32_t *state, const uint8_t *in,
size_t num);
#endif
@@ -172,7 +172,9 @@ int SHA224_Final(uint8_t out[SHA224_DIGEST_LENGTH], SHA256_CTX *ctx) {
return sha256_final_impl(out, SHA224_DIGEST_LENGTH, ctx);
}
#ifndef SHA256_ASM
#if !defined(SHA256_ASM)
#if !defined(SHA256_ASM_NOHW)
static const uint32_t K256[64] = {
0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL,
0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL,
@@ -221,8 +223,8 @@ static const uint32_t K256[64] = {
ROUND_00_15(i, a, b, c, d, e, f, g, h); \
} while (0)
static void sha256_block_data_order(uint32_t *state, const uint8_t *data,
size_t num) {
static void sha256_block_data_order_nohw(uint32_t *state, const uint8_t *data,
size_t num) {
uint32_t a, b, c, d, e, f, g, h, s0, s1, T1;
uint32_t X[16];
int i;
@@ -308,7 +310,33 @@ static void sha256_block_data_order(uint32_t *state, const uint8_t *data,
}
}
#endif // !SHA256_ASM
#endif // !defined(SHA256_ASM_NOHW)
static void sha256_block_data_order(uint32_t *state, const uint8_t *data,
size_t num) {
#if defined(SHA256_ASM_HW)
if (sha256_hw_capable()) {
sha256_block_data_order_hw(state, data, num);
return;
}
#endif
#if defined(SHA256_ASM_AVX)
if (sha256_avx_capable()) {
sha256_block_data_order_avx(state, data, num);
return;
}
#endif
#if defined(SHA256_ASM_SSSE3)
if (sha256_ssse3_capable()) {
sha256_block_data_order_ssse3(state, data, num);
return;
}
#endif
sha256_block_data_order_nohw(state, data, num);
}
#endif // !defined(SHA256_ASM)
void SHA256_TransformBlocks(uint32_t state[8], const uint8_t *data,
size_t num_blocks) {
crypto/fipsmodule/sha/sha512.c
+26 -5
View File
@@ -279,7 +279,9 @@ static int sha512_final_impl(uint8_t *out, size_t md_len, SHA512_CTX *sha) {
return 1;
}
#ifndef SHA512_ASM
#if !defined(SHA512_ASM)
#if !defined(SHA512_ASM_NOHW)
static const uint64_t K512[80] = {
UINT64_C(0x428a2f98d728ae22), UINT64_C(0x7137449123ef65cd),
UINT64_C(0xb5c0fbcfec4d3b2f), UINT64_C(0xe9b5dba58189dbbc),
@@ -341,8 +343,8 @@ static const uint64_t K512[80] = {
#if defined(__i386) || defined(__i386__) || defined(_M_IX86)
// This code should give better results on 32-bit CPU with less than
// ~24 registers, both size and performance wise...
static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
size_t num) {
static void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *in,
size_t num) {
uint64_t A, E, T;
uint64_t X[9 + 80], *F;
int i;
@@ -414,8 +416,8 @@ static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
ROUND_00_15(i + j, a, b, c, d, e, f, g, h); \
} while (0)
static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
size_t num) {
static void sha512_block_data_order_nohw(uint64_t *state, const uint8_t *in,
size_t num) {
uint64_t a, b, c, d, e, f, g, h, s0, s1, T1;
uint64_t X[16];
int i;
@@ -498,6 +500,25 @@ static void sha512_block_data_order(uint64_t *state, const uint8_t *in,
#endif
#endif // !SHA512_ASM_NOHW
static void sha512_block_data_order(uint64_t *state, const uint8_t *data,
size_t num) {
#if defined(SHA512_ASM_HW)
if (sha512_hw_capable()) {
sha512_block_data_order_hw(state, data, num);
return;
}
#endif
#if defined(SHA512_ASM_AVX)
if (sha512_avx_capable()) {
sha512_block_data_order_avx(state, data, num);
return;
}
#endif
sha512_block_data_order_nohw(state, data, num);
}
#endif // !SHA512_ASM
#undef Sigma0
crypto/fipsmodule/sha/sha_test.cc
+77 -18
View File
@@ -42,41 +42,100 @@ TEST(SHATest, FIPS1862PRF) {
}
}
#if defined(SHA1_ASM) && defined(SUPPORTS_ABI_TEST)
#if defined(SUPPORTS_ABI_TEST)
TEST(SHATest, SHA1ABI) {
SHA_CTX ctx;
SHA1_Init(&ctx);
static const uint8_t kBuf[SHA_CBLOCK * 8] = {0};
CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 1);
CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 2);
CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 4);
CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, 8);
for (size_t blocks : {1, 2, 4, 8}) {
#if defined(SHA1_ASM)
CHECK_ABI(sha1_block_data_order, ctx.h, kBuf, blocks);
#endif
#if defined(SHA1_ASM_HW)
if (sha1_hw_capable()) {
CHECK_ABI(sha1_block_data_order_hw, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA1_ASM_AVX2)
if (sha1_avx2_capable()) {
CHECK_ABI(sha1_block_data_order_avx2, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA1_ASM_AVX)
if (sha1_avx_capable()) {
CHECK_ABI(sha1_block_data_order_avx, ctx.h, kBuf, blocks);
return;
}
#endif
#if defined(SHA1_ASM_SSSE3)
if (sha1_ssse3_capable()) {
CHECK_ABI(sha1_block_data_order_ssse3, ctx.h, kBuf, blocks);
return;
}
#endif
#if defined(SHA1_ASM_NOHW)
CHECK_ABI(sha1_block_data_order_nohw, ctx.h, kBuf, blocks);
#endif
}
}
#endif // SHA1_ASM && SUPPORTS_ABI_TEST
#if defined(SHA256_ASM) && defined(SUPPORTS_ABI_TEST)
TEST(SHATest, SHA256ABI) {
SHA256_CTX ctx;
SHA256_Init(&ctx);
static const uint8_t kBuf[SHA256_CBLOCK * 8] = {0};
CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 1);
CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 2);
CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 4);
CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, 8);
for (size_t blocks : {1, 2, 4, 8}) {
#if defined(SHA256_ASM)
CHECK_ABI(sha256_block_data_order, ctx.h, kBuf, blocks);
#endif
#if defined(SHA256_ASM_HW)
if (sha256_hw_capable()) {
CHECK_ABI(sha256_block_data_order_hw, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA256_ASM_AVX)
if (sha256_avx_capable()) {
CHECK_ABI(sha256_block_data_order_avx, ctx.h, kBuf, blocks);
return;
}
#endif
#if defined(SHA256_ASM_SSSE3)
if (sha256_ssse3_capable()) {
CHECK_ABI(sha256_block_data_order_ssse3, ctx.h, kBuf, blocks);
return;
}
#endif
#if defined(SHA256_ASM_NOHW)
CHECK_ABI(sha256_block_data_order_nohw, ctx.h, kBuf, blocks);
#endif
}
}
#endif // SHA256_ASM && SUPPORTS_ABI_TEST
#if defined(SHA512_ASM) && defined(SUPPORTS_ABI_TEST)
TEST(SHATest, SHA512ABI) {
SHA512_CTX ctx;
SHA512_Init(&ctx);
static const uint8_t kBuf[SHA512_CBLOCK * 4] = {0};
CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 1);
CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 2);
CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 3);
CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, 4);
for (size_t blocks : {1, 2, 3, 4}) {
#if defined(SHA512_ASM)
CHECK_ABI(sha512_block_data_order, ctx.h, kBuf, blocks);
#endif
#if defined(SHA512_ASM_HW)
if (sha512_hw_capable()) {
CHECK_ABI(sha512_block_data_order_hw, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA512_ASM_AVX)
if (sha512_avx_capable()) {
CHECK_ABI(sha512_block_data_order_avx, ctx.h, kBuf, blocks);
}
#endif
#if defined(SHA512_ASM_NOHW)
CHECK_ABI(sha512_block_data_order_nohw, ctx.h, kBuf, blocks);
#endif
}
}
#endif // SHA512_ASM && SUPPORTS_ABI_TEST
#endif // SUPPORTS_ABI_TEST
crypto/internal.h
+44
View File
@@ -1450,6 +1450,15 @@ OPENSSL_INLINE int CRYPTO_is_ADX_capable(void) {
#endif
}
// SHA-1 and SHA-256 are defined as a single extension.
OPENSSL_INLINE int CRYPTO_is_x86_SHA_capable(void) {
#if defined(__SHA__)
return 1;
#else
return (OPENSSL_get_ia32cap(2) & (1u << 29)) != 0;
#endif
}
#endif // OPENSSL_X86 || OPENSSL_X86_64
#if defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
@@ -1519,6 +1528,41 @@ OPENSSL_INLINE int CRYPTO_is_ARMv8_PMULL_capable(void) {
#endif
}
OPENSSL_INLINE int CRYPTO_is_ARMv8_SHA1_capable(void) {
// SHA-1 and SHA-2 (only) share |__ARM_FEATURE_SHA2| but otherwise
// are dealt with independently.
#if defined(OPENSSL_STATIC_ARMCAP_SHA1) || defined(__ARM_FEATURE_SHA2)
return 1;
#elif defined(OPENSSL_STATIC_ARMCAP)
return 0;
#else
return (OPENSSL_get_armcap() & ARMV8_SHA1) != 0;
#endif
}
OPENSSL_INLINE int CRYPTO_is_ARMv8_SHA256_capable(void) {
// SHA-1 and SHA-2 (only) share |__ARM_FEATURE_SHA2| but otherwise
// are dealt with independently.
#if defined(OPENSSL_STATIC_ARMCAP_SHA256) || defined(__ARM_FEATURE_SHA2)
return 1;
#elif defined(OPENSSL_STATIC_ARMCAP)
return 0;
#else
return (OPENSSL_get_armcap() & ARMV8_SHA256) != 0;
#endif
}
OPENSSL_INLINE int CRYPTO_is_ARMv8_SHA512_capable(void) {
// There is no |OPENSSL_STATIC_ARMCAP_SHA512|.
#if defined(__ARM_FEATURE_SHA512)
return 1;
#elif defined(OPENSSL_STATIC_ARMCAP)
return 0;
#else
return (OPENSSL_get_armcap() & ARMV8_SHA512) != 0;
#endif
}
#endif // OPENSSL_ARM || OPENSSL_AARCH64
#if defined(BORINGSSL_DISPATCH_TEST)