diff --git a/crypto/fipsmodule/aes/aes.c b/crypto/fipsmodule/aes/aes.c index 56dfbe2e5..7eab5ac9e 100644 --- a/crypto/fipsmodule/aes/aes.c +++ b/crypto/fipsmodule/aes/aes.c @@ -116,4 +116,12 @@ int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits, AES_KEY *key) { } return ret; } + +int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits, AES_KEY *key) { + if (aes_hw_set_encrypt_key_alt_preferred()) { + return aes_hw_set_encrypt_key_alt(user_key, bits, key); + } else { + return aes_hw_set_encrypt_key_base(user_key, bits, key); + } +} #endif diff --git a/crypto/fipsmodule/aes/aes_test.cc b/crypto/fipsmodule/aes/aes_test.cc index dc90067d9..07feabbe3 100644 --- a/crypto/fipsmodule/aes/aes_test.cc +++ b/crypto/fipsmodule/aes/aes_test.cc @@ -347,7 +347,16 @@ TEST(AESTest, ABI) { } #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) - ASSERT_EQ(CHECK_ABI_SEH(aes_hw_set_encrypt_key, kKey, bits, &key), 0); + ASSERT_EQ(CHECK_ABI_SEH(aes_hw_set_encrypt_key_base, kKey, bits, &key), 0); + if (aes_hw_set_encrypt_key_alt_capable()) { + AES_KEY alt; + ASSERT_EQ(CHECK_ABI_SEH(aes_hw_set_encrypt_key_alt, kKey, bits, &alt), + 0); + EXPECT_EQ(alt.rounds, key.rounds); + for (unsigned i = 0; i <= alt.rounds; i++) { + EXPECT_EQ(alt.rd_key[i], key.rd_key[i]); + } + } CHECK_ABI_SEH(aes_hw_encrypt_key_to_decrypt_key, &key); #else ASSERT_EQ(CHECK_ABI_SEH(aes_hw_set_decrypt_key, kKey, bits, &key), 0); diff --git a/crypto/fipsmodule/aes/asm/aesni-x86.pl b/crypto/fipsmodule/aes/asm/aesni-x86.pl index d8fdfb822..077be94e7 100644 --- a/crypto/fipsmodule/aes/asm/aesni-x86.pl +++ b/crypto/fipsmodule/aes/asm/aesni-x86.pl @@ -83,7 +83,6 @@ open OUT,">$output"; &asm_init($ARGV[0]); -&external_label("OPENSSL_ia32cap_P"); &preprocessor_ifdef("BORINGSSL_DISPATCH_TEST") &external_label("BORINGSSL_function_hit"); &preprocessor_endif(); @@ -2109,18 +2108,15 @@ if ($PREFIX eq $AESNI_PREFIX) { ###################################################################### # Mechanical port from aesni-x86_64.pl. -# -# _aesni_set_encrypt_key is private interface, -# input: -# "eax" const unsigned char *userKey -# $rounds int bits -# $key AES_KEY *key -# output: -# "eax" return code -# $round rounds -&function_begin_B("_aesni_set_encrypt_key"); - &push ("ebp"); +# int $PREFIX_set_encrypt_key_base (const unsigned char *userKey, int bits, +# AES_KEY *key) +&function_begin_B("${PREFIX}_set_encrypt_key_base"); + &record_function_hit(3); + + &mov ("eax",&wparam(0)); + &mov ($rounds,&wparam(1)); + &mov ($key,&wparam(2)); &push ("ebx"); &call (&label("pic")); @@ -2128,12 +2124,9 @@ if ($PREFIX eq $AESNI_PREFIX) { &blindpop("ebx"); &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx")); - &picmeup("ebp","OPENSSL_ia32cap_P","ebx",&label("key_const")); &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0 - &mov ("ebp",&DWP(4,"ebp")); &lea ($key,&DWP(16,$key)); - &and ("ebp",1<<28|1<<11); # AVX and XOP bits &cmp ($rounds,256); &je (&label("14rounds")); &cmp ($rounds,192); @@ -2142,9 +2135,6 @@ if ($PREFIX eq $AESNI_PREFIX) { &jne (&label("bad_keybits")); &set_label("10rounds",16); - &cmp ("ebp",1<<28); - &je (&label("10rounds_alt")); - &mov ($rounds,9); &$movekey (&QWP(-16,$key),"xmm0"); # round 0 &aeskeygenassist("xmm1","xmm0",0x01); # round 1 @@ -2184,6 +2174,166 @@ if ($PREFIX eq $AESNI_PREFIX) { &xorps ("xmm0","xmm1"); &ret(); +&set_label("12rounds",16); + &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey + + &mov ($rounds,11); + &$movekey (&QWP(-16,$key),"xmm0"); # round 0 + &aeskeygenassist("xmm1","xmm2",0x01); # round 1,2 + &call (&label("key_192a_cold")); + &aeskeygenassist("xmm1","xmm2",0x02); # round 2,3 + &call (&label("key_192b")); + &aeskeygenassist("xmm1","xmm2",0x04); # round 4,5 + &call (&label("key_192a")); + &aeskeygenassist("xmm1","xmm2",0x08); # round 5,6 + &call (&label("key_192b")); + &aeskeygenassist("xmm1","xmm2",0x10); # round 7,8 + &call (&label("key_192a")); + &aeskeygenassist("xmm1","xmm2",0x20); # round 8,9 + &call (&label("key_192b")); + &aeskeygenassist("xmm1","xmm2",0x40); # round 10,11 + &call (&label("key_192a")); + &aeskeygenassist("xmm1","xmm2",0x80); # round 11,12 + &call (&label("key_192b")); + &$movekey (&QWP(0,$key),"xmm0"); + &mov (&DWP(48,$key),$rounds); + + &jmp (&label("good_key")); + +&set_label("key_192a",16); + &$movekey (&QWP(0,$key),"xmm0"); + &lea ($key,&DWP(16,$key)); +&set_label("key_192a_cold",16); + &movaps ("xmm5","xmm2"); +&set_label("key_192b_warm"); + &shufps ("xmm4","xmm0",0b00010000); + &movdqa ("xmm3","xmm2"); + &xorps ("xmm0","xmm4"); + &shufps ("xmm4","xmm0",0b10001100); + &pslldq ("xmm3",4); + &xorps ("xmm0","xmm4"); + &pshufd ("xmm1","xmm1",0b01010101); # critical path + &pxor ("xmm2","xmm3"); + &pxor ("xmm0","xmm1"); + &pshufd ("xmm3","xmm0",0b11111111); + &pxor ("xmm2","xmm3"); + &ret(); + +&set_label("key_192b",16); + &movaps ("xmm3","xmm0"); + &shufps ("xmm5","xmm0",0b01000100); + &$movekey (&QWP(0,$key),"xmm5"); + &shufps ("xmm3","xmm2",0b01001110); + &$movekey (&QWP(16,$key),"xmm3"); + &lea ($key,&DWP(32,$key)); + &jmp (&label("key_192b_warm")); + +&set_label("14rounds",16); + &movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey + &lea ($key,&DWP(16,$key)); + + &mov ($rounds,13); + &$movekey (&QWP(-32,$key),"xmm0"); # round 0 + &$movekey (&QWP(-16,$key),"xmm2"); # round 1 + &aeskeygenassist("xmm1","xmm2",0x01); # round 2 + &call (&label("key_256a_cold")); + &aeskeygenassist("xmm1","xmm0",0x01); # round 3 + &call (&label("key_256b")); + &aeskeygenassist("xmm1","xmm2",0x02); # round 4 + &call (&label("key_256a")); + &aeskeygenassist("xmm1","xmm0",0x02); # round 5 + &call (&label("key_256b")); + &aeskeygenassist("xmm1","xmm2",0x04); # round 6 + &call (&label("key_256a")); + &aeskeygenassist("xmm1","xmm0",0x04); # round 7 + &call (&label("key_256b")); + &aeskeygenassist("xmm1","xmm2",0x08); # round 8 + &call (&label("key_256a")); + &aeskeygenassist("xmm1","xmm0",0x08); # round 9 + &call (&label("key_256b")); + &aeskeygenassist("xmm1","xmm2",0x10); # round 10 + &call (&label("key_256a")); + &aeskeygenassist("xmm1","xmm0",0x10); # round 11 + &call (&label("key_256b")); + &aeskeygenassist("xmm1","xmm2",0x20); # round 12 + &call (&label("key_256a")); + &aeskeygenassist("xmm1","xmm0",0x20); # round 13 + &call (&label("key_256b")); + &aeskeygenassist("xmm1","xmm2",0x40); # round 14 + &call (&label("key_256a")); + &$movekey (&QWP(0,$key),"xmm0"); + &mov (&DWP(16,$key),$rounds); + &xor ("eax","eax"); + + &jmp (&label("good_key")); + +&set_label("key_256a",16); + &$movekey (&QWP(0,$key),"xmm2"); + &lea ($key,&DWP(16,$key)); +&set_label("key_256a_cold"); + &shufps ("xmm4","xmm0",0b00010000); + &xorps ("xmm0","xmm4"); + &shufps ("xmm4","xmm0",0b10001100); + &xorps ("xmm0","xmm4"); + &shufps ("xmm1","xmm1",0b11111111); # critical path + &xorps ("xmm0","xmm1"); + &ret(); + +&set_label("key_256b",16); + &$movekey (&QWP(0,$key),"xmm0"); + &lea ($key,&DWP(16,$key)); + + &shufps ("xmm4","xmm2",0b00010000); + &xorps ("xmm2","xmm4"); + &shufps ("xmm4","xmm2",0b10001100); + &xorps ("xmm2","xmm4"); + &shufps ("xmm1","xmm1",0b10101010); # critical path + &xorps ("xmm2","xmm1"); + &ret(); + +&set_label("good_key"); + &pxor ("xmm0","xmm0"); + &pxor ("xmm1","xmm1"); + &pxor ("xmm2","xmm2"); + &pxor ("xmm3","xmm3"); + &pxor ("xmm4","xmm4"); + &pxor ("xmm5","xmm5"); + &xor ("eax","eax"); + &pop ("ebx"); + &ret (); + +&set_label("bad_keybits",4); + &pxor ("xmm0","xmm0"); + &mov ("eax",-2); + &pop ("ebx"); + &ret (); +&function_end_B("${PREFIX}_set_encrypt_key_base"); + +# int $PREFIX_set_encrypt_key_alt (const unsigned char *userKey, int bits, +# AES_KEY *key) +&function_begin_B("${PREFIX}_set_encrypt_key_alt"); + &record_function_hit(3); + + &mov ("eax",&wparam(0)); + &mov ($rounds,&wparam(1)); + &mov ($key,&wparam(2)); + &push ("ebx"); + + &call (&label("pic")); +&set_label("pic"); + &blindpop("ebx"); + &lea ("ebx",&DWP(&label("key_const")."-".&label("pic"),"ebx")); + + &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey + &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0 + &lea ($key,&DWP(16,$key)); + &cmp ($rounds,256); + &je (&label("14rounds_alt")); + &cmp ($rounds,192); + &je (&label("12rounds_alt")); + &cmp ($rounds,128); + &jne (&label("bad_keybits")); + &set_label("10rounds_alt",16); &movdqa ("xmm5",&QWP(0x00,"ebx")); &mov ($rounds,8); @@ -2249,63 +2399,8 @@ if ($PREFIX eq $AESNI_PREFIX) { &jmp (&label("good_key")); -&set_label("12rounds",16); - &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey - &cmp ("ebp",1<<28); - &je (&label("12rounds_alt")); - - &mov ($rounds,11); - &$movekey (&QWP(-16,$key),"xmm0"); # round 0 - &aeskeygenassist("xmm1","xmm2",0x01); # round 1,2 - &call (&label("key_192a_cold")); - &aeskeygenassist("xmm1","xmm2",0x02); # round 2,3 - &call (&label("key_192b")); - &aeskeygenassist("xmm1","xmm2",0x04); # round 4,5 - &call (&label("key_192a")); - &aeskeygenassist("xmm1","xmm2",0x08); # round 5,6 - &call (&label("key_192b")); - &aeskeygenassist("xmm1","xmm2",0x10); # round 7,8 - &call (&label("key_192a")); - &aeskeygenassist("xmm1","xmm2",0x20); # round 8,9 - &call (&label("key_192b")); - &aeskeygenassist("xmm1","xmm2",0x40); # round 10,11 - &call (&label("key_192a")); - &aeskeygenassist("xmm1","xmm2",0x80); # round 11,12 - &call (&label("key_192b")); - &$movekey (&QWP(0,$key),"xmm0"); - &mov (&DWP(48,$key),$rounds); - - &jmp (&label("good_key")); - -&set_label("key_192a",16); - &$movekey (&QWP(0,$key),"xmm0"); - &lea ($key,&DWP(16,$key)); -&set_label("key_192a_cold",16); - &movaps ("xmm5","xmm2"); -&set_label("key_192b_warm"); - &shufps ("xmm4","xmm0",0b00010000); - &movdqa ("xmm3","xmm2"); - &xorps ("xmm0","xmm4"); - &shufps ("xmm4","xmm0",0b10001100); - &pslldq ("xmm3",4); - &xorps ("xmm0","xmm4"); - &pshufd ("xmm1","xmm1",0b01010101); # critical path - &pxor ("xmm2","xmm3"); - &pxor ("xmm0","xmm1"); - &pshufd ("xmm3","xmm0",0b11111111); - &pxor ("xmm2","xmm3"); - &ret(); - -&set_label("key_192b",16); - &movaps ("xmm3","xmm0"); - &shufps ("xmm5","xmm0",0b01000100); - &$movekey (&QWP(0,$key),"xmm5"); - &shufps ("xmm3","xmm2",0b01001110); - &$movekey (&QWP(16,$key),"xmm3"); - &lea ($key,&DWP(32,$key)); - &jmp (&label("key_192b_warm")); - &set_label("12rounds_alt",16); + &movq ("xmm2",&QWP(16,"eax")); # remaining 1/3 of *userKey &movdqa ("xmm5",&QWP(0x10,"ebx")); &movdqa ("xmm4",&QWP(0x20,"ebx")); &mov ($rounds,8); @@ -2344,72 +2439,9 @@ if ($PREFIX eq $AESNI_PREFIX) { &jmp (&label("good_key")); -&set_label("14rounds",16); +&set_label("14rounds_alt",16); &movups ("xmm2",&QWP(16,"eax")); # remaining half of *userKey &lea ($key,&DWP(16,$key)); - &cmp ("ebp",1<<28); - &je (&label("14rounds_alt")); - - &mov ($rounds,13); - &$movekey (&QWP(-32,$key),"xmm0"); # round 0 - &$movekey (&QWP(-16,$key),"xmm2"); # round 1 - &aeskeygenassist("xmm1","xmm2",0x01); # round 2 - &call (&label("key_256a_cold")); - &aeskeygenassist("xmm1","xmm0",0x01); # round 3 - &call (&label("key_256b")); - &aeskeygenassist("xmm1","xmm2",0x02); # round 4 - &call (&label("key_256a")); - &aeskeygenassist("xmm1","xmm0",0x02); # round 5 - &call (&label("key_256b")); - &aeskeygenassist("xmm1","xmm2",0x04); # round 6 - &call (&label("key_256a")); - &aeskeygenassist("xmm1","xmm0",0x04); # round 7 - &call (&label("key_256b")); - &aeskeygenassist("xmm1","xmm2",0x08); # round 8 - &call (&label("key_256a")); - &aeskeygenassist("xmm1","xmm0",0x08); # round 9 - &call (&label("key_256b")); - &aeskeygenassist("xmm1","xmm2",0x10); # round 10 - &call (&label("key_256a")); - &aeskeygenassist("xmm1","xmm0",0x10); # round 11 - &call (&label("key_256b")); - &aeskeygenassist("xmm1","xmm2",0x20); # round 12 - &call (&label("key_256a")); - &aeskeygenassist("xmm1","xmm0",0x20); # round 13 - &call (&label("key_256b")); - &aeskeygenassist("xmm1","xmm2",0x40); # round 14 - &call (&label("key_256a")); - &$movekey (&QWP(0,$key),"xmm0"); - &mov (&DWP(16,$key),$rounds); - &xor ("eax","eax"); - - &jmp (&label("good_key")); - -&set_label("key_256a",16); - &$movekey (&QWP(0,$key),"xmm2"); - &lea ($key,&DWP(16,$key)); -&set_label("key_256a_cold"); - &shufps ("xmm4","xmm0",0b00010000); - &xorps ("xmm0","xmm4"); - &shufps ("xmm4","xmm0",0b10001100); - &xorps ("xmm0","xmm4"); - &shufps ("xmm1","xmm1",0b11111111); # critical path - &xorps ("xmm0","xmm1"); - &ret(); - -&set_label("key_256b",16); - &$movekey (&QWP(0,$key),"xmm0"); - &lea ($key,&DWP(16,$key)); - - &shufps ("xmm4","xmm2",0b00010000); - &xorps ("xmm2","xmm4"); - &shufps ("xmm4","xmm2",0b10001100); - &xorps ("xmm2","xmm4"); - &shufps ("xmm1","xmm1",0b10101010); # critical path - &xorps ("xmm2","xmm1"); - &ret(); - -&set_label("14rounds_alt",16); &movdqa ("xmm5",&QWP(0x00,"ebx")); &movdqa ("xmm4",&QWP(0x20,"ebx")); &mov ($rounds,7); @@ -2467,28 +2499,14 @@ if ($PREFIX eq $AESNI_PREFIX) { &pxor ("xmm5","xmm5"); &xor ("eax","eax"); &pop ("ebx"); - &pop ("ebp"); &ret (); &set_label("bad_keybits",4); &pxor ("xmm0","xmm0"); &mov ("eax",-2); &pop ("ebx"); - &pop ("ebp"); &ret (); -&function_end_B("_aesni_set_encrypt_key"); - -# int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits, -# AES_KEY *key) -&function_begin_B("${PREFIX}_set_encrypt_key"); - &record_function_hit(3); - - &mov ("eax",&wparam(0)); - &mov ($rounds,&wparam(1)); - &mov ($key,&wparam(2)); - &call ("_aesni_set_encrypt_key"); - &ret (); -&function_end_B("${PREFIX}_set_encrypt_key"); +&function_end_B("${PREFIX}_set_encrypt_key_alt"); # void $PREFIX_encrypt_key_to_decrypt_key (AES_KEY *key) &function_begin_B("${PREFIX}_encrypt_key_to_decrypt_key"); diff --git a/crypto/fipsmodule/aes/asm/aesni-x86_64.pl b/crypto/fipsmodule/aes/asm/aesni-x86_64.pl index 8b6036ee0..ab45749c5 100644 --- a/crypto/fipsmodule/aes/asm/aesni-x86_64.pl +++ b/crypto/fipsmodule/aes/asm/aesni-x86_64.pl @@ -211,7 +211,6 @@ $movkey = $PREFIX eq "aes_hw" ? "movups" : "movups"; ("%rdi","%rsi","%rdx","%rcx"); # Unix order $code=".text\n"; -$code.=".extern OPENSSL_ia32cap_P\n"; $rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!! # this is natural Unix argument order for public $PREFIX_[ecb|cbc]_encrypt ... @@ -3245,11 +3244,14 @@ ___ # are used. Note that it's declared "abi-omnipotent", which means that # amount of volatile registers is smaller on Windows. # +# There are two variants of this function, one which uses aeskeygenassist +# ("base") and one which uses aesenclast + pshufb ("alt"). See aes/internal.h +# for details. $code.=<<___; -.globl ${PREFIX}_set_encrypt_key -.type ${PREFIX}_set_encrypt_key,\@abi-omnipotent +.globl ${PREFIX}_set_encrypt_key_base +.type ${PREFIX}_set_encrypt_key_base,\@abi-omnipotent .align 16 -${PREFIX}_set_encrypt_key: +${PREFIX}_set_encrypt_key_base: .cfi_startproc .seh_startproc _CET_ENDBR @@ -3262,9 +3264,6 @@ ${PREFIX}_set_encrypt_key: .seh_endprologue movups ($inp),%xmm0 # pull first 128 bits of *userKey xorps %xmm4,%xmm4 # low dword of xmm4 is assumed 0 - leaq OPENSSL_ia32cap_P(%rip),%r10 - movl 4(%r10),%r10d - and \$`1<<28|1<<11`,%r10d # AVX and XOP bits lea 16($key),%rax # %rax is used as modifiable copy of $key cmp \$256,$bits je .L14rounds @@ -3275,8 +3274,6 @@ ${PREFIX}_set_encrypt_key: .L10rounds: mov \$9,$bits # 10 rounds for 128-bit key - cmp \$`1<<28`,%r10d # AVX, bit no XOP - je .L10rounds_alt $movkey %xmm0,($key) # round 0 aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1 @@ -3304,79 +3301,10 @@ ${PREFIX}_set_encrypt_key: xor %eax,%eax jmp .Lenc_key_ret -.align 16 -.L10rounds_alt: - movdqa .Lkey_rotate(%rip),%xmm5 - mov \$8,%r10d - movdqa .Lkey_rcon1(%rip),%xmm4 - movdqa %xmm0,%xmm2 - movdqu %xmm0,($key) - jmp .Loop_key128 - -.align 16 -.Loop_key128: - pshufb %xmm5,%xmm0 - aesenclast %xmm4,%xmm0 - pslld \$1,%xmm4 - lea 16(%rax),%rax - - movdqa %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,-16(%rax) - movdqa %xmm0,%xmm2 - - dec %r10d - jnz .Loop_key128 - - movdqa .Lkey_rcon1b(%rip),%xmm4 - - pshufb %xmm5,%xmm0 - aesenclast %xmm4,%xmm0 - pslld \$1,%xmm4 - - movdqa %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,(%rax) - - movdqa %xmm0,%xmm2 - pshufb %xmm5,%xmm0 - aesenclast %xmm4,%xmm0 - - movdqa %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm2,%xmm3 - pslldq \$4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,16(%rax) - - mov $bits,96(%rax) # 240($key) - xor %eax,%eax - jmp .Lenc_key_ret - .align 16 .L12rounds: movq 16($inp),%xmm2 # remaining 1/3 of *userKey mov \$11,$bits # 12 rounds for 192 - cmp \$`1<<28`,%r10d # AVX, but no XOP - je .L12rounds_alt $movkey %xmm0,($key) # round 0 aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2 @@ -3400,54 +3328,11 @@ ${PREFIX}_set_encrypt_key: xor %rax, %rax jmp .Lenc_key_ret -.align 16 -.L12rounds_alt: - movdqa .Lkey_rotate192(%rip),%xmm5 - movdqa .Lkey_rcon1(%rip),%xmm4 - mov \$8,%r10d - movdqu %xmm0,($key) - jmp .Loop_key192 - -.align 16 -.Loop_key192: - movq %xmm2,0(%rax) - movdqa %xmm2,%xmm1 - pshufb %xmm5,%xmm2 - aesenclast %xmm4,%xmm2 - pslld \$1, %xmm4 - lea 24(%rax),%rax - - movdqa %xmm0,%xmm3 - pslldq \$4,%xmm0 - pxor %xmm0,%xmm3 - pslldq \$4,%xmm0 - pxor %xmm0,%xmm3 - pslldq \$4,%xmm0 - pxor %xmm3,%xmm0 - - pshufd \$0xff,%xmm0,%xmm3 - pxor %xmm1,%xmm3 - pslldq \$4,%xmm1 - pxor %xmm1,%xmm3 - - pxor %xmm2,%xmm0 - pxor %xmm3,%xmm2 - movdqu %xmm0,-16(%rax) - - dec %r10d - jnz .Loop_key192 - - mov $bits,32(%rax) # 240($key) - xor %eax,%eax - jmp .Lenc_key_ret - .align 16 .L14rounds: movups 16($inp),%xmm2 # remaining half of *userKey mov \$13,$bits # 14 rounds for 256 lea 16(%rax),%rax - cmp \$`1<<28`,%r10d # AVX, but no XOP - je .L14rounds_alt $movkey %xmm0,($key) # round 0 $movkey %xmm2,16($key) # round 1 @@ -3482,60 +3367,6 @@ ${PREFIX}_set_encrypt_key: xor %rax,%rax jmp .Lenc_key_ret -.align 16 -.L14rounds_alt: - movdqa .Lkey_rotate(%rip),%xmm5 - movdqa .Lkey_rcon1(%rip),%xmm4 - mov \$7,%r10d - movdqu %xmm0,0($key) - movdqa %xmm2,%xmm1 - movdqu %xmm2,16($key) - jmp .Loop_key256 - -.align 16 -.Loop_key256: - pshufb %xmm5,%xmm2 - aesenclast %xmm4,%xmm2 - - movdqa %xmm0,%xmm3 - pslldq \$4,%xmm0 - pxor %xmm0,%xmm3 - pslldq \$4,%xmm0 - pxor %xmm0,%xmm3 - pslldq \$4,%xmm0 - pxor %xmm3,%xmm0 - pslld \$1,%xmm4 - - pxor %xmm2,%xmm0 - movdqu %xmm0,(%rax) - - dec %r10d - jz .Ldone_key256 - - pshufd \$0xff,%xmm0,%xmm2 - pxor %xmm3,%xmm3 - aesenclast %xmm3,%xmm2 - - movdqa %xmm1,%xmm3 - pslldq \$4,%xmm1 - pxor %xmm1,%xmm3 - pslldq \$4,%xmm1 - pxor %xmm1,%xmm3 - pslldq \$4,%xmm1 - pxor %xmm3,%xmm1 - - pxor %xmm1,%xmm2 - movdqu %xmm2,16(%rax) - lea 32(%rax),%rax - movdqa %xmm2,%xmm1 - - jmp .Loop_key256 - -.Ldone_key256: - mov $bits,16(%rax) # 240($key) - xor %eax,%eax - jmp .Lenc_key_ret - .align 16 .Lbad_keybits: mov \$-2,%rax @@ -3620,7 +3451,214 @@ ${PREFIX}_set_encrypt_key: shufps \$0b10101010,%xmm1,%xmm1 # critical path xorps %xmm1,%xmm2 ret -.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key +.size ${PREFIX}_set_encrypt_key_base,.-${PREFIX}_set_encrypt_key_base + +.globl ${PREFIX}_set_encrypt_key_alt +.type ${PREFIX}_set_encrypt_key_alt,\@abi-omnipotent +.align 16 +${PREFIX}_set_encrypt_key_alt: +.cfi_startproc +.seh_startproc + _CET_ENDBR +#ifdef BORINGSSL_DISPATCH_TEST + movb \$1,BORINGSSL_function_hit+3(%rip) +#endif + sub \$8,%rsp +.cfi_adjust_cfa_offset 8 +.seh_stackalloc 8 +.seh_endprologue + movups ($inp),%xmm0 # pull first 128 bits of *userKey + xorps %xmm4,%xmm4 # low dword of xmm4 is assumed 0 + lea 16($key),%rax # %rax is used as modifiable copy of $key + cmp \$256,$bits + je .L14rounds_alt + cmp \$192,$bits + je .L12rounds_alt + cmp \$128,$bits + jne .Lbad_keybits_alt + + mov \$9,$bits # 10 rounds for 128-bit key + movdqa .Lkey_rotate(%rip),%xmm5 + mov \$8,%r10d + movdqa .Lkey_rcon1(%rip),%xmm4 + movdqa %xmm0,%xmm2 + movdqu %xmm0,($key) + jmp .Loop_key128 + +.align 16 +.Loop_key128: + pshufb %xmm5,%xmm0 + aesenclast %xmm4,%xmm0 + pslld \$1,%xmm4 + lea 16(%rax),%rax + + movdqa %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,-16(%rax) + movdqa %xmm0,%xmm2 + + dec %r10d + jnz .Loop_key128 + + movdqa .Lkey_rcon1b(%rip),%xmm4 + + pshufb %xmm5,%xmm0 + aesenclast %xmm4,%xmm0 + pslld \$1,%xmm4 + + movdqa %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + movdqa %xmm0,%xmm2 + pshufb %xmm5,%xmm0 + aesenclast %xmm4,%xmm0 + + movdqa %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm2,%xmm3 + pslldq \$4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,16(%rax) + + mov $bits,96(%rax) # 240($key) + xor %eax,%eax + jmp .Lenc_key_ret_alt + +.align 16 +.L12rounds_alt: + movq 16($inp),%xmm2 # remaining 1/3 of *userKey + mov \$11,$bits # 12 rounds for 192 + movdqa .Lkey_rotate192(%rip),%xmm5 + movdqa .Lkey_rcon1(%rip),%xmm4 + mov \$8,%r10d + movdqu %xmm0,($key) + jmp .Loop_key192 + +.align 16 +.Loop_key192: + movq %xmm2,0(%rax) + movdqa %xmm2,%xmm1 + pshufb %xmm5,%xmm2 + aesenclast %xmm4,%xmm2 + pslld \$1, %xmm4 + lea 24(%rax),%rax + + movdqa %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm3,%xmm0 + + pshufd \$0xff,%xmm0,%xmm3 + pxor %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm1,%xmm3 + + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm2 + movdqu %xmm0,-16(%rax) + + dec %r10d + jnz .Loop_key192 + + mov $bits,32(%rax) # 240($key) + xor %eax,%eax + jmp .Lenc_key_ret_alt + +.align 16 +.L14rounds_alt: + movups 16($inp),%xmm2 # remaining half of *userKey + mov \$13,$bits # 14 rounds for 256 + lea 16(%rax),%rax + movdqa .Lkey_rotate(%rip),%xmm5 + movdqa .Lkey_rcon1(%rip),%xmm4 + mov \$7,%r10d + movdqu %xmm0,0($key) + movdqa %xmm2,%xmm1 + movdqu %xmm2,16($key) + jmp .Loop_key256 + +.align 16 +.Loop_key256: + pshufb %xmm5,%xmm2 + aesenclast %xmm4,%xmm2 + + movdqa %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm0,%xmm3 + pslldq \$4,%xmm0 + pxor %xmm3,%xmm0 + pslld \$1,%xmm4 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + dec %r10d + jz .Ldone_key256 + + pshufd \$0xff,%xmm0,%xmm2 + pxor %xmm3,%xmm3 + aesenclast %xmm3,%xmm2 + + movdqa %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm1,%xmm3 + pslldq \$4,%xmm1 + pxor %xmm3,%xmm1 + + pxor %xmm1,%xmm2 + movdqu %xmm2,16(%rax) + lea 32(%rax),%rax + movdqa %xmm2,%xmm1 + + jmp .Loop_key256 + +.Ldone_key256: + mov $bits,16(%rax) # 240($key) + xor %eax,%eax + jmp .Lenc_key_ret_alt + +.align 16 +.Lbad_keybits_alt: + mov \$-2,%rax +.Lenc_key_ret_alt: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + add \$8,%rsp +.cfi_adjust_cfa_offset -8 + ret +.cfi_endproc +.seh_endproc +.size ${PREFIX}_set_encrypt_key_alt,.-${PREFIX}_set_encrypt_key_alt ___ } diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index e7f55d242..7d2db3b22 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h @@ -79,7 +79,27 @@ void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, // On x86 and x86_64, |aes_hw_set_decrypt_key| is implemented in terms of // |aes_hw_set_encrypt_key| and a conversion function. void aes_hw_encrypt_key_to_decrypt_key(AES_KEY *key); -#endif + +// There are two variants of this function, one which uses aeskeygenassist +// ("base") and one which uses aesenclast + pshufb ("alt"). aesenclast is +// overall faster but is slower on some older processors. It doesn't use AVX, +// but AVX is used as a proxy to detecting this. See +// https://groups.google.com/g/mailing.openssl.dev/c/OuFXwW4NfO8/m/7d2ZXVjkxVkJ +// +// TODO(davidben): It is unclear if the aeskeygenassist version is still +// worthwhile. However, the aesenclast version requires SSSE3. SSSE3 long +// predates AES-NI, but it's not clear if AES-NI implies SSSE3. In OpenSSL, the +// CCM AES-NI assembly seems to assume it does. +OPENSSL_INLINE int aes_hw_set_encrypt_key_alt_capable(void) { + return hwaes_capable() && CRYPTO_is_SSSE3_capable(); +} +OPENSSL_INLINE int aes_hw_set_encrypt_key_alt_preferred(void) { + return hwaes_capable() && CRYPTO_is_AVX_capable(); +} +int aes_hw_set_encrypt_key_base(const uint8_t *user_key, int bits, + AES_KEY *key); +int aes_hw_set_encrypt_key_alt(const uint8_t *user_key, int bits, AES_KEY *key); +#endif // OPENSSL_X86 || OPENSSL_X86_64 #else diff --git a/crypto/perlasm/x86asm.pl b/crypto/perlasm/x86asm.pl index f0d631067..b10eaf6be 100644 --- a/crypto/perlasm/x86asm.pl +++ b/crypto/perlasm/x86asm.pl @@ -42,10 +42,10 @@ sub ::record_function_hit &preprocessor_ifdef("BORINGSSL_DISPATCH_TEST"); &push("ebx"); &push("edx"); - &call(&label("pic")); - &set_label("pic"); + &call(&label("pic_for_function_hit")); + &set_label("pic_for_function_hit"); &blindpop("ebx"); - &lea("ebx",&DWP("BORINGSSL_function_hit+$index"."-".&label("pic"),"ebx")); + &lea("ebx",&DWP("BORINGSSL_function_hit+$index"."-".&label("pic_for_function_hit"),"ebx")); &mov("edx", 1); &movb(&BP(0, "ebx"), "dl"); &pop("edx"); diff --git a/gen/bcm/aesni-x86-apple.S b/gen/bcm/aesni-x86-apple.S index e64b4bb9f..dda66ed1d 100644 --- a/gen/bcm/aesni-x86-apple.S +++ b/gen/bcm/aesni-x86-apple.S @@ -15,10 +15,10 @@ L_aes_hw_encrypt_begin: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call L000pic -L000pic: + call L000pic_for_function_hit +L000pic_for_function_hit: popl %ebx - leal _BORINGSSL_function_hit+1-L000pic(%ebx),%ebx + leal _BORINGSSL_function_hit+1-L000pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx @@ -820,10 +820,10 @@ L_aes_hw_ctr32_encrypt_blocks_begin: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call L038pic -L038pic: + call L038pic_for_function_hit +L038pic_for_function_hit: popl %ebx - leal _BORINGSSL_function_hit+0-L038pic(%ebx),%ebx + leal _BORINGSSL_function_hit+0-L038pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx @@ -2065,31 +2065,42 @@ L076cbc_abort: popl %ebx popl %ebp ret -.private_extern __aesni_set_encrypt_key +.globl _aes_hw_set_encrypt_key_base +.private_extern _aes_hw_set_encrypt_key_base .align 4 -__aesni_set_encrypt_key: - pushl %ebp +_aes_hw_set_encrypt_key_base: +L_aes_hw_set_encrypt_key_base_begin: +#ifdef BORINGSSL_DISPATCH_TEST pushl %ebx - call L093pic -L093pic: + pushl %edx + call L093pic_for_function_hit +L093pic_for_function_hit: popl %ebx - leal Lkey_const-L093pic(%ebx),%ebx - movl L_OPENSSL_ia32cap_P$non_lazy_ptr-Lkey_const(%ebx),%ebp + leal _BORINGSSL_function_hit+3-L093pic_for_function_hit(%ebx),%ebx + movl $1,%edx + movb %dl,(%ebx) + popl %edx + popl %ebx +#endif + movl 4(%esp),%eax + movl 8(%esp),%ecx + movl 12(%esp),%edx + pushl %ebx + call L094pic +L094pic: + popl %ebx + leal Lkey_const-L094pic(%ebx),%ebx movups (%eax),%xmm0 xorps %xmm4,%xmm4 - movl 4(%ebp),%ebp leal 16(%edx),%edx - andl $268437504,%ebp cmpl $256,%ecx - je L09414rounds + je L09514rounds cmpl $192,%ecx - je L09512rounds + je L09612rounds cmpl $128,%ecx - jne L096bad_keybits + jne L097bad_keybits .align 4,0x90 -L09710rounds: - cmpl $268435456,%ebp - je L09810rounds_alt +L09810rounds: movl $9,%ecx movups %xmm0,-16(%edx) .byte 102,15,58,223,200,1 @@ -2128,13 +2139,176 @@ L099key_128_cold: xorps %xmm1,%xmm0 ret .align 4,0x90 -L09810rounds_alt: +L09612rounds: + movq 16(%eax),%xmm2 + movl $11,%ecx + movups %xmm0,-16(%edx) +.byte 102,15,58,223,202,1 + call L102key_192a_cold +.byte 102,15,58,223,202,2 + call L103key_192b +.byte 102,15,58,223,202,4 + call L104key_192a +.byte 102,15,58,223,202,8 + call L103key_192b +.byte 102,15,58,223,202,16 + call L104key_192a +.byte 102,15,58,223,202,32 + call L103key_192b +.byte 102,15,58,223,202,64 + call L104key_192a +.byte 102,15,58,223,202,128 + call L103key_192b + movups %xmm0,(%edx) + movl %ecx,48(%edx) + jmp L101good_key +.align 4,0x90 +L104key_192a: + movups %xmm0,(%edx) + leal 16(%edx),%edx +.align 4,0x90 +L102key_192a_cold: + movaps %xmm2,%xmm5 +L105key_192b_warm: + shufps $16,%xmm0,%xmm4 + movdqa %xmm2,%xmm3 + xorps %xmm4,%xmm0 + shufps $140,%xmm0,%xmm4 + pslldq $4,%xmm3 + xorps %xmm4,%xmm0 + pshufd $85,%xmm1,%xmm1 + pxor %xmm3,%xmm2 + pxor %xmm1,%xmm0 + pshufd $255,%xmm0,%xmm3 + pxor %xmm3,%xmm2 + ret +.align 4,0x90 +L103key_192b: + movaps %xmm0,%xmm3 + shufps $68,%xmm0,%xmm5 + movups %xmm5,(%edx) + shufps $78,%xmm2,%xmm3 + movups %xmm3,16(%edx) + leal 32(%edx),%edx + jmp L105key_192b_warm +.align 4,0x90 +L09514rounds: + movups 16(%eax),%xmm2 + leal 16(%edx),%edx + movl $13,%ecx + movups %xmm0,-32(%edx) + movups %xmm2,-16(%edx) +.byte 102,15,58,223,202,1 + call L106key_256a_cold +.byte 102,15,58,223,200,1 + call L107key_256b +.byte 102,15,58,223,202,2 + call L108key_256a +.byte 102,15,58,223,200,2 + call L107key_256b +.byte 102,15,58,223,202,4 + call L108key_256a +.byte 102,15,58,223,200,4 + call L107key_256b +.byte 102,15,58,223,202,8 + call L108key_256a +.byte 102,15,58,223,200,8 + call L107key_256b +.byte 102,15,58,223,202,16 + call L108key_256a +.byte 102,15,58,223,200,16 + call L107key_256b +.byte 102,15,58,223,202,32 + call L108key_256a +.byte 102,15,58,223,200,32 + call L107key_256b +.byte 102,15,58,223,202,64 + call L108key_256a + movups %xmm0,(%edx) + movl %ecx,16(%edx) + xorl %eax,%eax + jmp L101good_key +.align 4,0x90 +L108key_256a: + movups %xmm2,(%edx) + leal 16(%edx),%edx +L106key_256a_cold: + shufps $16,%xmm0,%xmm4 + xorps %xmm4,%xmm0 + shufps $140,%xmm0,%xmm4 + xorps %xmm4,%xmm0 + shufps $255,%xmm1,%xmm1 + xorps %xmm1,%xmm0 + ret +.align 4,0x90 +L107key_256b: + movups %xmm0,(%edx) + leal 16(%edx),%edx + shufps $16,%xmm2,%xmm4 + xorps %xmm4,%xmm2 + shufps $140,%xmm2,%xmm4 + xorps %xmm4,%xmm2 + shufps $170,%xmm1,%xmm1 + xorps %xmm1,%xmm2 + ret +L101good_key: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + xorl %eax,%eax + popl %ebx + ret +.align 2,0x90 +L097bad_keybits: + pxor %xmm0,%xmm0 + movl $-2,%eax + popl %ebx + ret +.globl _aes_hw_set_encrypt_key_alt +.private_extern _aes_hw_set_encrypt_key_alt +.align 4 +_aes_hw_set_encrypt_key_alt: +L_aes_hw_set_encrypt_key_alt_begin: +#ifdef BORINGSSL_DISPATCH_TEST + pushl %ebx + pushl %edx + call L109pic_for_function_hit +L109pic_for_function_hit: + popl %ebx + leal _BORINGSSL_function_hit+3-L109pic_for_function_hit(%ebx),%ebx + movl $1,%edx + movb %dl,(%ebx) + popl %edx + popl %ebx +#endif + movl 4(%esp),%eax + movl 8(%esp),%ecx + movl 12(%esp),%edx + pushl %ebx + call L110pic +L110pic: + popl %ebx + leal Lkey_const-L110pic(%ebx),%ebx + movups (%eax),%xmm0 + xorps %xmm4,%xmm4 + leal 16(%edx),%edx + cmpl $256,%ecx + je L11114rounds_alt + cmpl $192,%ecx + je L11212rounds_alt + cmpl $128,%ecx + jne L113bad_keybits +.align 4,0x90 +L11410rounds_alt: movdqa (%ebx),%xmm5 movl $8,%ecx movdqa 32(%ebx),%xmm4 movdqa %xmm0,%xmm2 movdqu %xmm0,-16(%edx) -L102loop_key128: +L115loop_key128: .byte 102,15,56,0,197 .byte 102,15,56,221,196 pslld $1,%xmm4 @@ -2150,7 +2324,7 @@ L102loop_key128: movdqu %xmm0,-16(%edx) movdqa %xmm0,%xmm2 decl %ecx - jnz L102loop_key128 + jnz L115loop_key128 movdqa 48(%ebx),%xmm4 .byte 102,15,56,0,197 .byte 102,15,56,221,196 @@ -2178,69 +2352,15 @@ L102loop_key128: movdqu %xmm0,16(%edx) movl $9,%ecx movl %ecx,96(%edx) - jmp L101good_key + jmp L116good_key .align 4,0x90 -L09512rounds: +L11212rounds_alt: movq 16(%eax),%xmm2 - cmpl $268435456,%ebp - je L10312rounds_alt - movl $11,%ecx - movups %xmm0,-16(%edx) -.byte 102,15,58,223,202,1 - call L104key_192a_cold -.byte 102,15,58,223,202,2 - call L105key_192b -.byte 102,15,58,223,202,4 - call L106key_192a -.byte 102,15,58,223,202,8 - call L105key_192b -.byte 102,15,58,223,202,16 - call L106key_192a -.byte 102,15,58,223,202,32 - call L105key_192b -.byte 102,15,58,223,202,64 - call L106key_192a -.byte 102,15,58,223,202,128 - call L105key_192b - movups %xmm0,(%edx) - movl %ecx,48(%edx) - jmp L101good_key -.align 4,0x90 -L106key_192a: - movups %xmm0,(%edx) - leal 16(%edx),%edx -.align 4,0x90 -L104key_192a_cold: - movaps %xmm2,%xmm5 -L107key_192b_warm: - shufps $16,%xmm0,%xmm4 - movdqa %xmm2,%xmm3 - xorps %xmm4,%xmm0 - shufps $140,%xmm0,%xmm4 - pslldq $4,%xmm3 - xorps %xmm4,%xmm0 - pshufd $85,%xmm1,%xmm1 - pxor %xmm3,%xmm2 - pxor %xmm1,%xmm0 - pshufd $255,%xmm0,%xmm3 - pxor %xmm3,%xmm2 - ret -.align 4,0x90 -L105key_192b: - movaps %xmm0,%xmm3 - shufps $68,%xmm0,%xmm5 - movups %xmm5,(%edx) - shufps $78,%xmm2,%xmm3 - movups %xmm3,16(%edx) - leal 32(%edx),%edx - jmp L107key_192b_warm -.align 4,0x90 -L10312rounds_alt: movdqa 16(%ebx),%xmm5 movdqa 32(%ebx),%xmm4 movl $8,%ecx movdqu %xmm0,-16(%edx) -L108loop_key192: +L117loop_key192: movq %xmm2,(%edx) movdqa %xmm2,%xmm1 .byte 102,15,56,0,213 @@ -2262,81 +2382,21 @@ L108loop_key192: pxor %xmm3,%xmm2 movdqu %xmm0,-16(%edx) decl %ecx - jnz L108loop_key192 + jnz L117loop_key192 movl $11,%ecx movl %ecx,32(%edx) - jmp L101good_key + jmp L116good_key .align 4,0x90 -L09414rounds: +L11114rounds_alt: movups 16(%eax),%xmm2 leal 16(%edx),%edx - cmpl $268435456,%ebp - je L10914rounds_alt - movl $13,%ecx - movups %xmm0,-32(%edx) - movups %xmm2,-16(%edx) -.byte 102,15,58,223,202,1 - call L110key_256a_cold -.byte 102,15,58,223,200,1 - call L111key_256b -.byte 102,15,58,223,202,2 - call L112key_256a -.byte 102,15,58,223,200,2 - call L111key_256b -.byte 102,15,58,223,202,4 - call L112key_256a -.byte 102,15,58,223,200,4 - call L111key_256b -.byte 102,15,58,223,202,8 - call L112key_256a -.byte 102,15,58,223,200,8 - call L111key_256b -.byte 102,15,58,223,202,16 - call L112key_256a -.byte 102,15,58,223,200,16 - call L111key_256b -.byte 102,15,58,223,202,32 - call L112key_256a -.byte 102,15,58,223,200,32 - call L111key_256b -.byte 102,15,58,223,202,64 - call L112key_256a - movups %xmm0,(%edx) - movl %ecx,16(%edx) - xorl %eax,%eax - jmp L101good_key -.align 4,0x90 -L112key_256a: - movups %xmm2,(%edx) - leal 16(%edx),%edx -L110key_256a_cold: - shufps $16,%xmm0,%xmm4 - xorps %xmm4,%xmm0 - shufps $140,%xmm0,%xmm4 - xorps %xmm4,%xmm0 - shufps $255,%xmm1,%xmm1 - xorps %xmm1,%xmm0 - ret -.align 4,0x90 -L111key_256b: - movups %xmm0,(%edx) - leal 16(%edx),%edx - shufps $16,%xmm2,%xmm4 - xorps %xmm4,%xmm2 - shufps $140,%xmm2,%xmm4 - xorps %xmm4,%xmm2 - shufps $170,%xmm1,%xmm1 - xorps %xmm1,%xmm2 - ret -.align 4,0x90 -L10914rounds_alt: movdqa (%ebx),%xmm5 movdqa 32(%ebx),%xmm4 movl $7,%ecx movdqu %xmm0,-32(%edx) movdqa %xmm2,%xmm1 movdqu %xmm2,-16(%edx) -L113loop_key256: +L118loop_key256: .byte 102,15,56,0,213 .byte 102,15,56,221,212 movdqa %xmm0,%xmm3 @@ -2350,7 +2410,7 @@ L113loop_key256: pxor %xmm2,%xmm0 movdqu %xmm0,(%edx) decl %ecx - jz L114done_key256 + jz L119done_key256 pshufd $255,%xmm0,%xmm2 pxor %xmm3,%xmm3 .byte 102,15,56,221,211 @@ -2365,11 +2425,11 @@ L113loop_key256: movdqu %xmm2,16(%edx) leal 32(%edx),%edx movdqa %xmm2,%xmm1 - jmp L113loop_key256 -L114done_key256: + jmp L118loop_key256 +L119done_key256: movl $13,%ecx movl %ecx,16(%edx) -L101good_key: +L116good_key: pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -2378,36 +2438,12 @@ L101good_key: pxor %xmm5,%xmm5 xorl %eax,%eax popl %ebx - popl %ebp ret .align 2,0x90 -L096bad_keybits: +L113bad_keybits: pxor %xmm0,%xmm0 movl $-2,%eax popl %ebx - popl %ebp - ret -.globl _aes_hw_set_encrypt_key -.private_extern _aes_hw_set_encrypt_key -.align 4 -_aes_hw_set_encrypt_key: -L_aes_hw_set_encrypt_key_begin: -#ifdef BORINGSSL_DISPATCH_TEST - pushl %ebx - pushl %edx - call L115pic -L115pic: - popl %ebx - leal _BORINGSSL_function_hit+3-L115pic(%ebx),%ebx - movl $1,%edx - movb %dl,(%ebx) - popl %edx - popl %ebx -#endif - movl 4(%esp),%eax - movl 8(%esp),%ecx - movl 12(%esp),%edx - call __aesni_set_encrypt_key ret .globl _aes_hw_encrypt_key_to_decrypt_key .private_extern _aes_hw_encrypt_key_to_decrypt_key @@ -2424,7 +2460,7 @@ L_aes_hw_encrypt_key_to_decrypt_key_begin: movups %xmm1,(%edx) leal 16(%edx),%edx leal -16(%eax),%eax -L116dec_key_inverse: +L120dec_key_inverse: movups (%edx),%xmm0 movups (%eax),%xmm1 .byte 102,15,56,219,192 @@ -2434,7 +2470,7 @@ L116dec_key_inverse: movups %xmm0,16(%eax) movups %xmm1,-16(%edx) cmpl %edx,%eax - ja L116dec_key_inverse + ja L120dec_key_inverse movups (%edx),%xmm0 .byte 102,15,56,219,192 movups %xmm0,(%edx) @@ -2451,8 +2487,4 @@ Lkey_const: .byte 83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83 .byte 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115 .byte 115,108,46,111,114,103,62,0 -.section __IMPORT,__pointers,non_lazy_symbol_pointers -L_OPENSSL_ia32cap_P$non_lazy_ptr: -.indirect_symbol _OPENSSL_ia32cap_P -.long 0 #endif // !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86) && defined(__APPLE__) diff --git a/gen/bcm/aesni-x86-linux.S b/gen/bcm/aesni-x86-linux.S index 1f15c7159..6fefe1755 100644 --- a/gen/bcm/aesni-x86-linux.S +++ b/gen/bcm/aesni-x86-linux.S @@ -16,10 +16,10 @@ aes_hw_encrypt: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call .L000pic -.L000pic: + call .L000pic_for_function_hit +.L000pic_for_function_hit: popl %ebx - leal BORINGSSL_function_hit+1-.L000pic(%ebx),%ebx + leal BORINGSSL_function_hit+1-.L000pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx @@ -847,10 +847,10 @@ aes_hw_ctr32_encrypt_blocks: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call .L038pic -.L038pic: + call .L038pic_for_function_hit +.L038pic_for_function_hit: popl %ebx - leal BORINGSSL_function_hit+0-.L038pic(%ebx),%ebx + leal BORINGSSL_function_hit+0-.L038pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx @@ -2099,32 +2099,43 @@ aes_hw_cbc_encrypt: popl %ebp ret .size aes_hw_cbc_encrypt,.-.L_aes_hw_cbc_encrypt_begin -.hidden _aesni_set_encrypt_key -.type _aesni_set_encrypt_key,@function +.globl aes_hw_set_encrypt_key_base +.hidden aes_hw_set_encrypt_key_base +.type aes_hw_set_encrypt_key_base,@function .align 16 -_aesni_set_encrypt_key: - pushl %ebp +aes_hw_set_encrypt_key_base: +.L_aes_hw_set_encrypt_key_base_begin: +#ifdef BORINGSSL_DISPATCH_TEST pushl %ebx - call .L093pic -.L093pic: + pushl %edx + call .L093pic_for_function_hit +.L093pic_for_function_hit: popl %ebx - leal .Lkey_const-.L093pic(%ebx),%ebx - leal OPENSSL_ia32cap_P-.Lkey_const(%ebx),%ebp + leal BORINGSSL_function_hit+3-.L093pic_for_function_hit(%ebx),%ebx + movl $1,%edx + movb %dl,(%ebx) + popl %edx + popl %ebx +#endif + movl 4(%esp),%eax + movl 8(%esp),%ecx + movl 12(%esp),%edx + pushl %ebx + call .L094pic +.L094pic: + popl %ebx + leal .Lkey_const-.L094pic(%ebx),%ebx movups (%eax),%xmm0 xorps %xmm4,%xmm4 - movl 4(%ebp),%ebp leal 16(%edx),%edx - andl $268437504,%ebp cmpl $256,%ecx - je .L09414rounds + je .L09514rounds cmpl $192,%ecx - je .L09512rounds + je .L09612rounds cmpl $128,%ecx - jne .L096bad_keybits + jne .L097bad_keybits .align 16 -.L09710rounds: - cmpl $268435456,%ebp - je .L09810rounds_alt +.L09810rounds: movl $9,%ecx movups %xmm0,-16(%edx) .byte 102,15,58,223,200,1 @@ -2163,13 +2174,178 @@ _aesni_set_encrypt_key: xorps %xmm1,%xmm0 ret .align 16 -.L09810rounds_alt: +.L09612rounds: + movq 16(%eax),%xmm2 + movl $11,%ecx + movups %xmm0,-16(%edx) +.byte 102,15,58,223,202,1 + call .L102key_192a_cold +.byte 102,15,58,223,202,2 + call .L103key_192b +.byte 102,15,58,223,202,4 + call .L104key_192a +.byte 102,15,58,223,202,8 + call .L103key_192b +.byte 102,15,58,223,202,16 + call .L104key_192a +.byte 102,15,58,223,202,32 + call .L103key_192b +.byte 102,15,58,223,202,64 + call .L104key_192a +.byte 102,15,58,223,202,128 + call .L103key_192b + movups %xmm0,(%edx) + movl %ecx,48(%edx) + jmp .L101good_key +.align 16 +.L104key_192a: + movups %xmm0,(%edx) + leal 16(%edx),%edx +.align 16 +.L102key_192a_cold: + movaps %xmm2,%xmm5 +.L105key_192b_warm: + shufps $16,%xmm0,%xmm4 + movdqa %xmm2,%xmm3 + xorps %xmm4,%xmm0 + shufps $140,%xmm0,%xmm4 + pslldq $4,%xmm3 + xorps %xmm4,%xmm0 + pshufd $85,%xmm1,%xmm1 + pxor %xmm3,%xmm2 + pxor %xmm1,%xmm0 + pshufd $255,%xmm0,%xmm3 + pxor %xmm3,%xmm2 + ret +.align 16 +.L103key_192b: + movaps %xmm0,%xmm3 + shufps $68,%xmm0,%xmm5 + movups %xmm5,(%edx) + shufps $78,%xmm2,%xmm3 + movups %xmm3,16(%edx) + leal 32(%edx),%edx + jmp .L105key_192b_warm +.align 16 +.L09514rounds: + movups 16(%eax),%xmm2 + leal 16(%edx),%edx + movl $13,%ecx + movups %xmm0,-32(%edx) + movups %xmm2,-16(%edx) +.byte 102,15,58,223,202,1 + call .L106key_256a_cold +.byte 102,15,58,223,200,1 + call .L107key_256b +.byte 102,15,58,223,202,2 + call .L108key_256a +.byte 102,15,58,223,200,2 + call .L107key_256b +.byte 102,15,58,223,202,4 + call .L108key_256a +.byte 102,15,58,223,200,4 + call .L107key_256b +.byte 102,15,58,223,202,8 + call .L108key_256a +.byte 102,15,58,223,200,8 + call .L107key_256b +.byte 102,15,58,223,202,16 + call .L108key_256a +.byte 102,15,58,223,200,16 + call .L107key_256b +.byte 102,15,58,223,202,32 + call .L108key_256a +.byte 102,15,58,223,200,32 + call .L107key_256b +.byte 102,15,58,223,202,64 + call .L108key_256a + movups %xmm0,(%edx) + movl %ecx,16(%edx) + xorl %eax,%eax + jmp .L101good_key +.align 16 +.L108key_256a: + movups %xmm2,(%edx) + leal 16(%edx),%edx +.L106key_256a_cold: + shufps $16,%xmm0,%xmm4 + xorps %xmm4,%xmm0 + shufps $140,%xmm0,%xmm4 + xorps %xmm4,%xmm0 + shufps $255,%xmm1,%xmm1 + xorps %xmm1,%xmm0 + ret +.align 16 +.L107key_256b: + movups %xmm0,(%edx) + leal 16(%edx),%edx + shufps $16,%xmm2,%xmm4 + xorps %xmm4,%xmm2 + shufps $140,%xmm2,%xmm4 + xorps %xmm4,%xmm2 + shufps $170,%xmm1,%xmm1 + xorps %xmm1,%xmm2 + ret +.L101good_key: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + xorl %eax,%eax + popl %ebx + ret +.align 4 +.L097bad_keybits: + pxor %xmm0,%xmm0 + movl $-2,%eax + popl %ebx + ret +.size aes_hw_set_encrypt_key_base,.-.L_aes_hw_set_encrypt_key_base_begin +.globl aes_hw_set_encrypt_key_alt +.hidden aes_hw_set_encrypt_key_alt +.type aes_hw_set_encrypt_key_alt,@function +.align 16 +aes_hw_set_encrypt_key_alt: +.L_aes_hw_set_encrypt_key_alt_begin: +#ifdef BORINGSSL_DISPATCH_TEST + pushl %ebx + pushl %edx + call .L109pic_for_function_hit +.L109pic_for_function_hit: + popl %ebx + leal BORINGSSL_function_hit+3-.L109pic_for_function_hit(%ebx),%ebx + movl $1,%edx + movb %dl,(%ebx) + popl %edx + popl %ebx +#endif + movl 4(%esp),%eax + movl 8(%esp),%ecx + movl 12(%esp),%edx + pushl %ebx + call .L110pic +.L110pic: + popl %ebx + leal .Lkey_const-.L110pic(%ebx),%ebx + movups (%eax),%xmm0 + xorps %xmm4,%xmm4 + leal 16(%edx),%edx + cmpl $256,%ecx + je .L11114rounds_alt + cmpl $192,%ecx + je .L11212rounds_alt + cmpl $128,%ecx + jne .L113bad_keybits +.align 16 +.L11410rounds_alt: movdqa (%ebx),%xmm5 movl $8,%ecx movdqa 32(%ebx),%xmm4 movdqa %xmm0,%xmm2 movdqu %xmm0,-16(%edx) -.L102loop_key128: +.L115loop_key128: .byte 102,15,56,0,197 .byte 102,15,56,221,196 pslld $1,%xmm4 @@ -2185,7 +2361,7 @@ _aesni_set_encrypt_key: movdqu %xmm0,-16(%edx) movdqa %xmm0,%xmm2 decl %ecx - jnz .L102loop_key128 + jnz .L115loop_key128 movdqa 48(%ebx),%xmm4 .byte 102,15,56,0,197 .byte 102,15,56,221,196 @@ -2213,69 +2389,15 @@ _aesni_set_encrypt_key: movdqu %xmm0,16(%edx) movl $9,%ecx movl %ecx,96(%edx) - jmp .L101good_key + jmp .L116good_key .align 16 -.L09512rounds: +.L11212rounds_alt: movq 16(%eax),%xmm2 - cmpl $268435456,%ebp - je .L10312rounds_alt - movl $11,%ecx - movups %xmm0,-16(%edx) -.byte 102,15,58,223,202,1 - call .L104key_192a_cold -.byte 102,15,58,223,202,2 - call .L105key_192b -.byte 102,15,58,223,202,4 - call .L106key_192a -.byte 102,15,58,223,202,8 - call .L105key_192b -.byte 102,15,58,223,202,16 - call .L106key_192a -.byte 102,15,58,223,202,32 - call .L105key_192b -.byte 102,15,58,223,202,64 - call .L106key_192a -.byte 102,15,58,223,202,128 - call .L105key_192b - movups %xmm0,(%edx) - movl %ecx,48(%edx) - jmp .L101good_key -.align 16 -.L106key_192a: - movups %xmm0,(%edx) - leal 16(%edx),%edx -.align 16 -.L104key_192a_cold: - movaps %xmm2,%xmm5 -.L107key_192b_warm: - shufps $16,%xmm0,%xmm4 - movdqa %xmm2,%xmm3 - xorps %xmm4,%xmm0 - shufps $140,%xmm0,%xmm4 - pslldq $4,%xmm3 - xorps %xmm4,%xmm0 - pshufd $85,%xmm1,%xmm1 - pxor %xmm3,%xmm2 - pxor %xmm1,%xmm0 - pshufd $255,%xmm0,%xmm3 - pxor %xmm3,%xmm2 - ret -.align 16 -.L105key_192b: - movaps %xmm0,%xmm3 - shufps $68,%xmm0,%xmm5 - movups %xmm5,(%edx) - shufps $78,%xmm2,%xmm3 - movups %xmm3,16(%edx) - leal 32(%edx),%edx - jmp .L107key_192b_warm -.align 16 -.L10312rounds_alt: movdqa 16(%ebx),%xmm5 movdqa 32(%ebx),%xmm4 movl $8,%ecx movdqu %xmm0,-16(%edx) -.L108loop_key192: +.L117loop_key192: movq %xmm2,(%edx) movdqa %xmm2,%xmm1 .byte 102,15,56,0,213 @@ -2297,81 +2419,21 @@ _aesni_set_encrypt_key: pxor %xmm3,%xmm2 movdqu %xmm0,-16(%edx) decl %ecx - jnz .L108loop_key192 + jnz .L117loop_key192 movl $11,%ecx movl %ecx,32(%edx) - jmp .L101good_key + jmp .L116good_key .align 16 -.L09414rounds: +.L11114rounds_alt: movups 16(%eax),%xmm2 leal 16(%edx),%edx - cmpl $268435456,%ebp - je .L10914rounds_alt - movl $13,%ecx - movups %xmm0,-32(%edx) - movups %xmm2,-16(%edx) -.byte 102,15,58,223,202,1 - call .L110key_256a_cold -.byte 102,15,58,223,200,1 - call .L111key_256b -.byte 102,15,58,223,202,2 - call .L112key_256a -.byte 102,15,58,223,200,2 - call .L111key_256b -.byte 102,15,58,223,202,4 - call .L112key_256a -.byte 102,15,58,223,200,4 - call .L111key_256b -.byte 102,15,58,223,202,8 - call .L112key_256a -.byte 102,15,58,223,200,8 - call .L111key_256b -.byte 102,15,58,223,202,16 - call .L112key_256a -.byte 102,15,58,223,200,16 - call .L111key_256b -.byte 102,15,58,223,202,32 - call .L112key_256a -.byte 102,15,58,223,200,32 - call .L111key_256b -.byte 102,15,58,223,202,64 - call .L112key_256a - movups %xmm0,(%edx) - movl %ecx,16(%edx) - xorl %eax,%eax - jmp .L101good_key -.align 16 -.L112key_256a: - movups %xmm2,(%edx) - leal 16(%edx),%edx -.L110key_256a_cold: - shufps $16,%xmm0,%xmm4 - xorps %xmm4,%xmm0 - shufps $140,%xmm0,%xmm4 - xorps %xmm4,%xmm0 - shufps $255,%xmm1,%xmm1 - xorps %xmm1,%xmm0 - ret -.align 16 -.L111key_256b: - movups %xmm0,(%edx) - leal 16(%edx),%edx - shufps $16,%xmm2,%xmm4 - xorps %xmm4,%xmm2 - shufps $140,%xmm2,%xmm4 - xorps %xmm4,%xmm2 - shufps $170,%xmm1,%xmm1 - xorps %xmm1,%xmm2 - ret -.align 16 -.L10914rounds_alt: movdqa (%ebx),%xmm5 movdqa 32(%ebx),%xmm4 movl $7,%ecx movdqu %xmm0,-32(%edx) movdqa %xmm2,%xmm1 movdqu %xmm2,-16(%edx) -.L113loop_key256: +.L118loop_key256: .byte 102,15,56,0,213 .byte 102,15,56,221,212 movdqa %xmm0,%xmm3 @@ -2385,7 +2447,7 @@ _aesni_set_encrypt_key: pxor %xmm2,%xmm0 movdqu %xmm0,(%edx) decl %ecx - jz .L114done_key256 + jz .L119done_key256 pshufd $255,%xmm0,%xmm2 pxor %xmm3,%xmm3 .byte 102,15,56,221,211 @@ -2400,11 +2462,11 @@ _aesni_set_encrypt_key: movdqu %xmm2,16(%edx) leal 32(%edx),%edx movdqa %xmm2,%xmm1 - jmp .L113loop_key256 -.L114done_key256: + jmp .L118loop_key256 +.L119done_key256: movl $13,%ecx movl %ecx,16(%edx) -.L101good_key: +.L116good_key: pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -2413,40 +2475,14 @@ _aesni_set_encrypt_key: pxor %xmm5,%xmm5 xorl %eax,%eax popl %ebx - popl %ebp ret .align 4 -.L096bad_keybits: +.L113bad_keybits: pxor %xmm0,%xmm0 movl $-2,%eax popl %ebx - popl %ebp ret -.size _aesni_set_encrypt_key,.-_aesni_set_encrypt_key -.globl aes_hw_set_encrypt_key -.hidden aes_hw_set_encrypt_key -.type aes_hw_set_encrypt_key,@function -.align 16 -aes_hw_set_encrypt_key: -.L_aes_hw_set_encrypt_key_begin: -#ifdef BORINGSSL_DISPATCH_TEST - pushl %ebx - pushl %edx - call .L115pic -.L115pic: - popl %ebx - leal BORINGSSL_function_hit+3-.L115pic(%ebx),%ebx - movl $1,%edx - movb %dl,(%ebx) - popl %edx - popl %ebx -#endif - movl 4(%esp),%eax - movl 8(%esp),%ecx - movl 12(%esp),%edx - call _aesni_set_encrypt_key - ret -.size aes_hw_set_encrypt_key,.-.L_aes_hw_set_encrypt_key_begin +.size aes_hw_set_encrypt_key_alt,.-.L_aes_hw_set_encrypt_key_alt_begin .globl aes_hw_encrypt_key_to_decrypt_key .hidden aes_hw_encrypt_key_to_decrypt_key .type aes_hw_encrypt_key_to_decrypt_key,@function @@ -2463,7 +2499,7 @@ aes_hw_encrypt_key_to_decrypt_key: movups %xmm1,(%edx) leal 16(%edx),%edx leal -16(%eax),%eax -.L116dec_key_inverse: +.L120dec_key_inverse: movups (%edx),%xmm0 movups (%eax),%xmm1 .byte 102,15,56,219,192 @@ -2473,7 +2509,7 @@ aes_hw_encrypt_key_to_decrypt_key: movups %xmm0,16(%eax) movups %xmm1,-16(%edx) cmpl %edx,%eax - ja .L116dec_key_inverse + ja .L120dec_key_inverse movups (%edx),%xmm0 .byte 102,15,56,219,192 movups %xmm0,(%edx) diff --git a/gen/bcm/aesni-x86-win.asm b/gen/bcm/aesni-x86-win.asm index 2d32e7756..4453afb1b 100644 --- a/gen/bcm/aesni-x86-win.asm +++ b/gen/bcm/aesni-x86-win.asm @@ -13,7 +13,6 @@ section .text code align=64 %else section .text code %endif -;extern _OPENSSL_ia32cap_P %ifdef BORINGSSL_DISPATCH_TEST extern _BORINGSSL_function_hit %endif @@ -24,10 +23,10 @@ L$_aes_hw_encrypt_begin: %ifdef BORINGSSL_DISPATCH_TEST push ebx push edx - call L$000pic -L$000pic: + call L$000pic_for_function_hit +L$000pic_for_function_hit: pop ebx - lea ebx,[(_BORINGSSL_function_hit+1-L$000pic)+ebx] + lea ebx,[(_BORINGSSL_function_hit+1-L$000pic_for_function_hit)+ebx] mov edx,1 mov BYTE [ebx],dl pop edx @@ -816,10 +815,10 @@ L$_aes_hw_ctr32_encrypt_blocks_begin: %ifdef BORINGSSL_DISPATCH_TEST push ebx push edx - call L$038pic -L$038pic: + call L$038pic_for_function_hit +L$038pic_for_function_hit: pop ebx - lea ebx,[(_BORINGSSL_function_hit+0-L$038pic)+ebx] + lea ebx,[(_BORINGSSL_function_hit+0-L$038pic_for_function_hit)+ebx] mov edx,1 mov BYTE [ebx],dl pop edx @@ -2058,30 +2057,41 @@ L$076cbc_abort: pop ebx pop ebp ret +global _aes_hw_set_encrypt_key_base align 16 -__aesni_set_encrypt_key: - push ebp +_aes_hw_set_encrypt_key_base: +L$_aes_hw_set_encrypt_key_base_begin: +%ifdef BORINGSSL_DISPATCH_TEST push ebx - call L$093pic -L$093pic: + push edx + call L$093pic_for_function_hit +L$093pic_for_function_hit: pop ebx - lea ebx,[(L$key_const-L$093pic)+ebx] - lea ebp,[_OPENSSL_ia32cap_P] + lea ebx,[(_BORINGSSL_function_hit+3-L$093pic_for_function_hit)+ebx] + mov edx,1 + mov BYTE [ebx],dl + pop edx + pop ebx +%endif + mov eax,DWORD [4+esp] + mov ecx,DWORD [8+esp] + mov edx,DWORD [12+esp] + push ebx + call L$094pic +L$094pic: + pop ebx + lea ebx,[(L$key_const-L$094pic)+ebx] movups xmm0,[eax] xorps xmm4,xmm4 - mov ebp,DWORD [4+ebp] lea edx,[16+edx] - and ebp,268437504 cmp ecx,256 - je NEAR L$09414rounds + je NEAR L$09514rounds cmp ecx,192 - je NEAR L$09512rounds + je NEAR L$09612rounds cmp ecx,128 - jne NEAR L$096bad_keybits + jne NEAR L$097bad_keybits align 16 -L$09710rounds: - cmp ebp,268435456 - je NEAR L$09810rounds_alt +L$09810rounds: mov ecx,9 movups [edx-16],xmm0 db 102,15,58,223,200,1 @@ -2120,13 +2130,175 @@ L$099key_128_cold: xorps xmm0,xmm1 ret align 16 -L$09810rounds_alt: +L$09612rounds: + movq xmm2,[16+eax] + mov ecx,11 + movups [edx-16],xmm0 +db 102,15,58,223,202,1 + call L$102key_192a_cold +db 102,15,58,223,202,2 + call L$103key_192b +db 102,15,58,223,202,4 + call L$104key_192a +db 102,15,58,223,202,8 + call L$103key_192b +db 102,15,58,223,202,16 + call L$104key_192a +db 102,15,58,223,202,32 + call L$103key_192b +db 102,15,58,223,202,64 + call L$104key_192a +db 102,15,58,223,202,128 + call L$103key_192b + movups [edx],xmm0 + mov DWORD [48+edx],ecx + jmp NEAR L$101good_key +align 16 +L$104key_192a: + movups [edx],xmm0 + lea edx,[16+edx] +align 16 +L$102key_192a_cold: + movaps xmm5,xmm2 +L$105key_192b_warm: + shufps xmm4,xmm0,16 + movdqa xmm3,xmm2 + xorps xmm0,xmm4 + shufps xmm4,xmm0,140 + pslldq xmm3,4 + xorps xmm0,xmm4 + pshufd xmm1,xmm1,85 + pxor xmm2,xmm3 + pxor xmm0,xmm1 + pshufd xmm3,xmm0,255 + pxor xmm2,xmm3 + ret +align 16 +L$103key_192b: + movaps xmm3,xmm0 + shufps xmm5,xmm0,68 + movups [edx],xmm5 + shufps xmm3,xmm2,78 + movups [16+edx],xmm3 + lea edx,[32+edx] + jmp NEAR L$105key_192b_warm +align 16 +L$09514rounds: + movups xmm2,[16+eax] + lea edx,[16+edx] + mov ecx,13 + movups [edx-32],xmm0 + movups [edx-16],xmm2 +db 102,15,58,223,202,1 + call L$106key_256a_cold +db 102,15,58,223,200,1 + call L$107key_256b +db 102,15,58,223,202,2 + call L$108key_256a +db 102,15,58,223,200,2 + call L$107key_256b +db 102,15,58,223,202,4 + call L$108key_256a +db 102,15,58,223,200,4 + call L$107key_256b +db 102,15,58,223,202,8 + call L$108key_256a +db 102,15,58,223,200,8 + call L$107key_256b +db 102,15,58,223,202,16 + call L$108key_256a +db 102,15,58,223,200,16 + call L$107key_256b +db 102,15,58,223,202,32 + call L$108key_256a +db 102,15,58,223,200,32 + call L$107key_256b +db 102,15,58,223,202,64 + call L$108key_256a + movups [edx],xmm0 + mov DWORD [16+edx],ecx + xor eax,eax + jmp NEAR L$101good_key +align 16 +L$108key_256a: + movups [edx],xmm2 + lea edx,[16+edx] +L$106key_256a_cold: + shufps xmm4,xmm0,16 + xorps xmm0,xmm4 + shufps xmm4,xmm0,140 + xorps xmm0,xmm4 + shufps xmm1,xmm1,255 + xorps xmm0,xmm1 + ret +align 16 +L$107key_256b: + movups [edx],xmm0 + lea edx,[16+edx] + shufps xmm4,xmm2,16 + xorps xmm2,xmm4 + shufps xmm4,xmm2,140 + xorps xmm2,xmm4 + shufps xmm1,xmm1,170 + xorps xmm2,xmm1 + ret +L$101good_key: + pxor xmm0,xmm0 + pxor xmm1,xmm1 + pxor xmm2,xmm2 + pxor xmm3,xmm3 + pxor xmm4,xmm4 + pxor xmm5,xmm5 + xor eax,eax + pop ebx + ret +align 4 +L$097bad_keybits: + pxor xmm0,xmm0 + mov eax,-2 + pop ebx + ret +global _aes_hw_set_encrypt_key_alt +align 16 +_aes_hw_set_encrypt_key_alt: +L$_aes_hw_set_encrypt_key_alt_begin: +%ifdef BORINGSSL_DISPATCH_TEST + push ebx + push edx + call L$109pic_for_function_hit +L$109pic_for_function_hit: + pop ebx + lea ebx,[(_BORINGSSL_function_hit+3-L$109pic_for_function_hit)+ebx] + mov edx,1 + mov BYTE [ebx],dl + pop edx + pop ebx +%endif + mov eax,DWORD [4+esp] + mov ecx,DWORD [8+esp] + mov edx,DWORD [12+esp] + push ebx + call L$110pic +L$110pic: + pop ebx + lea ebx,[(L$key_const-L$110pic)+ebx] + movups xmm0,[eax] + xorps xmm4,xmm4 + lea edx,[16+edx] + cmp ecx,256 + je NEAR L$11114rounds_alt + cmp ecx,192 + je NEAR L$11212rounds_alt + cmp ecx,128 + jne NEAR L$113bad_keybits +align 16 +L$11410rounds_alt: movdqa xmm5,[ebx] mov ecx,8 movdqa xmm4,[32+ebx] movdqa xmm2,xmm0 movdqu [edx-16],xmm0 -L$102loop_key128: +L$115loop_key128: db 102,15,56,0,197 db 102,15,56,221,196 pslld xmm4,1 @@ -2142,7 +2314,7 @@ db 102,15,56,221,196 movdqu [edx-16],xmm0 movdqa xmm2,xmm0 dec ecx - jnz NEAR L$102loop_key128 + jnz NEAR L$115loop_key128 movdqa xmm4,[48+ebx] db 102,15,56,0,197 db 102,15,56,221,196 @@ -2170,69 +2342,15 @@ db 102,15,56,221,196 movdqu [16+edx],xmm0 mov ecx,9 mov DWORD [96+edx],ecx - jmp NEAR L$101good_key + jmp NEAR L$116good_key align 16 -L$09512rounds: +L$11212rounds_alt: movq xmm2,[16+eax] - cmp ebp,268435456 - je NEAR L$10312rounds_alt - mov ecx,11 - movups [edx-16],xmm0 -db 102,15,58,223,202,1 - call L$104key_192a_cold -db 102,15,58,223,202,2 - call L$105key_192b -db 102,15,58,223,202,4 - call L$106key_192a -db 102,15,58,223,202,8 - call L$105key_192b -db 102,15,58,223,202,16 - call L$106key_192a -db 102,15,58,223,202,32 - call L$105key_192b -db 102,15,58,223,202,64 - call L$106key_192a -db 102,15,58,223,202,128 - call L$105key_192b - movups [edx],xmm0 - mov DWORD [48+edx],ecx - jmp NEAR L$101good_key -align 16 -L$106key_192a: - movups [edx],xmm0 - lea edx,[16+edx] -align 16 -L$104key_192a_cold: - movaps xmm5,xmm2 -L$107key_192b_warm: - shufps xmm4,xmm0,16 - movdqa xmm3,xmm2 - xorps xmm0,xmm4 - shufps xmm4,xmm0,140 - pslldq xmm3,4 - xorps xmm0,xmm4 - pshufd xmm1,xmm1,85 - pxor xmm2,xmm3 - pxor xmm0,xmm1 - pshufd xmm3,xmm0,255 - pxor xmm2,xmm3 - ret -align 16 -L$105key_192b: - movaps xmm3,xmm0 - shufps xmm5,xmm0,68 - movups [edx],xmm5 - shufps xmm3,xmm2,78 - movups [16+edx],xmm3 - lea edx,[32+edx] - jmp NEAR L$107key_192b_warm -align 16 -L$10312rounds_alt: movdqa xmm5,[16+ebx] movdqa xmm4,[32+ebx] mov ecx,8 movdqu [edx-16],xmm0 -L$108loop_key192: +L$117loop_key192: movq [edx],xmm2 movdqa xmm1,xmm2 db 102,15,56,0,213 @@ -2254,81 +2372,21 @@ db 102,15,56,221,212 pxor xmm2,xmm3 movdqu [edx-16],xmm0 dec ecx - jnz NEAR L$108loop_key192 + jnz NEAR L$117loop_key192 mov ecx,11 mov DWORD [32+edx],ecx - jmp NEAR L$101good_key + jmp NEAR L$116good_key align 16 -L$09414rounds: +L$11114rounds_alt: movups xmm2,[16+eax] lea edx,[16+edx] - cmp ebp,268435456 - je NEAR L$10914rounds_alt - mov ecx,13 - movups [edx-32],xmm0 - movups [edx-16],xmm2 -db 102,15,58,223,202,1 - call L$110key_256a_cold -db 102,15,58,223,200,1 - call L$111key_256b -db 102,15,58,223,202,2 - call L$112key_256a -db 102,15,58,223,200,2 - call L$111key_256b -db 102,15,58,223,202,4 - call L$112key_256a -db 102,15,58,223,200,4 - call L$111key_256b -db 102,15,58,223,202,8 - call L$112key_256a -db 102,15,58,223,200,8 - call L$111key_256b -db 102,15,58,223,202,16 - call L$112key_256a -db 102,15,58,223,200,16 - call L$111key_256b -db 102,15,58,223,202,32 - call L$112key_256a -db 102,15,58,223,200,32 - call L$111key_256b -db 102,15,58,223,202,64 - call L$112key_256a - movups [edx],xmm0 - mov DWORD [16+edx],ecx - xor eax,eax - jmp NEAR L$101good_key -align 16 -L$112key_256a: - movups [edx],xmm2 - lea edx,[16+edx] -L$110key_256a_cold: - shufps xmm4,xmm0,16 - xorps xmm0,xmm4 - shufps xmm4,xmm0,140 - xorps xmm0,xmm4 - shufps xmm1,xmm1,255 - xorps xmm0,xmm1 - ret -align 16 -L$111key_256b: - movups [edx],xmm0 - lea edx,[16+edx] - shufps xmm4,xmm2,16 - xorps xmm2,xmm4 - shufps xmm4,xmm2,140 - xorps xmm2,xmm4 - shufps xmm1,xmm1,170 - xorps xmm2,xmm1 - ret -align 16 -L$10914rounds_alt: movdqa xmm5,[ebx] movdqa xmm4,[32+ebx] mov ecx,7 movdqu [edx-32],xmm0 movdqa xmm1,xmm2 movdqu [edx-16],xmm2 -L$113loop_key256: +L$118loop_key256: db 102,15,56,0,213 db 102,15,56,221,212 movdqa xmm3,xmm0 @@ -2342,7 +2400,7 @@ db 102,15,56,221,212 pxor xmm0,xmm2 movdqu [edx],xmm0 dec ecx - jz NEAR L$114done_key256 + jz NEAR L$119done_key256 pshufd xmm2,xmm0,255 pxor xmm3,xmm3 db 102,15,56,221,211 @@ -2357,11 +2415,11 @@ db 102,15,56,221,211 movdqu [16+edx],xmm2 lea edx,[32+edx] movdqa xmm1,xmm2 - jmp NEAR L$113loop_key256 -L$114done_key256: + jmp NEAR L$118loop_key256 +L$119done_key256: mov ecx,13 mov DWORD [16+edx],ecx -L$101good_key: +L$116good_key: pxor xmm0,xmm0 pxor xmm1,xmm1 pxor xmm2,xmm2 @@ -2370,35 +2428,12 @@ L$101good_key: pxor xmm5,xmm5 xor eax,eax pop ebx - pop ebp ret align 4 -L$096bad_keybits: +L$113bad_keybits: pxor xmm0,xmm0 mov eax,-2 pop ebx - pop ebp - ret -global _aes_hw_set_encrypt_key -align 16 -_aes_hw_set_encrypt_key: -L$_aes_hw_set_encrypt_key_begin: -%ifdef BORINGSSL_DISPATCH_TEST - push ebx - push edx - call L$115pic -L$115pic: - pop ebx - lea ebx,[(_BORINGSSL_function_hit+3-L$115pic)+ebx] - mov edx,1 - mov BYTE [ebx],dl - pop edx - pop ebx -%endif - mov eax,DWORD [4+esp] - mov ecx,DWORD [8+esp] - mov edx,DWORD [12+esp] - call __aesni_set_encrypt_key ret global _aes_hw_encrypt_key_to_decrypt_key align 16 @@ -2414,7 +2449,7 @@ L$_aes_hw_encrypt_key_to_decrypt_key_begin: movups [edx],xmm1 lea edx,[16+edx] lea eax,[eax-16] -L$116dec_key_inverse: +L$120dec_key_inverse: movups xmm0,[edx] movups xmm1,[eax] db 102,15,56,219,192 @@ -2424,7 +2459,7 @@ db 102,15,56,219,201 movups [16+eax],xmm0 movups [edx-16],xmm1 cmp eax,edx - ja NEAR L$116dec_key_inverse + ja NEAR L$120dec_key_inverse movups xmm0,[edx] db 102,15,56,219,192 movups [edx],xmm0 @@ -2441,8 +2476,6 @@ db 65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69 db 83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83 db 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115 db 115,108,46,111,114,103,62,0 -segment .bss -common _OPENSSL_ia32cap_P 16 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret diff --git a/gen/bcm/aesni-x86_64-apple.S b/gen/bcm/aesni-x86_64-apple.S index ccf9f8f25..23c15c3d2 100644 --- a/gen/bcm/aesni-x86_64-apple.S +++ b/gen/bcm/aesni-x86_64-apple.S @@ -5,7 +5,6 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) .text - .globl _aes_hw_encrypt .private_extern _aes_hw_encrypt @@ -1945,11 +1944,11 @@ L$dec_key_inverse: ret -.globl _aes_hw_set_encrypt_key -.private_extern _aes_hw_set_encrypt_key +.globl _aes_hw_set_encrypt_key_base +.private_extern _aes_hw_set_encrypt_key_base .p2align 4 -_aes_hw_set_encrypt_key: +_aes_hw_set_encrypt_key_base: _CET_ENDBR @@ -1962,9 +1961,6 @@ _CET_ENDBR movups (%rdi),%xmm0 xorps %xmm4,%xmm4 - leaq _OPENSSL_ia32cap_P(%rip),%r10 - movl 4(%r10),%r10d - andl $268437504,%r10d leaq 16(%rdx),%rax cmpl $256,%esi je L$14rounds @@ -1975,8 +1971,6 @@ _CET_ENDBR L$10rounds: movl $9,%esi - cmpl $268435456,%r10d - je L$10rounds_alt movups %xmm0,(%rdx) .byte 102,15,58,223,200,1 @@ -2004,79 +1998,10 @@ L$10rounds: xorl %eax,%eax jmp L$enc_key_ret -.p2align 4 -L$10rounds_alt: - movdqa L$key_rotate(%rip),%xmm5 - movl $8,%r10d - movdqa L$key_rcon1(%rip),%xmm4 - movdqa %xmm0,%xmm2 - movdqu %xmm0,(%rdx) - jmp L$oop_key128 - -.p2align 4 -L$oop_key128: -.byte 102,15,56,0,197 -.byte 102,15,56,221,196 - pslld $1,%xmm4 - leaq 16(%rax),%rax - - movdqa %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,-16(%rax) - movdqa %xmm0,%xmm2 - - decl %r10d - jnz L$oop_key128 - - movdqa L$key_rcon1b(%rip),%xmm4 - -.byte 102,15,56,0,197 -.byte 102,15,56,221,196 - pslld $1,%xmm4 - - movdqa %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,(%rax) - - movdqa %xmm0,%xmm2 -.byte 102,15,56,0,197 -.byte 102,15,56,221,196 - - movdqa %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,16(%rax) - - movl %esi,96(%rax) - xorl %eax,%eax - jmp L$enc_key_ret - .p2align 4 L$12rounds: movq 16(%rdi),%xmm2 movl $11,%esi - cmpl $268435456,%r10d - je L$12rounds_alt movups %xmm0,(%rdx) .byte 102,15,58,223,202,1 @@ -2100,54 +2025,11 @@ L$12rounds: xorq %rax,%rax jmp L$enc_key_ret -.p2align 4 -L$12rounds_alt: - movdqa L$key_rotate192(%rip),%xmm5 - movdqa L$key_rcon1(%rip),%xmm4 - movl $8,%r10d - movdqu %xmm0,(%rdx) - jmp L$oop_key192 - -.p2align 4 -L$oop_key192: - movq %xmm2,0(%rax) - movdqa %xmm2,%xmm1 -.byte 102,15,56,0,213 -.byte 102,15,56,221,212 - pslld $1,%xmm4 - leaq 24(%rax),%rax - - movdqa %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm3,%xmm0 - - pshufd $0xff,%xmm0,%xmm3 - pxor %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm1,%xmm3 - - pxor %xmm2,%xmm0 - pxor %xmm3,%xmm2 - movdqu %xmm0,-16(%rax) - - decl %r10d - jnz L$oop_key192 - - movl %esi,32(%rax) - xorl %eax,%eax - jmp L$enc_key_ret - .p2align 4 L$14rounds: movups 16(%rdi),%xmm2 movl $13,%esi leaq 16(%rax),%rax - cmpl $268435456,%r10d - je L$14rounds_alt movups %xmm0,(%rdx) movups %xmm2,16(%rdx) @@ -2182,60 +2064,6 @@ L$14rounds: xorq %rax,%rax jmp L$enc_key_ret -.p2align 4 -L$14rounds_alt: - movdqa L$key_rotate(%rip),%xmm5 - movdqa L$key_rcon1(%rip),%xmm4 - movl $7,%r10d - movdqu %xmm0,0(%rdx) - movdqa %xmm2,%xmm1 - movdqu %xmm2,16(%rdx) - jmp L$oop_key256 - -.p2align 4 -L$oop_key256: -.byte 102,15,56,0,213 -.byte 102,15,56,221,212 - - movdqa %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm3,%xmm0 - pslld $1,%xmm4 - - pxor %xmm2,%xmm0 - movdqu %xmm0,(%rax) - - decl %r10d - jz L$done_key256 - - pshufd $0xff,%xmm0,%xmm2 - pxor %xmm3,%xmm3 -.byte 102,15,56,221,211 - - movdqa %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm3,%xmm1 - - pxor %xmm1,%xmm2 - movdqu %xmm2,16(%rax) - leaq 32(%rax),%rax - movdqa %xmm2,%xmm1 - - jmp L$oop_key256 - -L$done_key256: - movl %esi,16(%rax) - xorl %eax,%eax - jmp L$enc_key_ret - .p2align 4 L$bad_keybits: movq $-2,%rax @@ -2321,6 +2149,214 @@ L$key_expansion_256b: xorps %xmm1,%xmm2 ret + +.globl _aes_hw_set_encrypt_key_alt +.private_extern _aes_hw_set_encrypt_key_alt + +.p2align 4 +_aes_hw_set_encrypt_key_alt: + + +_CET_ENDBR +#ifdef BORINGSSL_DISPATCH_TEST + movb $1,_BORINGSSL_function_hit+3(%rip) +#endif + subq $8,%rsp + + + + movups (%rdi),%xmm0 + xorps %xmm4,%xmm4 + leaq 16(%rdx),%rax + cmpl $256,%esi + je L$14rounds_alt + cmpl $192,%esi + je L$12rounds_alt + cmpl $128,%esi + jne L$bad_keybits_alt + + movl $9,%esi + movdqa L$key_rotate(%rip),%xmm5 + movl $8,%r10d + movdqa L$key_rcon1(%rip),%xmm4 + movdqa %xmm0,%xmm2 + movdqu %xmm0,(%rdx) + jmp L$oop_key128 + +.p2align 4 +L$oop_key128: +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + pslld $1,%xmm4 + leaq 16(%rax),%rax + + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,-16(%rax) + movdqa %xmm0,%xmm2 + + decl %r10d + jnz L$oop_key128 + + movdqa L$key_rcon1b(%rip),%xmm4 + +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + pslld $1,%xmm4 + + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + movdqa %xmm0,%xmm2 +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,16(%rax) + + movl %esi,96(%rax) + xorl %eax,%eax + jmp L$enc_key_ret_alt + +.p2align 4 +L$12rounds_alt: + movq 16(%rdi),%xmm2 + movl $11,%esi + movdqa L$key_rotate192(%rip),%xmm5 + movdqa L$key_rcon1(%rip),%xmm4 + movl $8,%r10d + movdqu %xmm0,(%rdx) + jmp L$oop_key192 + +.p2align 4 +L$oop_key192: + movq %xmm2,0(%rax) + movdqa %xmm2,%xmm1 +.byte 102,15,56,0,213 +.byte 102,15,56,221,212 + pslld $1,%xmm4 + leaq 24(%rax),%rax + + movdqa %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm3,%xmm0 + + pshufd $0xff,%xmm0,%xmm3 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm2 + movdqu %xmm0,-16(%rax) + + decl %r10d + jnz L$oop_key192 + + movl %esi,32(%rax) + xorl %eax,%eax + jmp L$enc_key_ret_alt + +.p2align 4 +L$14rounds_alt: + movups 16(%rdi),%xmm2 + movl $13,%esi + leaq 16(%rax),%rax + movdqa L$key_rotate(%rip),%xmm5 + movdqa L$key_rcon1(%rip),%xmm4 + movl $7,%r10d + movdqu %xmm0,0(%rdx) + movdqa %xmm2,%xmm1 + movdqu %xmm2,16(%rdx) + jmp L$oop_key256 + +.p2align 4 +L$oop_key256: +.byte 102,15,56,0,213 +.byte 102,15,56,221,212 + + movdqa %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm3,%xmm0 + pslld $1,%xmm4 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + decl %r10d + jz L$done_key256 + + pshufd $0xff,%xmm0,%xmm2 + pxor %xmm3,%xmm3 +.byte 102,15,56,221,211 + + movdqa %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm3,%xmm1 + + pxor %xmm1,%xmm2 + movdqu %xmm2,16(%rax) + leaq 32(%rax),%rax + movdqa %xmm2,%xmm1 + + jmp L$oop_key256 + +L$done_key256: + movl %esi,16(%rax) + xorl %eax,%eax + jmp L$enc_key_ret_alt + +.p2align 4 +L$bad_keybits_alt: + movq $-2,%rax +L$enc_key_ret_alt: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + addq $8,%rsp + + ret + + + .section __DATA,__const .p2align 6 L$bswap_mask: diff --git a/gen/bcm/aesni-x86_64-linux.S b/gen/bcm/aesni-x86_64-linux.S index 38ed6e72a..cd695b425 100644 --- a/gen/bcm/aesni-x86_64-linux.S +++ b/gen/bcm/aesni-x86_64-linux.S @@ -5,8 +5,6 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) .text -.extern OPENSSL_ia32cap_P -.hidden OPENSSL_ia32cap_P .globl aes_hw_encrypt .hidden aes_hw_encrypt .type aes_hw_encrypt,@function @@ -1947,11 +1945,11 @@ _CET_ENDBR ret .cfi_endproc .size aes_hw_encrypt_key_to_decrypt_key,.-aes_hw_encrypt_key_to_decrypt_key -.globl aes_hw_set_encrypt_key -.hidden aes_hw_set_encrypt_key -.type aes_hw_set_encrypt_key,@function +.globl aes_hw_set_encrypt_key_base +.hidden aes_hw_set_encrypt_key_base +.type aes_hw_set_encrypt_key_base,@function .align 16 -aes_hw_set_encrypt_key: +aes_hw_set_encrypt_key_base: .cfi_startproc _CET_ENDBR @@ -1964,9 +1962,6 @@ _CET_ENDBR movups (%rdi),%xmm0 xorps %xmm4,%xmm4 - leaq OPENSSL_ia32cap_P(%rip),%r10 - movl 4(%r10),%r10d - andl $268437504,%r10d leaq 16(%rdx),%rax cmpl $256,%esi je .L14rounds @@ -1977,8 +1972,6 @@ _CET_ENDBR .L10rounds: movl $9,%esi - cmpl $268435456,%r10d - je .L10rounds_alt movups %xmm0,(%rdx) .byte 102,15,58,223,200,1 @@ -2006,79 +1999,10 @@ _CET_ENDBR xorl %eax,%eax jmp .Lenc_key_ret -.align 16 -.L10rounds_alt: - movdqa .Lkey_rotate(%rip),%xmm5 - movl $8,%r10d - movdqa .Lkey_rcon1(%rip),%xmm4 - movdqa %xmm0,%xmm2 - movdqu %xmm0,(%rdx) - jmp .Loop_key128 - -.align 16 -.Loop_key128: -.byte 102,15,56,0,197 -.byte 102,15,56,221,196 - pslld $1,%xmm4 - leaq 16(%rax),%rax - - movdqa %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,-16(%rax) - movdqa %xmm0,%xmm2 - - decl %r10d - jnz .Loop_key128 - - movdqa .Lkey_rcon1b(%rip),%xmm4 - -.byte 102,15,56,0,197 -.byte 102,15,56,221,196 - pslld $1,%xmm4 - - movdqa %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,(%rax) - - movdqa %xmm0,%xmm2 -.byte 102,15,56,0,197 -.byte 102,15,56,221,196 - - movdqa %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm2,%xmm3 - pslldq $4,%xmm2 - pxor %xmm3,%xmm2 - - pxor %xmm2,%xmm0 - movdqu %xmm0,16(%rax) - - movl %esi,96(%rax) - xorl %eax,%eax - jmp .Lenc_key_ret - .align 16 .L12rounds: movq 16(%rdi),%xmm2 movl $11,%esi - cmpl $268435456,%r10d - je .L12rounds_alt movups %xmm0,(%rdx) .byte 102,15,58,223,202,1 @@ -2102,54 +2026,11 @@ _CET_ENDBR xorq %rax,%rax jmp .Lenc_key_ret -.align 16 -.L12rounds_alt: - movdqa .Lkey_rotate192(%rip),%xmm5 - movdqa .Lkey_rcon1(%rip),%xmm4 - movl $8,%r10d - movdqu %xmm0,(%rdx) - jmp .Loop_key192 - -.align 16 -.Loop_key192: - movq %xmm2,0(%rax) - movdqa %xmm2,%xmm1 -.byte 102,15,56,0,213 -.byte 102,15,56,221,212 - pslld $1,%xmm4 - leaq 24(%rax),%rax - - movdqa %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm3,%xmm0 - - pshufd $0xff,%xmm0,%xmm3 - pxor %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm1,%xmm3 - - pxor %xmm2,%xmm0 - pxor %xmm3,%xmm2 - movdqu %xmm0,-16(%rax) - - decl %r10d - jnz .Loop_key192 - - movl %esi,32(%rax) - xorl %eax,%eax - jmp .Lenc_key_ret - .align 16 .L14rounds: movups 16(%rdi),%xmm2 movl $13,%esi leaq 16(%rax),%rax - cmpl $268435456,%r10d - je .L14rounds_alt movups %xmm0,(%rdx) movups %xmm2,16(%rdx) @@ -2184,60 +2065,6 @@ _CET_ENDBR xorq %rax,%rax jmp .Lenc_key_ret -.align 16 -.L14rounds_alt: - movdqa .Lkey_rotate(%rip),%xmm5 - movdqa .Lkey_rcon1(%rip),%xmm4 - movl $7,%r10d - movdqu %xmm0,0(%rdx) - movdqa %xmm2,%xmm1 - movdqu %xmm2,16(%rdx) - jmp .Loop_key256 - -.align 16 -.Loop_key256: -.byte 102,15,56,0,213 -.byte 102,15,56,221,212 - - movdqa %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm0,%xmm3 - pslldq $4,%xmm0 - pxor %xmm3,%xmm0 - pslld $1,%xmm4 - - pxor %xmm2,%xmm0 - movdqu %xmm0,(%rax) - - decl %r10d - jz .Ldone_key256 - - pshufd $0xff,%xmm0,%xmm2 - pxor %xmm3,%xmm3 -.byte 102,15,56,221,211 - - movdqa %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm1,%xmm3 - pslldq $4,%xmm1 - pxor %xmm3,%xmm1 - - pxor %xmm1,%xmm2 - movdqu %xmm2,16(%rax) - leaq 32(%rax),%rax - movdqa %xmm2,%xmm1 - - jmp .Loop_key256 - -.Ldone_key256: - movl %esi,16(%rax) - xorl %eax,%eax - jmp .Lenc_key_ret - .align 16 .Lbad_keybits: movq $-2,%rax @@ -2322,7 +2149,215 @@ _CET_ENDBR shufps $170,%xmm1,%xmm1 xorps %xmm1,%xmm2 ret -.size aes_hw_set_encrypt_key,.-aes_hw_set_encrypt_key +.size aes_hw_set_encrypt_key_base,.-aes_hw_set_encrypt_key_base + +.globl aes_hw_set_encrypt_key_alt +.hidden aes_hw_set_encrypt_key_alt +.type aes_hw_set_encrypt_key_alt,@function +.align 16 +aes_hw_set_encrypt_key_alt: +.cfi_startproc + +_CET_ENDBR +#ifdef BORINGSSL_DISPATCH_TEST + movb $1,BORINGSSL_function_hit+3(%rip) +#endif + subq $8,%rsp +.cfi_adjust_cfa_offset 8 + + + movups (%rdi),%xmm0 + xorps %xmm4,%xmm4 + leaq 16(%rdx),%rax + cmpl $256,%esi + je .L14rounds_alt + cmpl $192,%esi + je .L12rounds_alt + cmpl $128,%esi + jne .Lbad_keybits_alt + + movl $9,%esi + movdqa .Lkey_rotate(%rip),%xmm5 + movl $8,%r10d + movdqa .Lkey_rcon1(%rip),%xmm4 + movdqa %xmm0,%xmm2 + movdqu %xmm0,(%rdx) + jmp .Loop_key128 + +.align 16 +.Loop_key128: +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + pslld $1,%xmm4 + leaq 16(%rax),%rax + + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,-16(%rax) + movdqa %xmm0,%xmm2 + + decl %r10d + jnz .Loop_key128 + + movdqa .Lkey_rcon1b(%rip),%xmm4 + +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + pslld $1,%xmm4 + + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + movdqa %xmm0,%xmm2 +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + + pxor %xmm2,%xmm0 + movdqu %xmm0,16(%rax) + + movl %esi,96(%rax) + xorl %eax,%eax + jmp .Lenc_key_ret_alt + +.align 16 +.L12rounds_alt: + movq 16(%rdi),%xmm2 + movl $11,%esi + movdqa .Lkey_rotate192(%rip),%xmm5 + movdqa .Lkey_rcon1(%rip),%xmm4 + movl $8,%r10d + movdqu %xmm0,(%rdx) + jmp .Loop_key192 + +.align 16 +.Loop_key192: + movq %xmm2,0(%rax) + movdqa %xmm2,%xmm1 +.byte 102,15,56,0,213 +.byte 102,15,56,221,212 + pslld $1,%xmm4 + leaq 24(%rax),%rax + + movdqa %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm3,%xmm0 + + pshufd $0xff,%xmm0,%xmm3 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm2 + movdqu %xmm0,-16(%rax) + + decl %r10d + jnz .Loop_key192 + + movl %esi,32(%rax) + xorl %eax,%eax + jmp .Lenc_key_ret_alt + +.align 16 +.L14rounds_alt: + movups 16(%rdi),%xmm2 + movl $13,%esi + leaq 16(%rax),%rax + movdqa .Lkey_rotate(%rip),%xmm5 + movdqa .Lkey_rcon1(%rip),%xmm4 + movl $7,%r10d + movdqu %xmm0,0(%rdx) + movdqa %xmm2,%xmm1 + movdqu %xmm2,16(%rdx) + jmp .Loop_key256 + +.align 16 +.Loop_key256: +.byte 102,15,56,0,213 +.byte 102,15,56,221,212 + + movdqa %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm3,%xmm0 + pslld $1,%xmm4 + + pxor %xmm2,%xmm0 + movdqu %xmm0,(%rax) + + decl %r10d + jz .Ldone_key256 + + pshufd $0xff,%xmm0,%xmm2 + pxor %xmm3,%xmm3 +.byte 102,15,56,221,211 + + movdqa %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm3,%xmm1 + + pxor %xmm1,%xmm2 + movdqu %xmm2,16(%rax) + leaq 32(%rax),%rax + movdqa %xmm2,%xmm1 + + jmp .Loop_key256 + +.Ldone_key256: + movl %esi,16(%rax) + xorl %eax,%eax + jmp .Lenc_key_ret_alt + +.align 16 +.Lbad_keybits_alt: + movq $-2,%rax +.Lenc_key_ret_alt: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + addq $8,%rsp +.cfi_adjust_cfa_offset -8 + ret +.cfi_endproc + +.size aes_hw_set_encrypt_key_alt,.-aes_hw_set_encrypt_key_alt .section .rodata .align 64 .Lbswap_mask: diff --git a/gen/bcm/aesni-x86_64-win.asm b/gen/bcm/aesni-x86_64-win.asm index 64dd6be85..4876ed79e 100644 --- a/gen/bcm/aesni-x86_64-win.asm +++ b/gen/bcm/aesni-x86_64-win.asm @@ -13,7 +13,6 @@ default rel %endif section .text code align=64 -EXTERN OPENSSL_ia32cap_P global aes_hw_encrypt ALIGN 16 @@ -2051,25 +2050,22 @@ $L$dec_key_inverse: ret -global aes_hw_set_encrypt_key +global aes_hw_set_encrypt_key_base ALIGN 16 -aes_hw_set_encrypt_key: +aes_hw_set_encrypt_key_base: -$L$SEH_begin_aes_hw_set_encrypt_key_1: +$L$SEH_begin_aes_hw_set_encrypt_key_base_1: _CET_ENDBR %ifdef BORINGSSL_DISPATCH_TEST mov BYTE[((BORINGSSL_function_hit+3))],1 %endif sub rsp,8 -$L$SEH_prologue_aes_hw_set_encrypt_key_2: -$L$SEH_endprologue_aes_hw_set_encrypt_key_3: +$L$SEH_prologue_aes_hw_set_encrypt_key_base_2: +$L$SEH_endprologue_aes_hw_set_encrypt_key_base_3: movups xmm0,XMMWORD[rcx] xorps xmm4,xmm4 - lea r10,[OPENSSL_ia32cap_P] - mov r10d,DWORD[4+r10] - and r10d,268437504 lea rax,[16+r8] cmp edx,256 je NEAR $L$14rounds @@ -2080,8 +2076,6 @@ $L$SEH_endprologue_aes_hw_set_encrypt_key_3: $L$10rounds: mov edx,9 - cmp r10d,268435456 - je NEAR $L$10rounds_alt movups XMMWORD[r8],xmm0 DB 102,15,58,223,200,1 @@ -2109,79 +2103,10 @@ $L$10rounds: xor eax,eax jmp NEAR $L$enc_key_ret -ALIGN 16 -$L$10rounds_alt: - movdqa xmm5,XMMWORD[$L$key_rotate] - mov r10d,8 - movdqa xmm4,XMMWORD[$L$key_rcon1] - movdqa xmm2,xmm0 - movdqu XMMWORD[r8],xmm0 - jmp NEAR $L$oop_key128 - -ALIGN 16 -$L$oop_key128: -DB 102,15,56,0,197 - DB 102,15,56,221,196 - pslld xmm4,1 - lea rax,[16+rax] - - movdqa xmm3,xmm2 - pslldq xmm2,4 - pxor xmm3,xmm2 - pslldq xmm2,4 - pxor xmm3,xmm2 - pslldq xmm2,4 - pxor xmm2,xmm3 - - pxor xmm0,xmm2 - movdqu XMMWORD[(-16)+rax],xmm0 - movdqa xmm2,xmm0 - - dec r10d - jnz NEAR $L$oop_key128 - - movdqa xmm4,XMMWORD[$L$key_rcon1b] - -DB 102,15,56,0,197 - DB 102,15,56,221,196 - pslld xmm4,1 - - movdqa xmm3,xmm2 - pslldq xmm2,4 - pxor xmm3,xmm2 - pslldq xmm2,4 - pxor xmm3,xmm2 - pslldq xmm2,4 - pxor xmm2,xmm3 - - pxor xmm0,xmm2 - movdqu XMMWORD[rax],xmm0 - - movdqa xmm2,xmm0 -DB 102,15,56,0,197 - DB 102,15,56,221,196 - - movdqa xmm3,xmm2 - pslldq xmm2,4 - pxor xmm3,xmm2 - pslldq xmm2,4 - pxor xmm3,xmm2 - pslldq xmm2,4 - pxor xmm2,xmm3 - - pxor xmm0,xmm2 - movdqu XMMWORD[16+rax],xmm0 - - mov DWORD[96+rax],edx - xor eax,eax - jmp NEAR $L$enc_key_ret - ALIGN 16 $L$12rounds: movq xmm2,QWORD[16+rcx] mov edx,11 - cmp r10d,268435456 - je NEAR $L$12rounds_alt movups XMMWORD[r8],xmm0 DB 102,15,58,223,202,1 @@ -2205,54 +2130,11 @@ $L$12rounds: xor rax,rax jmp NEAR $L$enc_key_ret -ALIGN 16 -$L$12rounds_alt: - movdqa xmm5,XMMWORD[$L$key_rotate192] - movdqa xmm4,XMMWORD[$L$key_rcon1] - mov r10d,8 - movdqu XMMWORD[r8],xmm0 - jmp NEAR $L$oop_key192 - -ALIGN 16 -$L$oop_key192: - movq QWORD[rax],xmm2 - movdqa xmm1,xmm2 -DB 102,15,56,0,213 - DB 102,15,56,221,212 - pslld xmm4,1 - lea rax,[24+rax] - - movdqa xmm3,xmm0 - pslldq xmm0,4 - pxor xmm3,xmm0 - pslldq xmm0,4 - pxor xmm3,xmm0 - pslldq xmm0,4 - pxor xmm0,xmm3 - - pshufd xmm3,xmm0,0xff - pxor xmm3,xmm1 - pslldq xmm1,4 - pxor xmm3,xmm1 - - pxor xmm0,xmm2 - pxor xmm2,xmm3 - movdqu XMMWORD[(-16)+rax],xmm0 - - dec r10d - jnz NEAR $L$oop_key192 - - mov DWORD[32+rax],edx - xor eax,eax - jmp NEAR $L$enc_key_ret - ALIGN 16 $L$14rounds: movups xmm2,XMMWORD[16+rcx] mov edx,13 lea rax,[16+rax] - cmp r10d,268435456 - je NEAR $L$14rounds_alt movups XMMWORD[r8],xmm0 movups XMMWORD[16+r8],xmm2 @@ -2287,60 +2169,6 @@ $L$14rounds: xor rax,rax jmp NEAR $L$enc_key_ret -ALIGN 16 -$L$14rounds_alt: - movdqa xmm5,XMMWORD[$L$key_rotate] - movdqa xmm4,XMMWORD[$L$key_rcon1] - mov r10d,7 - movdqu XMMWORD[r8],xmm0 - movdqa xmm1,xmm2 - movdqu XMMWORD[16+r8],xmm2 - jmp NEAR $L$oop_key256 - -ALIGN 16 -$L$oop_key256: -DB 102,15,56,0,213 - DB 102,15,56,221,212 - - movdqa xmm3,xmm0 - pslldq xmm0,4 - pxor xmm3,xmm0 - pslldq xmm0,4 - pxor xmm3,xmm0 - pslldq xmm0,4 - pxor xmm0,xmm3 - pslld xmm4,1 - - pxor xmm0,xmm2 - movdqu XMMWORD[rax],xmm0 - - dec r10d - jz NEAR $L$done_key256 - - pshufd xmm2,xmm0,0xff - pxor xmm3,xmm3 - DB 102,15,56,221,211 - - movdqa xmm3,xmm1 - pslldq xmm1,4 - pxor xmm3,xmm1 - pslldq xmm1,4 - pxor xmm3,xmm1 - pslldq xmm1,4 - pxor xmm1,xmm3 - - pxor xmm2,xmm1 - movdqu XMMWORD[16+rax],xmm2 - lea rax,[32+rax] - movdqa xmm1,xmm2 - - jmp NEAR $L$oop_key256 - -$L$done_key256: - mov DWORD[16+rax],edx - xor eax,eax - jmp NEAR $L$enc_key_ret - ALIGN 16 $L$bad_keybits: mov rax,-2 @@ -2355,7 +2183,7 @@ $L$enc_key_ret: ret -$L$SEH_end_aes_hw_set_encrypt_key_4: +$L$SEH_end_aes_hw_set_encrypt_key_base_4: ALIGN 16 $L$key_expansion_128: @@ -2426,6 +2254,213 @@ $L$key_expansion_256b: xorps xmm2,xmm1 ret + +global aes_hw_set_encrypt_key_alt + +ALIGN 16 +aes_hw_set_encrypt_key_alt: + +$L$SEH_begin_aes_hw_set_encrypt_key_alt_1: +_CET_ENDBR +%ifdef BORINGSSL_DISPATCH_TEST + mov BYTE[((BORINGSSL_function_hit+3))],1 +%endif + sub rsp,8 + +$L$SEH_prologue_aes_hw_set_encrypt_key_alt_2: +$L$SEH_endprologue_aes_hw_set_encrypt_key_alt_3: + movups xmm0,XMMWORD[rcx] + xorps xmm4,xmm4 + lea rax,[16+r8] + cmp edx,256 + je NEAR $L$14rounds_alt + cmp edx,192 + je NEAR $L$12rounds_alt + cmp edx,128 + jne NEAR $L$bad_keybits_alt + + mov edx,9 + movdqa xmm5,XMMWORD[$L$key_rotate] + mov r10d,8 + movdqa xmm4,XMMWORD[$L$key_rcon1] + movdqa xmm2,xmm0 + movdqu XMMWORD[r8],xmm0 + jmp NEAR $L$oop_key128 + +ALIGN 16 +$L$oop_key128: +DB 102,15,56,0,197 + DB 102,15,56,221,196 + pslld xmm4,1 + lea rax,[16+rax] + + movdqa xmm3,xmm2 + pslldq xmm2,4 + pxor xmm3,xmm2 + pslldq xmm2,4 + pxor xmm3,xmm2 + pslldq xmm2,4 + pxor xmm2,xmm3 + + pxor xmm0,xmm2 + movdqu XMMWORD[(-16)+rax],xmm0 + movdqa xmm2,xmm0 + + dec r10d + jnz NEAR $L$oop_key128 + + movdqa xmm4,XMMWORD[$L$key_rcon1b] + +DB 102,15,56,0,197 + DB 102,15,56,221,196 + pslld xmm4,1 + + movdqa xmm3,xmm2 + pslldq xmm2,4 + pxor xmm3,xmm2 + pslldq xmm2,4 + pxor xmm3,xmm2 + pslldq xmm2,4 + pxor xmm2,xmm3 + + pxor xmm0,xmm2 + movdqu XMMWORD[rax],xmm0 + + movdqa xmm2,xmm0 +DB 102,15,56,0,197 + DB 102,15,56,221,196 + + movdqa xmm3,xmm2 + pslldq xmm2,4 + pxor xmm3,xmm2 + pslldq xmm2,4 + pxor xmm3,xmm2 + pslldq xmm2,4 + pxor xmm2,xmm3 + + pxor xmm0,xmm2 + movdqu XMMWORD[16+rax],xmm0 + + mov DWORD[96+rax],edx + xor eax,eax + jmp NEAR $L$enc_key_ret_alt + +ALIGN 16 +$L$12rounds_alt: + movq xmm2,QWORD[16+rcx] + mov edx,11 + movdqa xmm5,XMMWORD[$L$key_rotate192] + movdqa xmm4,XMMWORD[$L$key_rcon1] + mov r10d,8 + movdqu XMMWORD[r8],xmm0 + jmp NEAR $L$oop_key192 + +ALIGN 16 +$L$oop_key192: + movq QWORD[rax],xmm2 + movdqa xmm1,xmm2 +DB 102,15,56,0,213 + DB 102,15,56,221,212 + pslld xmm4,1 + lea rax,[24+rax] + + movdqa xmm3,xmm0 + pslldq xmm0,4 + pxor xmm3,xmm0 + pslldq xmm0,4 + pxor xmm3,xmm0 + pslldq xmm0,4 + pxor xmm0,xmm3 + + pshufd xmm3,xmm0,0xff + pxor xmm3,xmm1 + pslldq xmm1,4 + pxor xmm3,xmm1 + + pxor xmm0,xmm2 + pxor xmm2,xmm3 + movdqu XMMWORD[(-16)+rax],xmm0 + + dec r10d + jnz NEAR $L$oop_key192 + + mov DWORD[32+rax],edx + xor eax,eax + jmp NEAR $L$enc_key_ret_alt + +ALIGN 16 +$L$14rounds_alt: + movups xmm2,XMMWORD[16+rcx] + mov edx,13 + lea rax,[16+rax] + movdqa xmm5,XMMWORD[$L$key_rotate] + movdqa xmm4,XMMWORD[$L$key_rcon1] + mov r10d,7 + movdqu XMMWORD[r8],xmm0 + movdqa xmm1,xmm2 + movdqu XMMWORD[16+r8],xmm2 + jmp NEAR $L$oop_key256 + +ALIGN 16 +$L$oop_key256: +DB 102,15,56,0,213 + DB 102,15,56,221,212 + + movdqa xmm3,xmm0 + pslldq xmm0,4 + pxor xmm3,xmm0 + pslldq xmm0,4 + pxor xmm3,xmm0 + pslldq xmm0,4 + pxor xmm0,xmm3 + pslld xmm4,1 + + pxor xmm0,xmm2 + movdqu XMMWORD[rax],xmm0 + + dec r10d + jz NEAR $L$done_key256 + + pshufd xmm2,xmm0,0xff + pxor xmm3,xmm3 + DB 102,15,56,221,211 + + movdqa xmm3,xmm1 + pslldq xmm1,4 + pxor xmm3,xmm1 + pslldq xmm1,4 + pxor xmm3,xmm1 + pslldq xmm1,4 + pxor xmm1,xmm3 + + pxor xmm2,xmm1 + movdqu XMMWORD[16+rax],xmm2 + lea rax,[32+rax] + movdqa xmm1,xmm2 + + jmp NEAR $L$oop_key256 + +$L$done_key256: + mov DWORD[16+rax],edx + xor eax,eax + jmp NEAR $L$enc_key_ret_alt + +ALIGN 16 +$L$bad_keybits_alt: + mov rax,-2 +$L$enc_key_ret_alt: + pxor xmm0,xmm0 + pxor xmm1,xmm1 + pxor xmm2,xmm2 + pxor xmm3,xmm3 + pxor xmm4,xmm4 + pxor xmm5,xmm5 + add rsp,8 + + ret + +$L$SEH_end_aes_hw_set_encrypt_key_alt_4: + section .rdata rdata align=8 ALIGN 64 $L$bswap_mask: @@ -2650,19 +2685,32 @@ $L$SEH_info_cbc: DD cbc_se_handler wrt ..imagebase section .pdata ALIGN 4 - DD $L$SEH_begin_aes_hw_set_encrypt_key_1 wrt ..imagebase - DD $L$SEH_end_aes_hw_set_encrypt_key_4 wrt ..imagebase - DD $L$SEH_info_aes_hw_set_encrypt_key_0 wrt ..imagebase + DD $L$SEH_begin_aes_hw_set_encrypt_key_base_1 wrt ..imagebase + DD $L$SEH_end_aes_hw_set_encrypt_key_base_4 wrt ..imagebase + DD $L$SEH_info_aes_hw_set_encrypt_key_base_0 wrt ..imagebase + + DD $L$SEH_begin_aes_hw_set_encrypt_key_alt_1 wrt ..imagebase + DD $L$SEH_end_aes_hw_set_encrypt_key_alt_4 wrt ..imagebase + DD $L$SEH_info_aes_hw_set_encrypt_key_alt_0 wrt ..imagebase section .xdata ALIGN 4 -$L$SEH_info_aes_hw_set_encrypt_key_0: +$L$SEH_info_aes_hw_set_encrypt_key_base_0: DB 1 - DB $L$SEH_endprologue_aes_hw_set_encrypt_key_3-$L$SEH_begin_aes_hw_set_encrypt_key_1 + DB $L$SEH_endprologue_aes_hw_set_encrypt_key_base_3-$L$SEH_begin_aes_hw_set_encrypt_key_base_1 DB 1 DB 0 - DB $L$SEH_prologue_aes_hw_set_encrypt_key_2-$L$SEH_begin_aes_hw_set_encrypt_key_1 + DB $L$SEH_prologue_aes_hw_set_encrypt_key_base_2-$L$SEH_begin_aes_hw_set_encrypt_key_base_1 + DB 2 + + DW 0 +$L$SEH_info_aes_hw_set_encrypt_key_alt_0: + DB 1 + DB $L$SEH_endprologue_aes_hw_set_encrypt_key_alt_3-$L$SEH_begin_aes_hw_set_encrypt_key_alt_1 + DB 1 + DB 0 + DB $L$SEH_prologue_aes_hw_set_encrypt_key_alt_2-$L$SEH_begin_aes_hw_set_encrypt_key_alt_1 DB 2 DW 0 diff --git a/gen/bcm/vpaes-x86-apple.S b/gen/bcm/vpaes-x86-apple.S index 4d2c4856d..02d378704 100644 --- a/gen/bcm/vpaes-x86-apple.S +++ b/gen/bcm/vpaes-x86-apple.S @@ -470,10 +470,10 @@ L_vpaes_set_encrypt_key_begin: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call L016pic -L016pic: + call L016pic_for_function_hit +L016pic_for_function_hit: popl %ebx - leal _BORINGSSL_function_hit+5-L016pic(%ebx),%ebx + leal _BORINGSSL_function_hit+5-L016pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx @@ -551,10 +551,10 @@ L_vpaes_encrypt_begin: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call L019pic -L019pic: + call L019pic_for_function_hit +L019pic_for_function_hit: popl %ebx - leal _BORINGSSL_function_hit+4-L019pic(%ebx),%ebx + leal _BORINGSSL_function_hit+4-L019pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx diff --git a/gen/bcm/vpaes-x86-linux.S b/gen/bcm/vpaes-x86-linux.S index 02786a761..31dc9a081 100644 --- a/gen/bcm/vpaes-x86-linux.S +++ b/gen/bcm/vpaes-x86-linux.S @@ -487,10 +487,10 @@ vpaes_set_encrypt_key: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call .L016pic -.L016pic: + call .L016pic_for_function_hit +.L016pic_for_function_hit: popl %ebx - leal BORINGSSL_function_hit+5-.L016pic(%ebx),%ebx + leal BORINGSSL_function_hit+5-.L016pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx @@ -572,10 +572,10 @@ vpaes_encrypt: #ifdef BORINGSSL_DISPATCH_TEST pushl %ebx pushl %edx - call .L019pic -.L019pic: + call .L019pic_for_function_hit +.L019pic_for_function_hit: popl %ebx - leal BORINGSSL_function_hit+4-.L019pic(%ebx),%ebx + leal BORINGSSL_function_hit+4-.L019pic_for_function_hit(%ebx),%ebx movl $1,%edx movb %dl,(%ebx) popl %edx diff --git a/gen/bcm/vpaes-x86-win.asm b/gen/bcm/vpaes-x86-win.asm index 661496e36..3f087e101 100644 --- a/gen/bcm/vpaes-x86-win.asm +++ b/gen/bcm/vpaes-x86-win.asm @@ -470,10 +470,10 @@ L$_vpaes_set_encrypt_key_begin: %ifdef BORINGSSL_DISPATCH_TEST push ebx push edx - call L$016pic -L$016pic: + call L$016pic_for_function_hit +L$016pic_for_function_hit: pop ebx - lea ebx,[(_BORINGSSL_function_hit+5-L$016pic)+ebx] + lea ebx,[(_BORINGSSL_function_hit+5-L$016pic_for_function_hit)+ebx] mov edx,1 mov BYTE [ebx],dl pop edx @@ -549,10 +549,10 @@ L$_vpaes_encrypt_begin: %ifdef BORINGSSL_DISPATCH_TEST push ebx push edx - call L$019pic -L$019pic: + call L$019pic_for_function_hit +L$019pic_for_function_hit: pop ebx - lea ebx,[(_BORINGSSL_function_hit+4-L$019pic)+ebx] + lea ebx,[(_BORINGSSL_function_hit+4-L$019pic_for_function_hit)+ebx] mov edx,1 mov BYTE [ebx],dl pop edx