update main-with-bazel from master branch
This commit is contained in:
BoringSSL Robot
@@ -969,10 +969,7 @@ L$ctr32_bulk:
|
||||
leaq 7(%r8),%r9
|
||||
movl %r10d,96+12(%rsp)
|
||||
bswapl %r9d
|
||||
leaq _OPENSSL_ia32cap_P(%rip),%r10
|
||||
movl 4(%r10),%r10d
|
||||
xorl %ebp,%r9d
|
||||
andl $71303168,%r10d
|
||||
movl %r9d,112+12(%rsp)
|
||||
|
||||
movups 16(%rcx),%xmm1
|
||||
@@ -983,104 +980,10 @@ L$ctr32_bulk:
|
||||
cmpq $8,%rdx
|
||||
jb L$ctr32_tail
|
||||
|
||||
subq $6,%rdx
|
||||
cmpl $4194304,%r10d
|
||||
je L$ctr32_6x
|
||||
|
||||
leaq 128(%rcx),%rcx
|
||||
subq $2,%rdx
|
||||
subq $8,%rdx
|
||||
jmp L$ctr32_loop8
|
||||
|
||||
.p2align 4
|
||||
L$ctr32_6x:
|
||||
shll $4,%eax
|
||||
movl $48,%r10d
|
||||
bswapl %ebp
|
||||
leaq 32(%rcx,%rax,1),%rcx
|
||||
subq %rax,%r10
|
||||
jmp L$ctr32_loop6
|
||||
|
||||
.p2align 4
|
||||
L$ctr32_loop6:
|
||||
addl $6,%r8d
|
||||
movups -48(%rcx,%r10,1),%xmm0
|
||||
.byte 102,15,56,220,209
|
||||
movl %r8d,%eax
|
||||
xorl %ebp,%eax
|
||||
.byte 102,15,56,220,217
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,12
|
||||
leal 1(%r8),%eax
|
||||
.byte 102,15,56,220,225
|
||||
xorl %ebp,%eax
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,28
|
||||
.byte 102,15,56,220,233
|
||||
leal 2(%r8),%eax
|
||||
xorl %ebp,%eax
|
||||
.byte 102,15,56,220,241
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,44
|
||||
leal 3(%r8),%eax
|
||||
.byte 102,15,56,220,249
|
||||
movups -32(%rcx,%r10,1),%xmm1
|
||||
xorl %ebp,%eax
|
||||
|
||||
.byte 102,15,56,220,208
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,60
|
||||
leal 4(%r8),%eax
|
||||
.byte 102,15,56,220,216
|
||||
xorl %ebp,%eax
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,76
|
||||
.byte 102,15,56,220,224
|
||||
leal 5(%r8),%eax
|
||||
xorl %ebp,%eax
|
||||
.byte 102,15,56,220,232
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,92
|
||||
movq %r10,%rax
|
||||
.byte 102,15,56,220,240
|
||||
.byte 102,15,56,220,248
|
||||
movups -16(%rcx,%r10,1),%xmm0
|
||||
|
||||
call L$enc_loop6
|
||||
|
||||
movdqu (%rdi),%xmm8
|
||||
movdqu 16(%rdi),%xmm9
|
||||
movdqu 32(%rdi),%xmm10
|
||||
movdqu 48(%rdi),%xmm11
|
||||
movdqu 64(%rdi),%xmm12
|
||||
movdqu 80(%rdi),%xmm13
|
||||
leaq 96(%rdi),%rdi
|
||||
movups -64(%rcx,%r10,1),%xmm1
|
||||
pxor %xmm2,%xmm8
|
||||
movaps 0(%rsp),%xmm2
|
||||
pxor %xmm3,%xmm9
|
||||
movaps 16(%rsp),%xmm3
|
||||
pxor %xmm4,%xmm10
|
||||
movaps 32(%rsp),%xmm4
|
||||
pxor %xmm5,%xmm11
|
||||
movaps 48(%rsp),%xmm5
|
||||
pxor %xmm6,%xmm12
|
||||
movaps 64(%rsp),%xmm6
|
||||
pxor %xmm7,%xmm13
|
||||
movaps 80(%rsp),%xmm7
|
||||
movdqu %xmm8,(%rsi)
|
||||
movdqu %xmm9,16(%rsi)
|
||||
movdqu %xmm10,32(%rsi)
|
||||
movdqu %xmm11,48(%rsi)
|
||||
movdqu %xmm12,64(%rsi)
|
||||
movdqu %xmm13,80(%rsi)
|
||||
leaq 96(%rsi),%rsi
|
||||
|
||||
subq $6,%rdx
|
||||
jnc L$ctr32_loop6
|
||||
|
||||
addq $6,%rdx
|
||||
jz L$ctr32_done
|
||||
|
||||
leal -48(%r10),%eax
|
||||
leaq -80(%rcx,%r10,1),%rcx
|
||||
negl %eax
|
||||
shrl $4,%eax
|
||||
jmp L$ctr32_tail
|
||||
|
||||
.p2align 5
|
||||
L$ctr32_loop8:
|
||||
addl $8,%r8d
|
||||
@@ -1582,16 +1485,10 @@ L$cbc_decrypt_bulk:
|
||||
movdqa %xmm5,%xmm14
|
||||
movdqu 80(%rdi),%xmm7
|
||||
movdqa %xmm6,%xmm15
|
||||
leaq _OPENSSL_ia32cap_P(%rip),%r9
|
||||
movl 4(%r9),%r9d
|
||||
cmpq $0x70,%rdx
|
||||
jbe L$cbc_dec_six_or_seven
|
||||
|
||||
andl $71303168,%r9d
|
||||
subq $0x50,%rdx
|
||||
cmpl $4194304,%r9d
|
||||
je L$cbc_dec_loop6_enter
|
||||
subq $0x20,%rdx
|
||||
subq $0x70,%rdx
|
||||
leaq 112(%rcx),%rcx
|
||||
jmp L$cbc_dec_loop8_enter
|
||||
.p2align 4
|
||||
@@ -1862,51 +1759,6 @@ L$cbc_dec_seven:
|
||||
pxor %xmm9,%xmm9
|
||||
jmp L$cbc_dec_tail_collected
|
||||
|
||||
.p2align 4
|
||||
L$cbc_dec_loop6:
|
||||
movups %xmm7,(%rsi)
|
||||
leaq 16(%rsi),%rsi
|
||||
movdqu 0(%rdi),%xmm2
|
||||
movdqu 16(%rdi),%xmm3
|
||||
movdqa %xmm2,%xmm11
|
||||
movdqu 32(%rdi),%xmm4
|
||||
movdqa %xmm3,%xmm12
|
||||
movdqu 48(%rdi),%xmm5
|
||||
movdqa %xmm4,%xmm13
|
||||
movdqu 64(%rdi),%xmm6
|
||||
movdqa %xmm5,%xmm14
|
||||
movdqu 80(%rdi),%xmm7
|
||||
movdqa %xmm6,%xmm15
|
||||
L$cbc_dec_loop6_enter:
|
||||
leaq 96(%rdi),%rdi
|
||||
movdqa %xmm7,%xmm8
|
||||
|
||||
call _aesni_decrypt6
|
||||
|
||||
pxor %xmm10,%xmm2
|
||||
movdqa %xmm8,%xmm10
|
||||
pxor %xmm11,%xmm3
|
||||
movdqu %xmm2,(%rsi)
|
||||
pxor %xmm12,%xmm4
|
||||
movdqu %xmm3,16(%rsi)
|
||||
pxor %xmm13,%xmm5
|
||||
movdqu %xmm4,32(%rsi)
|
||||
pxor %xmm14,%xmm6
|
||||
movq %rbp,%rcx
|
||||
movdqu %xmm5,48(%rsi)
|
||||
pxor %xmm15,%xmm7
|
||||
movl %r10d,%eax
|
||||
movdqu %xmm6,64(%rsi)
|
||||
leaq 80(%rsi),%rsi
|
||||
subq $0x60,%rdx
|
||||
ja L$cbc_dec_loop6
|
||||
|
||||
movdqa %xmm7,%xmm2
|
||||
addq $0x50,%rdx
|
||||
jle L$cbc_dec_clear_tail_collected
|
||||
movups %xmm7,(%rsi)
|
||||
leaq 16(%rsi),%rsi
|
||||
|
||||
L$cbc_dec_tail:
|
||||
movups (%rdi),%xmm2
|
||||
subq $0x10,%rdx
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
|
||||
#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__)
|
||||
.text
|
||||
|
||||
.globl _gcm_init_clmul
|
||||
.private_extern _gcm_init_clmul
|
||||
|
||||
@@ -243,15 +242,9 @@ L$_ghash_clmul:
|
||||
jz L$odd_tail
|
||||
|
||||
movdqu 16(%rsi),%xmm6
|
||||
leaq _OPENSSL_ia32cap_P(%rip),%rax
|
||||
movl 4(%rax),%eax
|
||||
cmpq $0x30,%rcx
|
||||
jb L$skip4x
|
||||
|
||||
andl $71303168,%eax
|
||||
cmpl $4194304,%eax
|
||||
je L$skip4x
|
||||
|
||||
subq $0x30,%rcx
|
||||
movq $0xA040608020C0E000,%rax
|
||||
movdqu 48(%rsi),%xmm14
|
||||
|
||||
@@ -971,10 +971,7 @@ _CET_ENDBR
|
||||
leaq 7(%r8),%r9
|
||||
movl %r10d,96+12(%rsp)
|
||||
bswapl %r9d
|
||||
leaq OPENSSL_ia32cap_P(%rip),%r10
|
||||
movl 4(%r10),%r10d
|
||||
xorl %ebp,%r9d
|
||||
andl $71303168,%r10d
|
||||
movl %r9d,112+12(%rsp)
|
||||
|
||||
movups 16(%rcx),%xmm1
|
||||
@@ -985,104 +982,10 @@ _CET_ENDBR
|
||||
cmpq $8,%rdx
|
||||
jb .Lctr32_tail
|
||||
|
||||
subq $6,%rdx
|
||||
cmpl $4194304,%r10d
|
||||
je .Lctr32_6x
|
||||
|
||||
leaq 128(%rcx),%rcx
|
||||
subq $2,%rdx
|
||||
subq $8,%rdx
|
||||
jmp .Lctr32_loop8
|
||||
|
||||
.align 16
|
||||
.Lctr32_6x:
|
||||
shll $4,%eax
|
||||
movl $48,%r10d
|
||||
bswapl %ebp
|
||||
leaq 32(%rcx,%rax,1),%rcx
|
||||
subq %rax,%r10
|
||||
jmp .Lctr32_loop6
|
||||
|
||||
.align 16
|
||||
.Lctr32_loop6:
|
||||
addl $6,%r8d
|
||||
movups -48(%rcx,%r10,1),%xmm0
|
||||
.byte 102,15,56,220,209
|
||||
movl %r8d,%eax
|
||||
xorl %ebp,%eax
|
||||
.byte 102,15,56,220,217
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,12
|
||||
leal 1(%r8),%eax
|
||||
.byte 102,15,56,220,225
|
||||
xorl %ebp,%eax
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,28
|
||||
.byte 102,15,56,220,233
|
||||
leal 2(%r8),%eax
|
||||
xorl %ebp,%eax
|
||||
.byte 102,15,56,220,241
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,44
|
||||
leal 3(%r8),%eax
|
||||
.byte 102,15,56,220,249
|
||||
movups -32(%rcx,%r10,1),%xmm1
|
||||
xorl %ebp,%eax
|
||||
|
||||
.byte 102,15,56,220,208
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,60
|
||||
leal 4(%r8),%eax
|
||||
.byte 102,15,56,220,216
|
||||
xorl %ebp,%eax
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,76
|
||||
.byte 102,15,56,220,224
|
||||
leal 5(%r8),%eax
|
||||
xorl %ebp,%eax
|
||||
.byte 102,15,56,220,232
|
||||
.byte 0x0f,0x38,0xf1,0x44,0x24,92
|
||||
movq %r10,%rax
|
||||
.byte 102,15,56,220,240
|
||||
.byte 102,15,56,220,248
|
||||
movups -16(%rcx,%r10,1),%xmm0
|
||||
|
||||
call .Lenc_loop6
|
||||
|
||||
movdqu (%rdi),%xmm8
|
||||
movdqu 16(%rdi),%xmm9
|
||||
movdqu 32(%rdi),%xmm10
|
||||
movdqu 48(%rdi),%xmm11
|
||||
movdqu 64(%rdi),%xmm12
|
||||
movdqu 80(%rdi),%xmm13
|
||||
leaq 96(%rdi),%rdi
|
||||
movups -64(%rcx,%r10,1),%xmm1
|
||||
pxor %xmm2,%xmm8
|
||||
movaps 0(%rsp),%xmm2
|
||||
pxor %xmm3,%xmm9
|
||||
movaps 16(%rsp),%xmm3
|
||||
pxor %xmm4,%xmm10
|
||||
movaps 32(%rsp),%xmm4
|
||||
pxor %xmm5,%xmm11
|
||||
movaps 48(%rsp),%xmm5
|
||||
pxor %xmm6,%xmm12
|
||||
movaps 64(%rsp),%xmm6
|
||||
pxor %xmm7,%xmm13
|
||||
movaps 80(%rsp),%xmm7
|
||||
movdqu %xmm8,(%rsi)
|
||||
movdqu %xmm9,16(%rsi)
|
||||
movdqu %xmm10,32(%rsi)
|
||||
movdqu %xmm11,48(%rsi)
|
||||
movdqu %xmm12,64(%rsi)
|
||||
movdqu %xmm13,80(%rsi)
|
||||
leaq 96(%rsi),%rsi
|
||||
|
||||
subq $6,%rdx
|
||||
jnc .Lctr32_loop6
|
||||
|
||||
addq $6,%rdx
|
||||
jz .Lctr32_done
|
||||
|
||||
leal -48(%r10),%eax
|
||||
leaq -80(%rcx,%r10,1),%rcx
|
||||
negl %eax
|
||||
shrl $4,%eax
|
||||
jmp .Lctr32_tail
|
||||
|
||||
.align 32
|
||||
.Lctr32_loop8:
|
||||
addl $8,%r8d
|
||||
@@ -1584,16 +1487,10 @@ _CET_ENDBR
|
||||
movdqa %xmm5,%xmm14
|
||||
movdqu 80(%rdi),%xmm7
|
||||
movdqa %xmm6,%xmm15
|
||||
leaq OPENSSL_ia32cap_P(%rip),%r9
|
||||
movl 4(%r9),%r9d
|
||||
cmpq $0x70,%rdx
|
||||
jbe .Lcbc_dec_six_or_seven
|
||||
|
||||
andl $71303168,%r9d
|
||||
subq $0x50,%rdx
|
||||
cmpl $4194304,%r9d
|
||||
je .Lcbc_dec_loop6_enter
|
||||
subq $0x20,%rdx
|
||||
subq $0x70,%rdx
|
||||
leaq 112(%rcx),%rcx
|
||||
jmp .Lcbc_dec_loop8_enter
|
||||
.align 16
|
||||
@@ -1864,51 +1761,6 @@ _CET_ENDBR
|
||||
pxor %xmm9,%xmm9
|
||||
jmp .Lcbc_dec_tail_collected
|
||||
|
||||
.align 16
|
||||
.Lcbc_dec_loop6:
|
||||
movups %xmm7,(%rsi)
|
||||
leaq 16(%rsi),%rsi
|
||||
movdqu 0(%rdi),%xmm2
|
||||
movdqu 16(%rdi),%xmm3
|
||||
movdqa %xmm2,%xmm11
|
||||
movdqu 32(%rdi),%xmm4
|
||||
movdqa %xmm3,%xmm12
|
||||
movdqu 48(%rdi),%xmm5
|
||||
movdqa %xmm4,%xmm13
|
||||
movdqu 64(%rdi),%xmm6
|
||||
movdqa %xmm5,%xmm14
|
||||
movdqu 80(%rdi),%xmm7
|
||||
movdqa %xmm6,%xmm15
|
||||
.Lcbc_dec_loop6_enter:
|
||||
leaq 96(%rdi),%rdi
|
||||
movdqa %xmm7,%xmm8
|
||||
|
||||
call _aesni_decrypt6
|
||||
|
||||
pxor %xmm10,%xmm2
|
||||
movdqa %xmm8,%xmm10
|
||||
pxor %xmm11,%xmm3
|
||||
movdqu %xmm2,(%rsi)
|
||||
pxor %xmm12,%xmm4
|
||||
movdqu %xmm3,16(%rsi)
|
||||
pxor %xmm13,%xmm5
|
||||
movdqu %xmm4,32(%rsi)
|
||||
pxor %xmm14,%xmm6
|
||||
movq %rbp,%rcx
|
||||
movdqu %xmm5,48(%rsi)
|
||||
pxor %xmm15,%xmm7
|
||||
movl %r10d,%eax
|
||||
movdqu %xmm6,64(%rsi)
|
||||
leaq 80(%rsi),%rsi
|
||||
subq $0x60,%rdx
|
||||
ja .Lcbc_dec_loop6
|
||||
|
||||
movdqa %xmm7,%xmm2
|
||||
addq $0x50,%rdx
|
||||
jle .Lcbc_dec_clear_tail_collected
|
||||
movups %xmm7,(%rsi)
|
||||
leaq 16(%rsi),%rsi
|
||||
|
||||
.Lcbc_dec_tail:
|
||||
movups (%rdi),%xmm2
|
||||
subq $0x10,%rdx
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
|
||||
#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__)
|
||||
.text
|
||||
.extern OPENSSL_ia32cap_P
|
||||
.hidden OPENSSL_ia32cap_P
|
||||
.globl gcm_init_clmul
|
||||
.hidden gcm_init_clmul
|
||||
.type gcm_init_clmul,@function
|
||||
@@ -244,15 +242,9 @@ _CET_ENDBR
|
||||
jz .Lodd_tail
|
||||
|
||||
movdqu 16(%rsi),%xmm6
|
||||
leaq OPENSSL_ia32cap_P(%rip),%rax
|
||||
movl 4(%rax),%eax
|
||||
cmpq $0x30,%rcx
|
||||
jb .Lskip4x
|
||||
|
||||
andl $71303168,%eax
|
||||
cmpl $4194304,%eax
|
||||
je .Lskip4x
|
||||
|
||||
subq $0x30,%rcx
|
||||
movq $0xA040608020C0E000,%rax
|
||||
movdqu 48(%rsi),%xmm14
|
||||
|
||||
@@ -1303,10 +1303,7 @@ $code.=<<___;
|
||||
lea 7($ctr),%r9
|
||||
mov %r10d,0x60+12(%rsp)
|
||||
bswap %r9d
|
||||
leaq OPENSSL_ia32cap_P(%rip),%r10
|
||||
mov 4(%r10),%r10d
|
||||
xor $key0,%r9d
|
||||
and \$`1<<26|1<<22`,%r10d # isolate XSAVE+MOVBE
|
||||
mov %r9d,0x70+12(%rsp)
|
||||
|
||||
$movkey 0x10($key),$rndkey1
|
||||
@@ -1317,104 +1314,10 @@ $code.=<<___;
|
||||
cmp \$8,$len # $len is in blocks
|
||||
jb .Lctr32_tail # short input if ($len<8)
|
||||
|
||||
sub \$6,$len # $len is biased by -6
|
||||
cmp \$`1<<22`,%r10d # check for MOVBE without XSAVE
|
||||
je .Lctr32_6x # [which denotes Atom Silvermont]
|
||||
|
||||
lea 0x80($key),$key # size optimization
|
||||
sub \$2,$len # $len is biased by -8
|
||||
sub \$8,$len # $len is biased by -8
|
||||
jmp .Lctr32_loop8
|
||||
|
||||
.align 16
|
||||
.Lctr32_6x:
|
||||
shl \$4,$rounds
|
||||
mov \$48,$rnds_
|
||||
bswap $key0
|
||||
lea 32($key,$rounds),$key # end of key schedule
|
||||
sub %rax,%r10 # twisted $rounds
|
||||
jmp .Lctr32_loop6
|
||||
|
||||
.align 16
|
||||
.Lctr32_loop6:
|
||||
add \$6,$ctr # next counter value
|
||||
$movkey -48($key,$rnds_),$rndkey0
|
||||
aesenc $rndkey1,$inout0
|
||||
mov $ctr,%eax
|
||||
xor $key0,%eax
|
||||
aesenc $rndkey1,$inout1
|
||||
movbe %eax,`0x00+12`(%rsp) # store next counter value
|
||||
lea 1($ctr),%eax
|
||||
aesenc $rndkey1,$inout2
|
||||
xor $key0,%eax
|
||||
movbe %eax,`0x10+12`(%rsp)
|
||||
aesenc $rndkey1,$inout3
|
||||
lea 2($ctr),%eax
|
||||
xor $key0,%eax
|
||||
aesenc $rndkey1,$inout4
|
||||
movbe %eax,`0x20+12`(%rsp)
|
||||
lea 3($ctr),%eax
|
||||
aesenc $rndkey1,$inout5
|
||||
$movkey -32($key,$rnds_),$rndkey1
|
||||
xor $key0,%eax
|
||||
|
||||
aesenc $rndkey0,$inout0
|
||||
movbe %eax,`0x30+12`(%rsp)
|
||||
lea 4($ctr),%eax
|
||||
aesenc $rndkey0,$inout1
|
||||
xor $key0,%eax
|
||||
movbe %eax,`0x40+12`(%rsp)
|
||||
aesenc $rndkey0,$inout2
|
||||
lea 5($ctr),%eax
|
||||
xor $key0,%eax
|
||||
aesenc $rndkey0,$inout3
|
||||
movbe %eax,`0x50+12`(%rsp)
|
||||
mov %r10,%rax # mov $rnds_,$rounds
|
||||
aesenc $rndkey0,$inout4
|
||||
aesenc $rndkey0,$inout5
|
||||
$movkey -16($key,$rnds_),$rndkey0
|
||||
|
||||
call .Lenc_loop6
|
||||
|
||||
movdqu ($inp),$inout6 # load 6 input blocks
|
||||
movdqu 0x10($inp),$inout7
|
||||
movdqu 0x20($inp),$in0
|
||||
movdqu 0x30($inp),$in1
|
||||
movdqu 0x40($inp),$in2
|
||||
movdqu 0x50($inp),$in3
|
||||
lea 0x60($inp),$inp # $inp+=6*16
|
||||
$movkey -64($key,$rnds_),$rndkey1
|
||||
pxor $inout0,$inout6 # inp^=E(ctr)
|
||||
movaps 0x00(%rsp),$inout0 # load next counter [xor-ed with 0 round]
|
||||
pxor $inout1,$inout7
|
||||
movaps 0x10(%rsp),$inout1
|
||||
pxor $inout2,$in0
|
||||
movaps 0x20(%rsp),$inout2
|
||||
pxor $inout3,$in1
|
||||
movaps 0x30(%rsp),$inout3
|
||||
pxor $inout4,$in2
|
||||
movaps 0x40(%rsp),$inout4
|
||||
pxor $inout5,$in3
|
||||
movaps 0x50(%rsp),$inout5
|
||||
movdqu $inout6,($out) # store 6 output blocks
|
||||
movdqu $inout7,0x10($out)
|
||||
movdqu $in0,0x20($out)
|
||||
movdqu $in1,0x30($out)
|
||||
movdqu $in2,0x40($out)
|
||||
movdqu $in3,0x50($out)
|
||||
lea 0x60($out),$out # $out+=6*16
|
||||
|
||||
sub \$6,$len
|
||||
jnc .Lctr32_loop6 # loop if $len-=6 didn't borrow
|
||||
|
||||
add \$6,$len # restore real remaining $len
|
||||
jz .Lctr32_done # done if ($len==0)
|
||||
|
||||
lea -48($rnds_),$rounds
|
||||
lea -80($key,$rnds_),$key # restore $key
|
||||
neg $rounds
|
||||
shr \$4,$rounds # restore $rounds
|
||||
jmp .Lctr32_tail
|
||||
|
||||
.align 32
|
||||
.Lctr32_loop8:
|
||||
add \$8,$ctr # next counter value
|
||||
@@ -2906,16 +2809,10 @@ $code.=<<___;
|
||||
movdqa $inout3,$in3
|
||||
movdqu 0x50($inp),$inout5
|
||||
movdqa $inout4,$in4
|
||||
leaq OPENSSL_ia32cap_P(%rip),%r9
|
||||
mov 4(%r9),%r9d
|
||||
cmp \$0x70,$len
|
||||
jbe .Lcbc_dec_six_or_seven
|
||||
|
||||
and \$`1<<26|1<<22`,%r9d # isolate XSAVE+MOVBE
|
||||
sub \$0x50,$len # $len is biased by -5*16
|
||||
cmp \$`1<<22`,%r9d # check for MOVBE without XSAVE
|
||||
je .Lcbc_dec_loop6_enter # [which denotes Atom Silvermont]
|
||||
sub \$0x20,$len # $len is biased by -7*16
|
||||
sub \$0x70,$len # $len is biased by -7*16
|
||||
lea 0x70($key),$key # size optimization
|
||||
jmp .Lcbc_dec_loop8_enter
|
||||
.align 16
|
||||
@@ -3107,51 +3004,6 @@ $code.=<<___;
|
||||
pxor $inout7,$inout7
|
||||
jmp .Lcbc_dec_tail_collected
|
||||
|
||||
.align 16
|
||||
.Lcbc_dec_loop6:
|
||||
movups $inout5,($out)
|
||||
lea 0x10($out),$out
|
||||
movdqu 0x00($inp),$inout0 # load input
|
||||
movdqu 0x10($inp),$inout1
|
||||
movdqa $inout0,$in0
|
||||
movdqu 0x20($inp),$inout2
|
||||
movdqa $inout1,$in1
|
||||
movdqu 0x30($inp),$inout3
|
||||
movdqa $inout2,$in2
|
||||
movdqu 0x40($inp),$inout4
|
||||
movdqa $inout3,$in3
|
||||
movdqu 0x50($inp),$inout5
|
||||
movdqa $inout4,$in4
|
||||
.Lcbc_dec_loop6_enter:
|
||||
lea 0x60($inp),$inp
|
||||
movdqa $inout5,$inout6
|
||||
|
||||
call _aesni_decrypt6
|
||||
|
||||
pxor $iv,$inout0 # ^= IV
|
||||
movdqa $inout6,$iv
|
||||
pxor $in0,$inout1
|
||||
movdqu $inout0,($out)
|
||||
pxor $in1,$inout2
|
||||
movdqu $inout1,0x10($out)
|
||||
pxor $in2,$inout3
|
||||
movdqu $inout2,0x20($out)
|
||||
pxor $in3,$inout4
|
||||
mov $key_,$key
|
||||
movdqu $inout3,0x30($out)
|
||||
pxor $in4,$inout5
|
||||
mov $rnds_,$rounds
|
||||
movdqu $inout4,0x40($out)
|
||||
lea 0x50($out),$out
|
||||
sub \$0x60,$len
|
||||
ja .Lcbc_dec_loop6
|
||||
|
||||
movdqa $inout5,$inout0
|
||||
add \$0x50,$len
|
||||
jle .Lcbc_dec_clear_tail_collected
|
||||
movups $inout5,($out)
|
||||
lea 0x10($out),$out
|
||||
|
||||
.Lcbc_dec_tail:
|
||||
movups ($inp),$inout0
|
||||
sub \$0x10,$len
|
||||
|
||||
@@ -120,7 +120,6 @@ $do4xaggr=1;
|
||||
|
||||
$code=<<___;
|
||||
.text
|
||||
.extern OPENSSL_ia32cap_P
|
||||
___
|
||||
|
||||
|
||||
@@ -387,15 +386,9 @@ if ($do4xaggr) {
|
||||
my ($Xl,$Xm,$Xh,$Hkey3,$Hkey4)=map("%xmm$_",(11..15));
|
||||
|
||||
$code.=<<___;
|
||||
leaq OPENSSL_ia32cap_P(%rip),%rax
|
||||
mov 4(%rax),%eax
|
||||
cmp \$0x30,$len
|
||||
jb .Lskip4x
|
||||
|
||||
and \$`1<<26|1<<22`,%eax # isolate MOVBE+XSAVE
|
||||
cmp \$`1<<22`,%eax # check for MOVBE without XSAVE
|
||||
je .Lskip4x
|
||||
|
||||
sub \$0x30,$len
|
||||
mov \$0xA040608020C0E000,%rax # ((7..0)·0xE0)&0xff
|
||||
movdqu 0x30($Htbl),$Hkey3
|
||||
|
||||
@@ -1024,10 +1024,7 @@ DB 102,15,58,34,232,3
|
||||
lea r9,[7+r8]
|
||||
mov DWORD[((96+12))+rsp],r10d
|
||||
bswap r9d
|
||||
lea r10,[OPENSSL_ia32cap_P]
|
||||
mov r10d,DWORD[4+r10]
|
||||
xor r9d,ebp
|
||||
and r10d,71303168
|
||||
mov DWORD[((112+12))+rsp],r9d
|
||||
|
||||
movups xmm1,XMMWORD[16+rcx]
|
||||
@@ -1038,104 +1035,10 @@ DB 102,15,58,34,232,3
|
||||
cmp rdx,8
|
||||
jb NEAR $L$ctr32_tail
|
||||
|
||||
sub rdx,6
|
||||
cmp r10d,4194304
|
||||
je NEAR $L$ctr32_6x
|
||||
|
||||
lea rcx,[128+rcx]
|
||||
sub rdx,2
|
||||
sub rdx,8
|
||||
jmp NEAR $L$ctr32_loop8
|
||||
|
||||
ALIGN 16
|
||||
$L$ctr32_6x:
|
||||
shl eax,4
|
||||
mov r10d,48
|
||||
bswap ebp
|
||||
lea rcx,[32+rax*1+rcx]
|
||||
sub r10,rax
|
||||
jmp NEAR $L$ctr32_loop6
|
||||
|
||||
ALIGN 16
|
||||
$L$ctr32_loop6:
|
||||
add r8d,6
|
||||
movups xmm0,XMMWORD[((-48))+r10*1+rcx]
|
||||
DB 102,15,56,220,209
|
||||
mov eax,r8d
|
||||
xor eax,ebp
|
||||
DB 102,15,56,220,217
|
||||
DB 0x0f,0x38,0xf1,0x44,0x24,12
|
||||
lea eax,[1+r8]
|
||||
DB 102,15,56,220,225
|
||||
xor eax,ebp
|
||||
DB 0x0f,0x38,0xf1,0x44,0x24,28
|
||||
DB 102,15,56,220,233
|
||||
lea eax,[2+r8]
|
||||
xor eax,ebp
|
||||
DB 102,15,56,220,241
|
||||
DB 0x0f,0x38,0xf1,0x44,0x24,44
|
||||
lea eax,[3+r8]
|
||||
DB 102,15,56,220,249
|
||||
movups xmm1,XMMWORD[((-32))+r10*1+rcx]
|
||||
xor eax,ebp
|
||||
|
||||
DB 102,15,56,220,208
|
||||
DB 0x0f,0x38,0xf1,0x44,0x24,60
|
||||
lea eax,[4+r8]
|
||||
DB 102,15,56,220,216
|
||||
xor eax,ebp
|
||||
DB 0x0f,0x38,0xf1,0x44,0x24,76
|
||||
DB 102,15,56,220,224
|
||||
lea eax,[5+r8]
|
||||
xor eax,ebp
|
||||
DB 102,15,56,220,232
|
||||
DB 0x0f,0x38,0xf1,0x44,0x24,92
|
||||
mov rax,r10
|
||||
DB 102,15,56,220,240
|
||||
DB 102,15,56,220,248
|
||||
movups xmm0,XMMWORD[((-16))+r10*1+rcx]
|
||||
|
||||
call $L$enc_loop6
|
||||
|
||||
movdqu xmm8,XMMWORD[rdi]
|
||||
movdqu xmm9,XMMWORD[16+rdi]
|
||||
movdqu xmm10,XMMWORD[32+rdi]
|
||||
movdqu xmm11,XMMWORD[48+rdi]
|
||||
movdqu xmm12,XMMWORD[64+rdi]
|
||||
movdqu xmm13,XMMWORD[80+rdi]
|
||||
lea rdi,[96+rdi]
|
||||
movups xmm1,XMMWORD[((-64))+r10*1+rcx]
|
||||
pxor xmm8,xmm2
|
||||
movaps xmm2,XMMWORD[rsp]
|
||||
pxor xmm9,xmm3
|
||||
movaps xmm3,XMMWORD[16+rsp]
|
||||
pxor xmm10,xmm4
|
||||
movaps xmm4,XMMWORD[32+rsp]
|
||||
pxor xmm11,xmm5
|
||||
movaps xmm5,XMMWORD[48+rsp]
|
||||
pxor xmm12,xmm6
|
||||
movaps xmm6,XMMWORD[64+rsp]
|
||||
pxor xmm13,xmm7
|
||||
movaps xmm7,XMMWORD[80+rsp]
|
||||
movdqu XMMWORD[rsi],xmm8
|
||||
movdqu XMMWORD[16+rsi],xmm9
|
||||
movdqu XMMWORD[32+rsi],xmm10
|
||||
movdqu XMMWORD[48+rsi],xmm11
|
||||
movdqu XMMWORD[64+rsi],xmm12
|
||||
movdqu XMMWORD[80+rsi],xmm13
|
||||
lea rsi,[96+rsi]
|
||||
|
||||
sub rdx,6
|
||||
jnc NEAR $L$ctr32_loop6
|
||||
|
||||
add rdx,6
|
||||
jz NEAR $L$ctr32_done
|
||||
|
||||
lea eax,[((-48))+r10]
|
||||
lea rcx,[((-80))+r10*1+rcx]
|
||||
neg eax
|
||||
shr eax,4
|
||||
jmp NEAR $L$ctr32_tail
|
||||
|
||||
ALIGN 32
|
||||
$L$ctr32_loop8:
|
||||
add r8d,8
|
||||
@@ -1671,16 +1574,10 @@ $L$cbc_decrypt_body:
|
||||
movdqa xmm14,xmm5
|
||||
movdqu xmm7,XMMWORD[80+rdi]
|
||||
movdqa xmm15,xmm6
|
||||
lea r9,[OPENSSL_ia32cap_P]
|
||||
mov r9d,DWORD[4+r9]
|
||||
cmp rdx,0x70
|
||||
jbe NEAR $L$cbc_dec_six_or_seven
|
||||
|
||||
and r9d,71303168
|
||||
sub rdx,0x50
|
||||
cmp r9d,4194304
|
||||
je NEAR $L$cbc_dec_loop6_enter
|
||||
sub rdx,0x20
|
||||
sub rdx,0x70
|
||||
lea rcx,[112+rcx]
|
||||
jmp NEAR $L$cbc_dec_loop8_enter
|
||||
ALIGN 16
|
||||
@@ -1951,51 +1848,6 @@ $L$cbc_dec_seven:
|
||||
pxor xmm9,xmm9
|
||||
jmp NEAR $L$cbc_dec_tail_collected
|
||||
|
||||
ALIGN 16
|
||||
$L$cbc_dec_loop6:
|
||||
movups XMMWORD[rsi],xmm7
|
||||
lea rsi,[16+rsi]
|
||||
movdqu xmm2,XMMWORD[rdi]
|
||||
movdqu xmm3,XMMWORD[16+rdi]
|
||||
movdqa xmm11,xmm2
|
||||
movdqu xmm4,XMMWORD[32+rdi]
|
||||
movdqa xmm12,xmm3
|
||||
movdqu xmm5,XMMWORD[48+rdi]
|
||||
movdqa xmm13,xmm4
|
||||
movdqu xmm6,XMMWORD[64+rdi]
|
||||
movdqa xmm14,xmm5
|
||||
movdqu xmm7,XMMWORD[80+rdi]
|
||||
movdqa xmm15,xmm6
|
||||
$L$cbc_dec_loop6_enter:
|
||||
lea rdi,[96+rdi]
|
||||
movdqa xmm8,xmm7
|
||||
|
||||
call _aesni_decrypt6
|
||||
|
||||
pxor xmm2,xmm10
|
||||
movdqa xmm10,xmm8
|
||||
pxor xmm3,xmm11
|
||||
movdqu XMMWORD[rsi],xmm2
|
||||
pxor xmm4,xmm12
|
||||
movdqu XMMWORD[16+rsi],xmm3
|
||||
pxor xmm5,xmm13
|
||||
movdqu XMMWORD[32+rsi],xmm4
|
||||
pxor xmm6,xmm14
|
||||
mov rcx,rbp
|
||||
movdqu XMMWORD[48+rsi],xmm5
|
||||
pxor xmm7,xmm15
|
||||
mov eax,r10d
|
||||
movdqu XMMWORD[64+rsi],xmm6
|
||||
lea rsi,[80+rsi]
|
||||
sub rdx,0x60
|
||||
ja NEAR $L$cbc_dec_loop6
|
||||
|
||||
movdqa xmm2,xmm7
|
||||
add rdx,0x50
|
||||
jle NEAR $L$cbc_dec_clear_tail_collected
|
||||
movups XMMWORD[rsi],xmm7
|
||||
lea rsi,[16+rsi]
|
||||
|
||||
$L$cbc_dec_tail:
|
||||
movups xmm2,XMMWORD[rdi]
|
||||
sub rdx,0x10
|
||||
|
||||
@@ -13,7 +13,6 @@ default rel
|
||||
%endif
|
||||
section .text code align=64
|
||||
|
||||
EXTERN OPENSSL_ia32cap_P
|
||||
global gcm_init_clmul
|
||||
|
||||
ALIGN 16
|
||||
@@ -277,15 +276,9 @@ DB 102,65,15,56,0,194
|
||||
jz NEAR $L$odd_tail
|
||||
|
||||
movdqu xmm6,XMMWORD[16+rdx]
|
||||
lea rax,[OPENSSL_ia32cap_P]
|
||||
mov eax,DWORD[4+rax]
|
||||
cmp r9,0x30
|
||||
jb NEAR $L$skip4x
|
||||
|
||||
and eax,71303168
|
||||
cmp eax,4194304
|
||||
je NEAR $L$skip4x
|
||||
|
||||
sub r9,0x30
|
||||
mov rax,0xA040608020C0E000
|
||||
movdqu xmm14,XMMWORD[48+rdx]
|
||||
|
||||
Reference in New Issue
Block a user