From 13840dd094f9e9c1b00a7368aa25e656554221f1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 14 Jan 2025 20:47:08 +0000 Subject: [PATCH] Add explicit prefetching to the new AES-GCM code Add explicit prefetching to the main loop of the new AES-GCM code, following the same rationale as change I6312e01ff0da70cc52f09194846b82cc6b69d37a. For now the same prefetch distance of 512 bytes is used. Change-Id: Ib57affb414e88675f3a4c8e124728a0cf412bc0a Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/75267 Reviewed-by: David Benjamin Commit-Queue: David Benjamin --- crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl | 7 +++++++ crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl | 6 ++++++ gen/bcm/aes-gcm-avx10-x86_64-apple.S | 8 ++++++++ gen/bcm/aes-gcm-avx10-x86_64-linux.S | 8 ++++++++ gen/bcm/aes-gcm-avx10-x86_64-win.asm | 8 ++++++++ gen/bcm/aes-gcm-avx2-x86_64-apple.S | 4 ++++ gen/bcm/aes-gcm-avx2-x86_64-linux.S | 4 ++++ gen/bcm/aes-gcm-avx2-x86_64-win.asm | 4 ++++ 8 files changed, 49 insertions(+) diff --git a/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl b/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl index 06ea7e617..eab607178 100644 --- a/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl +++ b/crypto/fipsmodule/modes/asm/aes-gcm-avx10-x86_64.pl @@ -1118,6 +1118,13 @@ ___ .Laes128$local_label_suffix: ___ + # Prefetch the source data 512 bytes ahead into the L1 data cache, to + # improve performance when the hardware prefetcher is disabled. Assumes the + # L1 data cache line size is 64 bytes (de facto standard on x86_64). + for ( my $i = 0 ; $i < 4 * $VL ; $i += 64 ) { + $code .= "prefetcht0 512+$i($SRC)\n"; + } + # Finish the AES encryption of the counter blocks in V0-V3, interleaved # with the GHASH update of the ciphertext blocks in GHASHDATA[0-3]. for my $i ( reverse 1 .. 9 ) { diff --git a/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl b/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl index 6ea956bc8..c8294b10d 100644 --- a/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl +++ b/crypto/fipsmodule/modes/asm/aes-gcm-avx2-x86_64.pl @@ -805,6 +805,12 @@ ___ .Laes128$local_label_suffix: ___ + # Prefetch the source data 512 bytes ahead into the L1 data cache, to + # improve performance when the hardware prefetcher is disabled. Assumes the + # L1 data cache line size is 64 bytes (de facto standard on x86_64). + $code .= "prefetcht0 512($SRC)\n"; + $code .= "prefetcht0 512+64($SRC)\n"; + # Finish the AES encryption of the counter blocks in AESDATA[0-3], # interleaved with the GHASH update of the ciphertext blocks. for my $i ( reverse 1 .. 9 ) { diff --git a/gen/bcm/aes-gcm-avx10-x86_64-apple.S b/gen/bcm/aes-gcm-avx10-x86_64-apple.S index 54fcde048..be666051f 100644 --- a/gen/bcm/aes-gcm-avx10-x86_64-apple.S +++ b/gen/bcm/aes-gcm-avx10-x86_64-apple.S @@ -512,6 +512,10 @@ L$aes192__func1: vaesenc %zmm9,%zmm3,%zmm3 L$aes128__func1: + prefetcht0 512+0(%rdi) + prefetcht0 512+64(%rdi) + prefetcht0 512+128(%rdi) + prefetcht0 512+192(%rdi) vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 @@ -953,6 +957,10 @@ L$aes192__func2: vaesenc %zmm9,%zmm3,%zmm3 L$aes128__func2: + prefetcht0 512+0(%rdi) + prefetcht0 512+64(%rdi) + prefetcht0 512+128(%rdi) + prefetcht0 512+192(%rdi) vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 diff --git a/gen/bcm/aes-gcm-avx10-x86_64-linux.S b/gen/bcm/aes-gcm-avx10-x86_64-linux.S index 2be6a8cb7..b52562302 100644 --- a/gen/bcm/aes-gcm-avx10-x86_64-linux.S +++ b/gen/bcm/aes-gcm-avx10-x86_64-linux.S @@ -514,6 +514,10 @@ _CET_ENDBR vaesenc %zmm9,%zmm3,%zmm3 .Laes128__func1: + prefetcht0 512+0(%rdi) + prefetcht0 512+64(%rdi) + prefetcht0 512+128(%rdi) + prefetcht0 512+192(%rdi) vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 @@ -957,6 +961,10 @@ _CET_ENDBR vaesenc %zmm9,%zmm3,%zmm3 .Laes128__func2: + prefetcht0 512+0(%rdi) + prefetcht0 512+64(%rdi) + prefetcht0 512+128(%rdi) + prefetcht0 512+192(%rdi) vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 diff --git a/gen/bcm/aes-gcm-avx10-x86_64-win.asm b/gen/bcm/aes-gcm-avx10-x86_64-win.asm index fb9f896a9..733ae7217 100644 --- a/gen/bcm/aes-gcm-avx10-x86_64-win.asm +++ b/gen/bcm/aes-gcm-avx10-x86_64-win.asm @@ -579,6 +579,10 @@ $L$aes192__func1: vaesenc zmm3,zmm3,zmm9 $L$aes128__func1: + prefetcht0 [((512+0))+rcx] + prefetcht0 [((512+64))+rcx] + prefetcht0 [((512+128))+rcx] + prefetcht0 [((512+192))+rcx] vpshufb zmm4,zmm4,zmm8 vpxord zmm4,zmm4,zmm10 vpshufb zmm5,zmm5,zmm8 @@ -1061,6 +1065,10 @@ $L$aes192__func2: vaesenc zmm3,zmm3,zmm9 $L$aes128__func2: + prefetcht0 [((512+0))+rcx] + prefetcht0 [((512+64))+rcx] + prefetcht0 [((512+128))+rcx] + prefetcht0 [((512+192))+rcx] vpshufb zmm4,zmm4,zmm8 vpxord zmm4,zmm4,zmm10 vpshufb zmm5,zmm5,zmm8 diff --git a/gen/bcm/aes-gcm-avx2-x86_64-apple.S b/gen/bcm/aes-gcm-avx2-x86_64-apple.S index e401e6604..d896f2a6e 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-apple.S +++ b/gen/bcm/aes-gcm-avx2-x86_64-apple.S @@ -498,6 +498,8 @@ L$aes192__func1: vaesenc %ymm2,%ymm15,%ymm15 L$aes128__func1: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) vmovdqu 0(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 @@ -983,6 +985,8 @@ L$aes192__func2: vaesenc %ymm2,%ymm15,%ymm15 L$aes128__func2: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) vmovdqu 0(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 diff --git a/gen/bcm/aes-gcm-avx2-x86_64-linux.S b/gen/bcm/aes-gcm-avx2-x86_64-linux.S index b7816cfc5..583f02fc5 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-linux.S +++ b/gen/bcm/aes-gcm-avx2-x86_64-linux.S @@ -500,6 +500,8 @@ _CET_ENDBR vaesenc %ymm2,%ymm15,%ymm15 .Laes128__func1: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) vmovdqu 0(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 @@ -987,6 +989,8 @@ _CET_ENDBR vaesenc %ymm2,%ymm15,%ymm15 .Laes128__func2: + prefetcht0 512(%rdi) + prefetcht0 512+64(%rdi) vmovdqu 0(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 diff --git a/gen/bcm/aes-gcm-avx2-x86_64-win.asm b/gen/bcm/aes-gcm-avx2-x86_64-win.asm index 92015534b..00e2a2bf8 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-win.asm +++ b/gen/bcm/aes-gcm-avx2-x86_64-win.asm @@ -559,6 +559,8 @@ $L$aes192__func1: vaesenc ymm15,ymm15,ymm2 $L$aes128__func1: + prefetcht0 [512+rcx] + prefetcht0 [((512+64))+rcx] vmovdqu ymm3,YMMWORD[rdx] vpshufb ymm3,ymm3,ymm0 @@ -1085,6 +1087,8 @@ $L$aes192__func2: vaesenc ymm15,ymm15,ymm2 $L$aes128__func2: + prefetcht0 [512+rcx] + prefetcht0 [((512+64))+rcx] vmovdqu ymm3,YMMWORD[rcx] vpshufb ymm3,ymm3,ymm0