dlan17/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'bpf-arm64-support-for-timed-may_goto'

Browse Source 
Puranjay Mohan says:

====================
bpf, arm64: support for timed may_goto

Changes in v2->v3:
v2: https://lore.kernel.org/all/20250809204833.44803-1-puranjay@kernel.org/
- Rebased on bpf-next/master
- Added Acked-by: tags from Xu and Kumar

Changes in v1->v2:
v1: https://lore.kernel.org/bpf/20250724125443.26182-1-puranjay@kernel.org/
- Added comment in arch_bpf_timed_may_goto() about BPF_REG_FP setup (Xu
  Kuohai)

This set adds support for the timed may_goto instruction for the arm64.
The timed may_goto instruction is implemented by the verifier by
reserving 2 8byte slots in the program stack and then calling
arch_bpf_timed_may_goto() in a loop with the stack offset of these two
slots in BPF_REG_AX. It expects the function to put a timestamp in the
first slot and the returned count in BPF_REG_AX is put into the second
slot by a store instruction emitted by the verifier.

arch_bpf_timed_may_goto() is special as it receives the parameter in
BPF_REG_AX and is expected to return the result in BPF_REG_AX as well.
It can't clobber any caller saved registers because verifier doesn't
save anything before emitting the call.

So, arch_bpf_timed_may_goto() is implemented in assembly so the exact
registers that are stored/restored can be controlled (BPF caller saved
registers here) and it also needs to take care of moving arguments and
return values to and from BPF_REG_AX <-> arm64 R0.

So, arch_bpf_timed_may_goto() acts as a trampoline to call
bpf_check_timed_may_goto() which does the main logic of placing the
timestamp and returning the count.

All tests that use may_goto instruction pass after the changing some of
them in patch 2

 #404     stream_errors:OK
 [...]
 #406/2   stream_success/stream_cond_break:OK
 [...]
 #494/23  verifier_bpf_fastcall/may_goto_interaction_x86_64:SKIP
 #494/24  verifier_bpf_fastcall/may_goto_interaction_arm64:OK
 [...]
 #539/1   verifier_may_goto_1/may_goto 0:OK
 #539/2   verifier_may_goto_1/batch 2 of may_goto 0:OK
 #539/3   verifier_may_goto_1/may_goto batch with offsets 2/1/0:OK
 #539/4   verifier_may_goto_1/may_goto batch with offsets 2/0:OK
 #539     verifier_may_goto_1:OK
 #540/1   verifier_may_goto_2/C code with may_goto 0:OK
 #540     verifier_may_goto_2:OK
 Summary: 7/16 PASSED, 25 SKIPPED, 0 FAILED
====================

Link: https://patch.msgid.link/20250827113245.52629-1-puranjay@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov
2025-08-27 17:16:23 -07:00
parent 4c229f337e 22b22bf9ee
commit bd27626f48
6 changed files with 76 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/arm64/net/Makefile
+1 -1
View File
@@ -2,4 +2,4 @@
#
# ARM64 networking code
#
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
arch/arm64/net/bpf_jit_comp.c
+12 -1
View File
@@ -1558,7 +1558,13 @@ emit_cond_jmp:
if (ret < 0)
return ret;
emit_call(func_addr, ctx);
emit(A64_MOV(1, r0, A64_R(0)), ctx);
/*
* Call to arch_bpf_timed_may_goto() is emitted by the
* verifier and called with custom calling convention with
* first argument and return value in BPF_REG_AX (x9).
*/
if (func_addr != (u64)arch_bpf_timed_may_goto)
emit(A64_MOV(1, r0, A64_R(0)), ctx);
break;
}
/* tail call */
@@ -3038,6 +3044,11 @@ bool bpf_jit_bypass_spec_v4(void)
return true;
}
bool bpf_jit_supports_timed_may_goto(void)
{
return true;
}
bool bpf_jit_inlines_helper_call(s32 imm)
{
switch (imm) {
arch/arm64/net/bpf_timed_may_goto.S
+40
View File
@@ -0,0 +1,40 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2025 Puranjay Mohan <puranjay@kernel.org> */
#include <linux/linkage.h>
SYM_FUNC_START(arch_bpf_timed_may_goto)
/* Allocate stack space and emit frame record */
stp x29, x30, [sp, #-64]!
mov x29, sp
/* Save BPF registers R0 - R5 (x7, x0-x4)*/
stp x7, x0, [sp, #16]
stp x1, x2, [sp, #32]
stp x3, x4, [sp, #48]
/*
* Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP
* (x25) to get the pointer to count and timestamp and pass it as the
* first argument in x0.
*
* Before generating the call to arch_bpf_timed_may_goto, the verifier
* generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP -
* stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64
* jit in this case.
*/
add x0, x9, x25
bl bpf_check_timed_may_goto
/* BPF_REG_AX(x9) will be stored into count, so move return value to it. */
mov x9, x0
/* Restore BPF registers R0 - R5 (x7, x0-x4) */
ldp x7, x0, [sp, #16]
ldp x1, x2, [sp, #32]
ldp x3, x4, [sp, #48]
/* Restore FP and LR */
ldp x29, x30, [sp], #64
ret
SYM_FUNC_END(arch_bpf_timed_may_goto)
tools/testing/selftests/bpf/prog_tests/stream.c
+1 -1
View File
@@ -77,7 +77,7 @@ void test_stream_errors(void)
ASSERT_OK(ret, "ret");
ASSERT_OK(opts.retval, "retval");
#if !defined(__x86_64__) && !defined(__s390x__)
#if !defined(__x86_64__) && !defined(__s390x__) && !defined(__aarch64__)
ASSERT_TRUE(1, "Timed may_goto unsupported, skip.");
if (i == 0) {
ret = bpf_prog_stream_read(prog_fd, 2, buf, sizeof(buf), &ropts);
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
+16 -11
View File
@@ -660,19 +660,24 @@ __naked void may_goto_interaction_x86_64(void)
SEC("raw_tp")
__arch_arm64
__log_level(4) __msg("stack depth 16")
/* may_goto counter at -16 */
__xlated("0: *(u64 *)(r10 -16) =")
__xlated("1: r1 = 1")
__xlated("2: call bpf_get_smp_processor_id")
__log_level(4) __msg("stack depth 24")
/* may_goto counter at -24 */
__xlated("0: *(u64 *)(r10 -24) =")
/* may_goto timestamp at -16 */
__xlated("1: *(u64 *)(r10 -16) =")
__xlated("2: r1 = 1")
__xlated("3: call bpf_get_smp_processor_id")
/* may_goto expansion starts */
__xlated("3: r11 = *(u64 *)(r10 -16)")
__xlated("4: if r11 == 0x0 goto pc+3")
__xlated("5: r11 -= 1")
__xlated("6: *(u64 *)(r10 -16) = r11")
__xlated("4: r11 = *(u64 *)(r10 -24)")
__xlated("5: if r11 == 0x0 goto pc+6")
__xlated("6: r11 -= 1")
__xlated("7: if r11 != 0x0 goto pc+2")
__xlated("8: r11 = -24")
__xlated("9: call unknown")
__xlated("10: *(u64 *)(r10 -24) = r11")
/* may_goto expansion ends */
__xlated("7: *(u64 *)(r10 -8) = r1")
__xlated("8: exit")
__xlated("11: *(u64 *)(r10 -8) = r1")
__xlated("12: exit")
__success
__naked void may_goto_interaction_arm64(void)
{
tools/testing/selftests/bpf/progs/verifier_may_goto_1.c
+6 -28
View File
@@ -10,6 +10,7 @@ SEC("raw_tp")
__description("may_goto 0")
__arch_x86_64
__arch_s390x
__arch_arm64
__xlated("0: r0 = 1")
__xlated("1: exit")
__success
@@ -29,6 +30,7 @@ SEC("raw_tp")
__description("batch 2 of may_goto 0")
__arch_x86_64
__arch_s390x
__arch_arm64
__xlated("0: r0 = 1")
__xlated("1: exit")
__success
@@ -50,6 +52,7 @@ SEC("raw_tp")
__description("may_goto batch with offsets 2/1/0")
__arch_x86_64
__arch_s390x
__arch_arm64
__xlated("0: r0 = 1")
__xlated("1: exit")
__success
@@ -72,9 +75,10 @@ __naked void may_goto_batch_1(void)
}
SEC("raw_tp")
__description("may_goto batch with offsets 2/0 - x86_64 and s390x")
__description("may_goto batch with offsets 2/0")
__arch_x86_64
__arch_s390x
__arch_arm64
__xlated("0: *(u64 *)(r10 -16) = 65535")
__xlated("1: *(u64 *)(r10 -8) = 0")
__xlated("2: r11 = *(u64 *)(r10 -16)")
@@ -88,33 +92,7 @@ __xlated("9: r0 = 1")
__xlated("10: r0 = 2")
__xlated("11: exit")
__success
__naked void may_goto_batch_2_x86_64_s390x(void)
{
asm volatile (
".8byte %[may_goto1];"
".8byte %[may_goto3];"
"r0 = 1;"
"r0 = 2;"
"exit;"
:
: __imm_insn(may_goto1, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 2 /* offset */, 0)),
__imm_insn(may_goto3, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 0 /* offset */, 0))
: __clobber_all);
}
SEC("raw_tp")
__description("may_goto batch with offsets 2/0 - arm64")
__arch_arm64
__xlated("0: *(u64 *)(r10 -8) = 8388608")
__xlated("1: r11 = *(u64 *)(r10 -8)")
__xlated("2: if r11 == 0x0 goto pc+3")
__xlated("3: r11 -= 1")
__xlated("4: *(u64 *)(r10 -8) = r11")
__xlated("5: r0 = 1")
__xlated("6: r0 = 2")
__xlated("7: exit")
__success
__naked void may_goto_batch_2_arm64(void)
__naked void may_goto_batch_2(void)
{
asm volatile (
".8byte %[may_goto1];"