From dac628e9563640e2de7878decc03a508b1ba319a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 26 Jan 2025 08:46:30 +0100 Subject: [PATCH 1/7] x86/efistub: Merge PE and handover entrypoints The difference between the PE and handover entrypoints in the EFI stub is that the former allocates a struct boot_params whereas the latter expects one from the caller. Currently, these are two completely separate entrypoints, duplicating some logic and both relying of efi_exit() to return straight back to the firmware on an error. Simplify this by making the PE entrypoint call the handover entrypoint with NULL as the argument for the struct boot_params parameter. This makes the code easier to follow, and removes the need to support two different calling conventions in the mixed mode asm code. While at it, move the assignment of boot_params_ptr into the function that actually calls into the legacy decompressor, which is where its value is required. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 16 +------- drivers/firmware/efi/libstub/x86-stub.c | 52 ++++++++++++++----------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index 876fc6d46a13..d681e30c6732 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -56,22 +56,8 @@ SYM_FUNC_START(startup_64_mixed_mode) movl efi32_boot_sp(%rip), %esp andl $~7, %esp -#ifdef CONFIG_EFI_HANDOVER_PROTOCOL mov 8(%rdx), %edx // saved bootparams pointer - test %edx, %edx - jnz efi_stub_entry -#endif - /* - * efi_pe_entry uses MS calling convention, which requires 32 bytes of - * shadow space on the stack even if all arguments are passed in - * registers. We also need an additional 8 bytes for the space that - * would be occupied by the return address, and this also results in - * the correct stack alignment for entry. - */ - sub $40, %rsp - mov %rdi, %rcx // MS calling convention - mov %rsi, %rdx - jmp efi_pe_entry + call efi_stub_entry SYM_FUNC_END(startup_64_mixed_mode) SYM_FUNC_START(__efi64_thunk) diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index 863910e9eefc..cafc90d4caaf 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -397,17 +397,13 @@ static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status) asm("hlt"); } -void __noreturn efi_stub_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg, - struct boot_params *boot_params); - /* * Because the x86 boot code expects to be passed a boot_params we * need to create one ourselves (usually the bootloader would create * one for us). */ -efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg) +static efi_status_t efi_allocate_bootparams(efi_handle_t handle, + struct boot_params **bp) { efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; struct boot_params *boot_params; @@ -416,21 +412,15 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, unsigned long alloc; char *cmdline_ptr; - efi_system_table = sys_table_arg; - - /* Check if we were booted by the EFI firmware */ - if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) - efi_exit(handle, EFI_INVALID_PARAMETER); - status = efi_bs_call(handle_protocol, handle, &proto, (void **)&image); if (status != EFI_SUCCESS) { efi_err("Failed to get handle for LOADED_IMAGE_PROTOCOL\n"); - efi_exit(handle, status); + return status; } status = efi_allocate_pages(PARAM_SIZE, &alloc, ULONG_MAX); if (status != EFI_SUCCESS) - efi_exit(handle, status); + return status; boot_params = memset((void *)alloc, 0x0, PARAM_SIZE); hdr = &boot_params->hdr; @@ -446,14 +436,14 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, cmdline_ptr = efi_convert_cmdline(image); if (!cmdline_ptr) { efi_free(PARAM_SIZE, alloc); - efi_exit(handle, EFI_OUT_OF_RESOURCES); + return EFI_OUT_OF_RESOURCES; } efi_set_u64_split((unsigned long)cmdline_ptr, &hdr->cmd_line_ptr, &boot_params->ext_cmd_line_ptr); - efi_stub_entry(handle, sys_table_arg, boot_params); - /* not reached */ + *bp = boot_params; + return EFI_SUCCESS; } static void add_e820ext(struct boot_params *params, @@ -740,13 +730,16 @@ static efi_status_t parse_options(const char *cmdline) return efi_parse_options(cmdline); } -static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry) +static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry, + struct boot_params *boot_params) { unsigned long virt_addr = LOAD_PHYSICAL_ADDR; unsigned long addr, alloc_size, entry; efi_status_t status; u32 seed[2] = {}; + boot_params_ptr = boot_params; + /* determine the required size of the allocation */ alloc_size = ALIGN(max_t(unsigned long, output_len, kernel_total_size), MIN_KERNEL_ALIGN); @@ -777,7 +770,7 @@ static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry) seed[0] = 0; } - boot_params_ptr->hdr.loadflags |= KASLR_FLAG; + boot_params->hdr.loadflags |= KASLR_FLAG; } status = efi_random_alloc(alloc_size, CONFIG_PHYSICAL_ALIGN, &addr, @@ -815,20 +808,27 @@ static void __noreturn enter_kernel(unsigned long kernel_addr, void __noreturn efi_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, struct boot_params *boot_params) + { efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID; - struct setup_header *hdr = &boot_params->hdr; const struct linux_efi_initrd *initrd = NULL; unsigned long kernel_entry; + struct setup_header *hdr; efi_status_t status; - boot_params_ptr = boot_params; - efi_system_table = sys_table_arg; /* Check if we were booted by the EFI firmware */ if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) efi_exit(handle, EFI_INVALID_PARAMETER); + if (!IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL) || !boot_params) { + status = efi_allocate_bootparams(handle, &boot_params); + if (status != EFI_SUCCESS) + efi_exit(handle, status); + } + + hdr = &boot_params->hdr; + if (have_unsupported_snp_features()) efi_exit(handle, EFI_UNSUPPORTED); @@ -870,7 +870,7 @@ void __noreturn efi_stub_entry(efi_handle_t handle, if (efi_mem_encrypt > 0) hdr->xloadflags |= XLF_MEM_ENCRYPTION; - status = efi_decompress_kernel(&kernel_entry); + status = efi_decompress_kernel(&kernel_entry, boot_params); if (status != EFI_SUCCESS) { efi_err("Failed to decompress kernel\n"); goto fail; @@ -940,6 +940,12 @@ fail: efi_exit(handle, status); } +efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, + efi_system_table_t *sys_table_arg) +{ + efi_stub_entry(handle, sys_table_arg, NULL); +} + #ifdef CONFIG_EFI_HANDOVER_PROTOCOL void efi_handover_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, struct boot_params *boot_params) From eaed89559591f73ca06a3e6534c381e3bd948ee6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Jan 2025 15:57:16 +0100 Subject: [PATCH 2/7] x86/efi/mixed: Check CPU compatibility without relying on verify_cpu() In order for the EFI mixed mode startup code to be reusable in a context where the legacy decompressor is not used, replace the call to verify_cpu() [which performs an elaborate set of checks] with a simple check against the 'long mode' bit in the appropriate CPUID leaf. This is reasonable, given that EFI support is implied when booting in this manner, and so there is no need to consider very old CPUs when performing this check. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index d681e30c6732..b7886e2591fc 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -279,24 +279,20 @@ SYM_FUNC_END(efi32_entry) * efi_system_table_32_t *sys_table) */ SYM_FUNC_START(efi32_pe_entry) - pushl %ebp - movl %esp, %ebp pushl %ebx // save callee-save registers - pushl %edi - call verify_cpu // check for long mode support - testl %eax, %eax - movl $0x80000003, %eax // EFI_UNSUPPORTED - jnz 2f - - movl 8(%ebp), %ecx // image_handle - movl 12(%ebp), %edx // sys_table + /* Check whether the CPU supports long mode */ + movl $0x80000001, %eax // assume extended info support + cpuid + btl $29, %edx // check long mode bit + jnc 1f + leal 8(%esp), %esp // preserve stack alignment + movl (%esp), %ecx // image_handle + movl 4(%esp), %edx // sys_table jmp efi32_entry // pass %ecx, %edx // no other registers remain live - -2: popl %edi // restore callee-save registers +1: movl $0x80000003, %eax // EFI_UNSUPPORTED popl %ebx - leave RET SYM_FUNC_END(efi32_pe_entry) From ff38bbbac39eade57e4672f9601f602ed8d4b493 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 12 Jan 2025 13:56:46 +0100 Subject: [PATCH 3/7] x86/efi/mixed: Factor out and clean up long mode entry Entering long mode involves setting the EFER_LME and CR4.PAE bits before enabling paging by setting CR0.PG bit. It also involves disabling interrupts, given that the firmware's 32-bit IDT becomes invalid as soon as the CPU transitions into long mode. Reloading the CR3 register is not necessary at boot time, given that the EFI firmware as well as the kernel's EFI stub use a 1:1 mapping of the 32-bit addressable memory in the system. Break out this code into a separate helper for clarity, and so that it can be reused in a subsequent patch. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index b7886e2591fc..0b6b37b08f82 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -170,10 +170,6 @@ SYM_FUNC_START_LOCAL(efi_enter32) movl %edx, %gs movl %edx, %ss - /* Reload pgtables */ - movl %cr3, %eax - movl %eax, %cr3 - /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -199,30 +195,35 @@ SYM_FUNC_START_LOCAL(efi_enter32) lidtl 16(%ebx) lgdtl (%ebx) + xorl %eax, %eax + lldt %ax + + call efi32_enable_long_mode + + pushl $__KERNEL_CS + pushl %ebp + lret +SYM_FUNC_END(efi_enter32) + +SYM_FUNC_START_LOCAL(efi32_enable_long_mode) movl %cr4, %eax btsl $(X86_CR4_PAE_BIT), %eax movl %eax, %cr4 - movl %cr3, %eax - movl %eax, %cr3 - movl $MSR_EFER, %ecx rdmsr btsl $_EFER_LME, %eax wrmsr - xorl %eax, %eax - lldt %ax - - pushl $__KERNEL_CS - pushl %ebp + /* Disable interrupts - the firmware's IDT does not work in long mode */ + cli /* Enable paging */ movl %cr0, %eax btsl $X86_CR0_PG_BIT, %eax movl %eax, %cr0 - lret -SYM_FUNC_END(efi_enter32) + ret +SYM_FUNC_END(efi32_enable_long_mode) /* * This is the common EFI stub entry point for mixed mode. From d545e182a8bb37bce3c00d89ab88e33414add1c3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 5 Feb 2025 12:23:21 +0100 Subject: [PATCH 4/7] x86/efi/mixed: Set up 1:1 mapping of lower 4GiB in the stub In preparation for dropping the dependency on startup_32 entirely in the next patch, add the code that sets up the 1:1 mapping of the lower 4 GiB of system RAM to the mixed mode stub. The reload of CR3 after the long mode switch will be removed in a subsequent patch, when it is no longer needed. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index 0b6b37b08f82..dca916c3e6f0 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,9 @@ SYM_FUNC_START(startup_64_mixed_mode) mov 0(%rdx), %edi mov 4(%rdx), %esi + leaq (pte + 5 * PAGE_SIZE)(%rip), %rax + movq %rax, %cr3 // reload after startup_32 + /* Switch to the firmware's stack */ movl efi32_boot_sp(%rip), %esp andl $~7, %esp @@ -267,11 +271,32 @@ SYM_FUNC_START_LOCAL(efi32_entry) movl $_end - 1b, BP_init_size(%esi) subl $startup_32 - 1b, BP_init_size(%esi) + call 1f +1: pop %edi + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax movl %eax, %cr0 + /* Set up 1:1 mapping */ + leal (pte - 1b)(%edi), %eax + movl $_PAGE_PRESENT | _PAGE_RW | _PAGE_PSE, %ecx + leal (_PAGE_PRESENT | _PAGE_RW)(%eax), %edx +2: movl %ecx, (%eax) + addl $8, %eax + addl $PMD_SIZE, %ecx + jnc 2b + + movl $PAGE_SIZE, %ecx + .irpc l, 0123 + movl %edx, \l * 8(%eax) + addl %ecx, %edx + .endr + addl %ecx, %eax + movl %edx, (%eax) + movl %eax, %cr3 + jmp startup_32 SYM_FUNC_END(efi32_entry) @@ -322,3 +347,7 @@ SYM_DATA_LOCAL(efi32_boot_ds, .word 0) SYM_DATA_LOCAL(efi32_boot_sp, .long 0) SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0) SYM_DATA(efi_is64, .byte 1) + + .bss + .balign PAGE_SIZE +SYM_DATA_LOCAL(pte, .fill 6 * PAGE_SIZE, 1, 0) From 6e2da8d87c9c1133d8d6e1f93a0bc0c416dbc8ee Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Jan 2025 17:55:07 +0100 Subject: [PATCH 5/7] x86/efi/mixed: Remove dependency on legacy startup_32 code The EFI mixed mode startup code calls into startup_32 in the legacy decompressor with a mocked up boot_params struct, only to get it to set up the 1:1 mapping of the lower 4 GiB of memory and switch to a GDT that supports 64-bit mode. In order to be able to reuse the EFI mixed mode startup code in EFI zboot images, which do not incorporate the legacy decompressor code, decouple it, by dealing with the GDT and IDT directly. Doing so makes it possible to construct a GDT that is compatible with the one the firmware uses, with one additional entry for a 64-bit mode code segment appended. This removes the need entirely to switch between GDTs and IDTs or data segment selector values and all of this code can be removed. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 221 ++++++++------------------- arch/x86/boot/compressed/head_64.S | 7 - 2 files changed, 62 insertions(+), 166 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index dca916c3e6f0..984956931ed7 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -15,70 +15,23 @@ */ #include -#include +#include #include #include #include #include #include -#include .code64 .text -/* - * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode() - * is the first thing that runs after switching to long mode. Depending on - * whether the EFI handover protocol or the compat entry point was used to - * enter the kernel, it will either branch to the common 64-bit EFI stub - * entrypoint efi_stub_entry() directly, or via the 64-bit EFI PE/COFF - * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a - * struct bootparams pointer as the third argument, so the presence of such a - * pointer is used to disambiguate. - * - * +--------------+ - * +------------------+ +------------+ +------>| efi_pe_entry | - * | efi32_pe_entry |---->| | | +-----------+--+ - * +------------------+ | | +------+----------------+ | - * | startup_32 |---->| startup_64_mixed_mode | | - * +------------------+ | | +------+----------------+ | - * | efi32_stub_entry |---->| | | | - * +------------------+ +------------+ | | - * V | - * +------------+ +----------------+ | - * | startup_64 |<----| efi_stub_entry |<--------+ - * +------------+ +----------------+ - */ -SYM_FUNC_START(startup_64_mixed_mode) - lea efi32_boot_args(%rip), %rdx - mov 0(%rdx), %edi - mov 4(%rdx), %esi - - leaq (pte + 5 * PAGE_SIZE)(%rip), %rax - movq %rax, %cr3 // reload after startup_32 - - /* Switch to the firmware's stack */ - movl efi32_boot_sp(%rip), %esp - andl $~7, %esp - - mov 8(%rdx), %edx // saved bootparams pointer - call efi_stub_entry -SYM_FUNC_END(startup_64_mixed_mode) - SYM_FUNC_START(__efi64_thunk) push %rbp push %rbx - movl %ds, %eax - push %rax - movl %es, %eax - push %rax - movl %ss, %eax - push %rax - /* Copy args passed on stack */ - movq 0x30(%rsp), %rbp - movq 0x38(%rsp), %rbx - movq 0x40(%rsp), %rax + movq 0x18(%rsp), %rbp + movq 0x20(%rsp), %rbx + movq 0x28(%rsp), %rax /* * Convert x86-64 ABI params to i386 ABI @@ -93,44 +46,14 @@ SYM_FUNC_START(__efi64_thunk) movl %ebx, 0x18(%rsp) movl %eax, 0x1c(%rsp) - leaq 0x20(%rsp), %rbx - sgdt (%rbx) - sidt 16(%rbx) - leaq 1f(%rip), %rbp + movl %cs, %ebx - /* - * Switch to IDT and GDT with 32-bit segments. These are the firmware - * GDT and IDT that were installed when the kernel started executing. - * The pointers were saved by the efi32_entry() routine below. - * - * Pass the saved DS selector to the 32-bit code, and use far return to - * restore the saved CS selector. - */ - lidt efi32_boot_idt(%rip) - lgdt efi32_boot_gdt(%rip) - - movzwl efi32_boot_ds(%rip), %edx - movzwq efi32_boot_cs(%rip), %rax - pushq %rax - leaq efi_enter32(%rip), %rax - pushq %rax - lretq + ljmpl *efi32_call(%rip) 1: addq $64, %rsp movq %rdi, %rax - pop %rbx - movl %ebx, %ss - pop %rbx - movl %ebx, %es - pop %rbx - movl %ebx, %ds - /* Clear out 32-bit selector from FS and GS */ - xorl %ebx, %ebx - movl %ebx, %fs - movl %ebx, %gs - pop %rbx pop %rbp RET @@ -141,7 +64,6 @@ SYM_FUNC_END(__efi64_thunk) SYM_FUNC_START(efi32_stub_entry) call 1f 1: popl %ecx - leal (efi32_boot_args - 1b)(%ecx), %ebx /* Clear BSS */ xorl %eax, %eax @@ -153,11 +75,8 @@ SYM_FUNC_START(efi32_stub_entry) rep stosl add $0x4, %esp /* Discard return address */ - popl %ecx - popl %edx - popl %esi - movl %esi, 8(%ebx) - jmp efi32_entry + movl 8(%esp), %ebx /* struct boot_params pointer */ + jmp efi32_startup SYM_FUNC_END(efi32_stub_entry) #endif @@ -167,13 +86,6 @@ SYM_FUNC_END(efi32_stub_entry) * The stack should represent the 32-bit calling convention. */ SYM_FUNC_START_LOCAL(efi_enter32) - /* Load firmware selector into data and stack segment registers */ - movl %edx, %ds - movl %edx, %es - movl %edx, %fs - movl %edx, %gs - movl %edx, %ss - /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -190,21 +102,9 @@ SYM_FUNC_START_LOCAL(efi_enter32) /* We must preserve return value */ movl %eax, %edi - /* - * Some firmware will return with interrupts enabled. Be sure to - * disable them before we switch GDTs and IDTs. - */ - cli - - lidtl 16(%ebx) - lgdtl (%ebx) - - xorl %eax, %eax - lldt %ax - call efi32_enable_long_mode - pushl $__KERNEL_CS + pushl %ebx pushl %ebp lret SYM_FUNC_END(efi_enter32) @@ -230,50 +130,56 @@ SYM_FUNC_START_LOCAL(efi32_enable_long_mode) SYM_FUNC_END(efi32_enable_long_mode) /* - * This is the common EFI stub entry point for mixed mode. + * This is the common EFI stub entry point for mixed mode. It sets up the GDT + * and page tables needed for 64-bit execution, after which it calls the + * common 64-bit EFI entrypoint efi_stub_entry(). * - * Arguments: %ecx image handle - * %edx EFI system table pointer + * Arguments: 0(%esp) image handle + * 4(%esp) EFI system table pointer + * %ebx struct boot_params pointer (or NULL) * * Since this is the point of no return for ordinary execution, no registers * are considered live except for the function parameters. [Note that the EFI * stub may still exit and return to the firmware using the Exit() EFI boot * service.] */ -SYM_FUNC_START_LOCAL(efi32_entry) - call 1f -1: pop %ebx +SYM_FUNC_START_LOCAL(efi32_startup) + movl %esp, %ebp - /* Save firmware GDTR and code/data selectors */ - sgdtl (efi32_boot_gdt - 1b)(%ebx) - movw %cs, (efi32_boot_cs - 1b)(%ebx) - movw %ds, (efi32_boot_ds - 1b)(%ebx) + subl $8, %esp + sgdtl (%esp) /* Save GDT descriptor to the stack */ + movl 2(%esp), %esi /* Existing GDT pointer */ + movzwl (%esp), %ecx /* Existing GDT limit */ + inc %ecx /* Existing GDT size */ + andl $~7, %ecx /* Ensure size is multiple of 8 */ - /* Store firmware IDT descriptor */ - sidtl (efi32_boot_idt - 1b)(%ebx) + subl %ecx, %esp /* Allocate new GDT */ + andl $~15, %esp /* Realign the stack */ + movl %esp, %edi /* New GDT address */ + leal 7(%ecx), %eax /* New GDT limit */ + pushw %cx /* Push 64-bit CS (for LJMP below) */ + pushl %edi /* Push new GDT address */ + pushw %ax /* Push new GDT limit */ - /* Store firmware stack pointer */ - movl %esp, (efi32_boot_sp - 1b)(%ebx) - - /* Store boot arguments */ - leal (efi32_boot_args - 1b)(%ebx), %ebx - movl %ecx, 0(%ebx) - movl %edx, 4(%ebx) - movb $0x0, 12(%ebx) // efi_is64 - - /* - * Allocate some memory for a temporary struct boot_params, which only - * needs the minimal pieces that startup_32() relies on. - */ - subl $PARAM_SIZE, %esp - movl %esp, %esi - movl $PAGE_SIZE, BP_kernel_alignment(%esi) - movl $_end - 1b, BP_init_size(%esi) - subl $startup_32 - 1b, BP_init_size(%esi) + /* Copy GDT to the stack and add a 64-bit code segment at the end */ + movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) & 0xffffffff, (%edi,%ecx) + movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) >> 32, 4(%edi,%ecx) + shrl $2, %ecx + cld + rep movsl /* Copy the firmware GDT */ + lgdtl (%esp) /* Switch to the new GDT */ call 1f 1: pop %edi + /* Record mixed mode entry */ + movb $0x0, (efi_is64 - 1b)(%edi) + + /* Set up indirect far call to re-enter 32-bit mode */ + leal (efi32_call - 1b)(%edi), %eax + addl %eax, (%eax) + movw %cs, 4(%eax) + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -297,8 +203,17 @@ SYM_FUNC_START_LOCAL(efi32_entry) movl %edx, (%eax) movl %eax, %cr3 - jmp startup_32 -SYM_FUNC_END(efi32_entry) + call efi32_enable_long_mode + + /* Set up far jump to 64-bit mode (CS is already on the stack) */ + leal (efi_stub_entry - 1b)(%edi), %eax + movl %eax, 2(%esp) + + movl 0(%ebp), %edi + movl 4(%ebp), %esi + movl %ebx, %edx + ljmpl *2(%esp) +SYM_FUNC_END(efi32_startup) /* * efi_status_t efi32_pe_entry(efi_handle_t image_handle, @@ -313,10 +228,8 @@ SYM_FUNC_START(efi32_pe_entry) btl $29, %edx // check long mode bit jnc 1f leal 8(%esp), %esp // preserve stack alignment - movl (%esp), %ecx // image_handle - movl 4(%esp), %edx // sys_table - jmp efi32_entry // pass %ecx, %edx - // no other registers remain live + xor %ebx, %ebx // no struct boot_params pointer + jmp efi32_startup // only ESP and EBX remain live 1: movl $0x80000003, %eax // EFI_UNSUPPORTED popl %ebx RET @@ -332,20 +245,10 @@ SYM_FUNC_END(efi64_stub_entry) .data .balign 8 -SYM_DATA_START_LOCAL(efi32_boot_gdt) - .word 0 - .quad 0 -SYM_DATA_END(efi32_boot_gdt) - -SYM_DATA_START_LOCAL(efi32_boot_idt) - .word 0 - .quad 0 -SYM_DATA_END(efi32_boot_idt) - -SYM_DATA_LOCAL(efi32_boot_cs, .word 0) -SYM_DATA_LOCAL(efi32_boot_ds, .word 0) -SYM_DATA_LOCAL(efi32_boot_sp, .long 0) -SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0) +SYM_DATA_START_LOCAL(efi32_call) + .long efi_enter32 - . + .word 0x0 +SYM_DATA_END(efi32_call) SYM_DATA(efi_is64, .byte 1) .bss diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 1dcb794c5479..5db6495a3bb9 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -263,13 +263,6 @@ SYM_FUNC_START(startup_32) * used to perform that far jump. */ leal rva(startup_64)(%ebp), %eax -#ifdef CONFIG_EFI_MIXED - cmpb $1, rva(efi_is64)(%ebp) - je 1f - leal rva(startup_64_mixed_mode)(%ebp), %eax -1: -#endif - pushl $__KERNEL_CS pushl %eax From b891e4209c9f96e25a4e90b86e460f515e902c37 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 6 Jan 2025 14:20:35 +0100 Subject: [PATCH 6/7] x86/efi/mixed: Simplify and document thunking logic Now that the GDT/IDT and data segment selector preserve/restore logic has been removed from the boot-time EFI mixed mode thunking routines, the remaining logic to handle the function arguments can be simplified: the setup of the arguments on the stack can be moved into the 32-bit callee, which is able to use a more idiomatic sequence of PUSH instructions. This, in turn, allows the far call and far return to be issued using plain LCALL and LRET instructions, removing the need to set up the return explicitly. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 77 +++++++++++++--------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index 984956931ed7..e04ed99bc449 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -22,43 +22,7 @@ #include #include - .code64 .text -SYM_FUNC_START(__efi64_thunk) - push %rbp - push %rbx - - /* Copy args passed on stack */ - movq 0x18(%rsp), %rbp - movq 0x20(%rsp), %rbx - movq 0x28(%rsp), %rax - - /* - * Convert x86-64 ABI params to i386 ABI - */ - subq $64, %rsp - movl %esi, 0x0(%rsp) - movl %edx, 0x4(%rsp) - movl %ecx, 0x8(%rsp) - movl %r8d, 0xc(%rsp) - movl %r9d, 0x10(%rsp) - movl %ebp, 0x14(%rsp) - movl %ebx, 0x18(%rsp) - movl %eax, 0x1c(%rsp) - - leaq 1f(%rip), %rbp - movl %cs, %ebx - - ljmpl *efi32_call(%rip) - -1: addq $64, %rsp - movq %rdi, %rax - - pop %rbx - pop %rbp - RET -SYM_FUNC_END(__efi64_thunk) - .code32 #ifdef CONFIG_EFI_HANDOVER_PROTOCOL SYM_FUNC_START(efi32_stub_entry) @@ -81,11 +45,26 @@ SYM_FUNC_END(efi32_stub_entry) #endif /* - * EFI service pointer must be in %edi. + * Called using a far call from __efi64_thunk() below, using the x86_64 SysV + * ABI (except for R8/R9 which are inaccessible to 32-bit code - EAX/EBX are + * used instead). EBP+16 points to the arguments passed via the stack. * - * The stack should represent the 32-bit calling convention. + * The first argument (EDI) is a pointer to the boot service or protocol, to + * which the remaining arguments are passed, each truncated to 32 bits. */ SYM_FUNC_START_LOCAL(efi_enter32) + /* + * Convert x86-64 SysV ABI params to i386 ABI + */ + pushl 32(%ebp) /* Up to 3 args passed via the stack */ + pushl 24(%ebp) + pushl 16(%ebp) + pushl %ebx /* R9 */ + pushl %eax /* R8 */ + pushl %ecx + pushl %edx + pushl %esi + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -104,11 +83,29 @@ SYM_FUNC_START_LOCAL(efi_enter32) call efi32_enable_long_mode - pushl %ebx - pushl %ebp + addl $32, %esp + movl %edi, %eax lret SYM_FUNC_END(efi_enter32) + .code64 +SYM_FUNC_START(__efi64_thunk) + push %rbp + movl %esp, %ebp + push %rbx + + /* Move args #5 and #6 into 32-bit accessible registers */ + movl %r8d, %eax + movl %r9d, %ebx + + lcalll *efi32_call(%rip) + + pop %rbx + pop %rbp + RET +SYM_FUNC_END(__efi64_thunk) + + .code32 SYM_FUNC_START_LOCAL(efi32_enable_long_mode) movl %cr4, %eax btsl $(X86_CR4_PAE_BIT), %eax From fb84cefd4ce77c3d63cd3d23adaa7faaef3737cc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Jan 2025 18:16:59 +0100 Subject: [PATCH 7/7] x86/efi/mixed: Move mixed mode startup code into libstub The EFI mixed mode code has been decoupled from the legacy decompressor, in order to be able to reuse it with generic EFI zboot images for x86. Move the source file into the libstub source directory to facilitate this. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/Makefile | 1 - drivers/firmware/efi/libstub/Makefile | 3 +++ .../efi_mixed.S => drivers/firmware/efi/libstub/x86-mixed.S | 0 3 files changed, 3 insertions(+), 1 deletion(-) rename arch/x86/boot/compressed/efi_mixed.S => drivers/firmware/efi/libstub/x86-mixed.S (100%) diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index f2051644de94..fc5563704466 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -104,7 +104,6 @@ vmlinux-objs-$(CONFIG_INTEL_TDX_GUEST) += $(obj)/tdx.o $(obj)/tdcall.o $(obj)/td vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o -vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_mixed.o vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 1141cd06011f..903afd2d3d58 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -62,6 +62,8 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) # `-fdata-sections` flag from KBUILD_CFLAGS_KERNEL KBUILD_CFLAGS_KERNEL := $(filter-out -fdata-sections, $(KBUILD_CFLAGS_KERNEL)) +KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ + lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ file.o mem.o random.o randomalloc.o pci.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ @@ -83,6 +85,7 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \ lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o lib-$(CONFIG_X86) += x86-stub.o smbios.o +lib-$(CONFIG_EFI_MIXED) += x86-mixed.o lib-$(CONFIG_X86_64) += x86-5lvl.o lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o diff --git a/arch/x86/boot/compressed/efi_mixed.S b/drivers/firmware/efi/libstub/x86-mixed.S similarity index 100% rename from arch/x86/boot/compressed/efi_mixed.S rename to drivers/firmware/efi/libstub/x86-mixed.S