lkml.org 
[lkml]   [2021]   [Nov]   [10]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH V5 24/50] x86/entry: Implement the whole error_entry() as C code
    Date
    From: Lai Jiangshan <laijs@linux.alibaba.com>

    All the needed facilities are set in entry64.c, the whole error_entry()
    can be implemented in C in entry64.c. The C version generally has better
    readability and easier to be updated/improved.

    No function change intended.

    Signed-off-by: Lai Jiangshan <laijs@linux.alibaba.com>
    ---
    arch/x86/entry/entry64.c | 68 ++++++++++++++++++++++++++++++
    arch/x86/entry/entry_64.S | 82 +-----------------------------------
    arch/x86/include/asm/traps.h | 1 +
    3 files changed, 70 insertions(+), 81 deletions(-)

    diff --git a/arch/x86/entry/entry64.c b/arch/x86/entry/entry64.c
    index 3db503ea0703..0dc63ae8153a 100644
    --- a/arch/x86/entry/entry64.c
    +++ b/arch/x86/entry/entry64.c
    @@ -68,3 +68,71 @@ static __always_inline void user_entry_swapgs_and_fence(void)
    native_swapgs();
    fence_swapgs_user_entry();
    }
    +
    +/*
    + * Put pt_regs onto the task stack and switch GS and CR3 if needed.
    + * The actual stack switch is done in entry_64.S.
    + *
    + * Be careful, it might be in the user CR3 and user GS base at the start
    + * of the function.
    + */
    +asmlinkage __visible __entry_text
    +struct pt_regs *error_entry(struct pt_regs *eregs)
    +{
    + unsigned long iret_ip = (unsigned long)native_irq_return_iret;
    +
    + if (user_mode(eregs)) {
    + /*
    + * We entered from user mode.
    + * Switch to kernel gsbase and CR3.
    + */
    + user_entry_swapgs_and_fence();
    + switch_to_kernel_cr3();
    +
    + /* Put pt_regs onto the task stack. */
    + return sync_regs(eregs);
    + }
    +
    + /*
    + * There are two places in the kernel that can potentially fault with
    + * usergs. Handle them here. B stepping K8s sometimes report a
    + * truncated RIP for IRET exceptions returning to compat mode. Check
    + * for these here too.
    + */
    + if ((eregs->ip == iret_ip) || (eregs->ip == (unsigned int)iret_ip)) {
    + eregs->ip = iret_ip; /* Fix truncated RIP */
    +
    + /*
    + * We came from an IRET to user mode, so we have user
    + * gsbase and CR3. Switch to kernel gsbase and CR3:
    + */
    + user_entry_swapgs_and_fence();
    + switch_to_kernel_cr3();
    +
    + /*
    + * Pretend that the exception came from user mode: set up
    + * pt_regs as if we faulted immediately after IRET and put
    + * pt_regs onto the real task stack.
    + */
    + return sync_regs(fixup_bad_iret(eregs));
    + }
    +
    + /*
    + * Hack: asm_load_gs_index_gs_change can fail with user gsbase.
    + * If this happens, fix up gsbase and proceed. We'll fix up the
    + * exception and land in asm_load_gs_index_gs_change's error
    + * handler with kernel gsbase.
    + */
    + if (eregs->ip == (unsigned long)asm_load_gs_index_gs_change)
    + native_swapgs();
    +
    + /*
    + * The above code has no serializing instruction. So do an lfence
    + * to prevent GS speculation, regardless of whether it is kernel
    + * gsbase or user gsbase.
    + */
    + fence_swapgs_kernel_entry();
    +
    + /* Enter from kernel, don't move pt_regs */
    + return eregs;
    +}
    diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
    index ebc7419a0367..dac327c56204 100644
    --- a/arch/x86/entry/entry_64.S
    +++ b/arch/x86/entry/entry_64.S
    @@ -333,7 +333,7 @@ SYM_CODE_END(ret_from_fork)
    * XENPV uses its own pvops for iret and load_gs_index(). And it
    * doesn't need to switch CR3. So it can skip invoking error_entry().
    */
    - ALTERNATIVE "call error_entry; movq %rax, %rsp", \
    + ALTERNATIVE "movq %rsp, %rdi; call error_entry; movq %rax, %rsp", \
    "", X86_FEATURE_XENPV

    ENCODE_FRAME_POINTER
    @@ -985,86 +985,6 @@ SYM_CODE_START_LOCAL(paranoid_exit)
    jmp restore_regs_and_return_to_kernel
    SYM_CODE_END(paranoid_exit)

    -/*
    - * Save all registers in pt_regs, and switch GS if needed.
    - */
    -SYM_CODE_START_LOCAL(error_entry)
    - UNWIND_HINT_FUNC
    - testb $3, CS+8(%rsp)
    - jz .Lerror_kernelspace
    -
    - /*
    - * We entered from user mode or we're pretending to have entered
    - * from user mode due to an IRET fault.
    - */
    - swapgs
    - FENCE_SWAPGS_USER_ENTRY
    - /* We have user CR3. Change to kernel CR3. */
    - SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
    -
    - leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
    -.Lerror_entry_from_usermode_after_swapgs:
    - /* Put us onto the real thread stack. */
    - call sync_regs
    - ret
    -
    - /*
    - * There are two places in the kernel that can potentially fault with
    - * usergs. Handle them here. B stepping K8s sometimes report a
    - * truncated RIP for IRET exceptions returning to compat mode. Check
    - * for these here too.
    - */
    -.Lerror_kernelspace:
    - leaq native_irq_return_iret(%rip), %rcx
    - cmpq %rcx, RIP+8(%rsp)
    - je .Lerror_bad_iret
    - movl %ecx, %eax /* zero extend */
    - cmpq %rax, RIP+8(%rsp)
    - je .Lbstep_iret
    - cmpq $asm_load_gs_index_gs_change, RIP+8(%rsp)
    - jne .Lerror_entry_done_lfence
    -
    - /*
    - * hack: .Lgs_change can fail with user gsbase. If this happens, fix up
    - * gsbase and proceed. We'll fix up the exception and land in
    - * .Lgs_change's error handler with kernel gsbase.
    - */
    - swapgs
    -
    - /*
    - * The above code has no serializing instruction. So do an lfence
    - * to prevent GS speculation, regardless of whether it is kernel
    - * gsbase or user gsbase.
    - */
    -.Lerror_entry_done_lfence:
    - FENCE_SWAPGS_KERNEL_ENTRY
    - leaq 8(%rsp), %rax /* return pt_regs pointer */
    - ret
    -
    -.Lbstep_iret:
    - /* Fix truncated RIP */
    - movq %rcx, RIP+8(%rsp)
    - /* fall through */
    -
    -.Lerror_bad_iret:
    - /*
    - * We came from an IRET to user mode, so we have user
    - * gsbase and CR3. Switch to kernel gsbase and CR3:
    - */
    - swapgs
    - FENCE_SWAPGS_USER_ENTRY
    - SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
    -
    - /*
    - * Pretend that the exception came from user mode: set up pt_regs
    - * as if we faulted immediately after IRET.
    - */
    - leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
    - call fixup_bad_iret
    - mov %rax, %rdi
    - jmp .Lerror_entry_from_usermode_after_swapgs
    -SYM_CODE_END(error_entry)
    -
    SYM_CODE_START_LOCAL(error_return)
    UNWIND_HINT_REGS
    DEBUG_ENTRY_ASSERT_IRQS_OFF
    diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
    index 1cdd7e8bcba7..686461ac9803 100644
    --- a/arch/x86/include/asm/traps.h
    +++ b/arch/x86/include/asm/traps.h
    @@ -14,6 +14,7 @@
    asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs);
    asmlinkage __visible notrace
    struct pt_regs *fixup_bad_iret(struct pt_regs *bad_regs);
    +asmlinkage __visible notrace struct pt_regs *error_entry(struct pt_regs *eregs);
    void __init trap_init(void);
    asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);
    #endif
    --
    2.19.1.6.gb485710b
    \
     
     \ /
      Last update: 2021-11-10 13:02    [W:3.472 / U:0.012 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site