Messages in this thread Patch in this message | | | Date | Thu, 04 Jul 2019 21:56:02 +0200 | From | Peter Zijlstra <> | Subject | [RFC][PATCH v2 7/7] x86/entry/64: Pull bits into C |
| |
TODO: 32bit, Xen
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> --- arch/x86/entry/entry_64.S | 13 +++------- arch/x86/include/asm/idtentry.h | 49 ++++++++++++++++++++++++++++++++++++++++ arch/x86/include/asm/traps.h | 1 arch/x86/kernel/cpu/mce/core.c | 2 - arch/x86/kernel/kvm.c | 4 --- arch/x86/kernel/traps.c | 30 ++++++++---------------- arch/x86/mm/fault.c | 4 --- 7 files changed, 68 insertions(+), 35 deletions(-)
--- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -841,15 +841,6 @@ apicinterrupt IRQ_WORK_VECTOR irq_work GET_CR2_INTO(%rdx); /* can clobber %rax */ .endif - TRACE_IRQS_OFF - - .if \paranoid == 0 - testb $3, CS(%rsp) - jz .Lfrom_kernel_no_context_tracking_\@ - CALL_enter_from_user_mode -.Lfrom_kernel_no_context_tracking_\@: - .endif - movq %rsp, %rdi /* pt_regs pointer */ .if \has_error_code @@ -863,7 +854,11 @@ apicinterrupt IRQ_WORK_VECTOR irq_work subq $\ist_offset, CPU_TSS_IST(\shift_ist) .endif + .if \paranoid + call \do_sym\()_paranoid + .else call \do_sym + .endif .if \shift_ist != -1 addq $\ist_offset, CPU_TSS_IST(\shift_ist) --- /dev/null +++ b/arch/x86/include/asm/idtentry.h @@ -0,0 +1,49 @@ +#ifndef __ASM_IDTENTRY_H +#define __ASM_IDTENTRY_H + +/* shamelessly stolen from linux/syscalls.h; XXX share */ + +#define __IDT_MAP0(m,...) +#define __IDT_MAP1(m,t,a,...) m(t,a) +#define __IDT_MAP2(m,t,a,...) m(t,a), __IDT_MAP1(m,__VA_ARGS__) +#define __IDT_MAP3(m,t,a,...) m(t,a), __IDT_MAP2(m,__VA_ARGS__) + +#define __IDT_MAP(n,...) __IDT_MAP##n(__VA_ARGS__) + +#define __IDT_DECL(t, a) t a +#define __IDT_ARGS(t, a) a +#define __IDT_TEST(t, a) (void)BUILD_BUG_ON_ZERO(sizeof(t) != sizeof(long)) + +#ifdef CONFIG_CONTEXT_TRACKING +#define CALL_enter_from_user_mode(_regs) \ + if (static_branch_unlikely(&context_tracking_enabled) && user_mode(_regs)) \ + enter_from_user_mode() +#else +#define CALL_enter_from_user_mode(_regs) +#endif + +#define IDTENTRYx(n, name, ...) \ + static notrace void __idt_##name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)); \ + NOKPROBE_SYMBOL(__idt_##name); \ + dotraplinkage notrace void name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) \ + { \ + __IDT_MAP(n, __IDT_TEST, __VA_ARGS__); \ + trace_hardirqs_off(); \ + CALL_enter_from_user_mode(regs); \ + __idt_##name(__IDT_MAP(n, __IDT_ARGS, __VA_ARGS__)); \ + } \ + NOKPROBE_SYMBOL(name); \ + dotraplinkage notrace void name##_paranoid(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) \ + { \ + __IDT_MAP(n, __IDT_TEST, __VA_ARGS__); \ + trace_hardirqs_off(); \ + __idt_##name(__IDT_MAP(n, __IDT_ARGS, __VA_ARGS__)); \ + } \ + NOKPROBE_SYMBOL(name##_paranoid); \ + static notrace void __idt_##name(__IDT_MAP(n, __IDT_DECL, __VA_ARGS__)) + +#define IDTENTRY1(name,...) IDTENTRYx(1, name, __VA_ARGS__) +#define IDTENTRY2(name,...) IDTENTRYx(2, name, __VA_ARGS__) +#define IDTENTRY3(name,...) IDTENTRYx(3, name, __VA_ARGS__) + +#endif /* __ASM_IDTENTRY_H */ --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -7,6 +7,7 @@ #include <asm/debugreg.h> #include <asm/siginfo.h> /* TRAP_TRACE, ... */ +#include <asm/idtentry.h> #define dotraplinkage __visible --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1842,7 +1842,7 @@ static void unexpected_machine_check(str void (*machine_check_vector)(struct pt_regs *, long error_code) = unexpected_machine_check; -dotraplinkage void do_mce(struct pt_regs *regs, long error_code) +IDTENTRY2(do_mce, struct pt_regs *, regs, long, error_code) { machine_check_vector(regs, error_code); } --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -241,8 +241,7 @@ u32 kvm_read_and_reset_pf_reason(void) EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason); -dotraplinkage void -do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address) +IDTENTRY3(do_async_page_fault, struct pt_regs *, regs, unsigned long, error_code, unsigned long, address) { enum ctx_state prev_state; @@ -263,7 +262,6 @@ do_async_page_fault(struct pt_regs *regs break; } } -NOKPROBE_SYMBOL(do_async_page_fault); static void __init paravirt_ops_setup(void) { --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -281,7 +281,7 @@ static void do_error_trap(struct pt_regs #define IP ((void __user *)uprobe_get_trap_addr(regs)) #define DO_ERROR(trapnr, signr, sicode, addr, str, name) \ -dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ +IDTENTRY2(do_##name, struct pt_regs *, regs, long, error_code) \ { \ do_error_trap(regs, error_code, str, trapnr, signr, sicode, addr); \ } @@ -313,7 +313,7 @@ __visible void __noreturn handle_stack_o #ifdef CONFIG_X86_64 /* Runs on IST stack */ -dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2) +IDTENTRY3(do_double_fault, struct pt_regs *, regs, long, error_code, unsigned long, cr2) { static const char str[] = "double fault"; struct task_struct *tsk = current; @@ -428,7 +428,7 @@ dotraplinkage void do_double_fault(struc } #endif -dotraplinkage void do_bounds(struct pt_regs *regs, long error_code) +IDTENTRY2(do_bounds, struct pt_regs *, regs, long, error_code) { const struct mpx_bndcsr *bndcsr; @@ -514,8 +514,7 @@ dotraplinkage void do_bounds(struct pt_r do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, 0, NULL); } -dotraplinkage void -do_general_protection(struct pt_regs *regs, long error_code) +IDTENTRY2(do_general_protection, struct pt_regs *, regs, long, error_code) { const char *desc = "general protection fault"; struct task_struct *tsk; @@ -564,9 +563,8 @@ do_general_protection(struct pt_regs *re force_sig(SIGSEGV, tsk); } -NOKPROBE_SYMBOL(do_general_protection); -dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code) +IDTENTRY2(do_int3, struct pt_regs *, regs, long, error_code) { #ifdef CONFIG_DYNAMIC_FTRACE /* @@ -611,7 +609,6 @@ dotraplinkage void notrace do_int3(struc exit: ist_exit(regs); } -NOKPROBE_SYMBOL(do_int3); #ifdef CONFIG_X86_64 /* @@ -706,7 +703,7 @@ static bool is_sysenter_singlestep(struc * * May run on IST stack. */ -dotraplinkage void do_debug(struct pt_regs *regs, long error_code) +IDTENTRY2(do_debug, struct pt_regs *, regs, long, error_code) { struct task_struct *tsk = current; int user_icebp = 0; @@ -808,7 +805,6 @@ dotraplinkage void do_debug(struct pt_re exit: ist_exit(regs); } -NOKPROBE_SYMBOL(do_debug); /* * Note that we play around with the 'TS' bit in an attempt to get @@ -855,27 +851,24 @@ static void math_error(struct pt_regs *r (void __user *)uprobe_get_trap_addr(regs), task); } -dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code) +IDTENTRY2(do_coprocessor_error, struct pt_regs *, regs, long, error_code) { RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); math_error(regs, error_code, X86_TRAP_MF); } -dotraplinkage void -do_simd_coprocessor_error(struct pt_regs *regs, long error_code) +IDTENTRY2(do_simd_coprocessor_error, struct pt_regs *, regs, long, error_code) { RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); math_error(regs, error_code, X86_TRAP_XF); } -dotraplinkage void -do_spurious_interrupt_bug(struct pt_regs *regs, long error_code) +IDTENTRY2(do_spurious_interrupt_bug, struct pt_regs *, regs, long, error_code) { cond_local_irq_enable(regs); } -dotraplinkage void -do_device_not_available(struct pt_regs *regs, long error_code) +IDTENTRY2(do_device_not_available, struct pt_regs *, regs, long, error_code) { unsigned long cr0 = read_cr0(); @@ -906,10 +899,9 @@ do_device_not_available(struct pt_regs * die("unexpected #NM exception", regs, error_code); } } -NOKPROBE_SYMBOL(do_device_not_available); #ifdef CONFIG_X86_32 -dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) +IDTENTRY2(do_iret_error, struct pt_regs *, regs, long, error_code) { RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); local_irq_enable(); --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1542,8 +1542,7 @@ trace_page_fault_entries(struct pt_regs trace_page_fault_kernel(address, regs, error_code); } -dotraplinkage void -do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address) +IDTENTRY3(do_page_fault, struct pt_regs *, regs, unsigned long, error_code, unsigned long, address) { enum ctx_state prev_state; @@ -1552,4 +1551,3 @@ do_page_fault(struct pt_regs *regs, unsi __do_page_fault(regs, error_code, address); exception_exit(prev_state); } -NOKPROBE_SYMBOL(do_page_fault);
| |