lkml.org 
[lkml]   [2022]   [Sep]   [2]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[PATCH v2 26/59] x86/percpu: Move irq_stack variables next to current_task
    From: Thomas Gleixner <tglx@linutronix.de>

    Further extend struct pcpu_hot with the hard and soft irq stack
    pointers.

    Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    ---
    arch/x86/include/asm/current.h | 6 ++++++
    arch/x86/include/asm/irq_stack.h | 12 ++++++------
    arch/x86/include/asm/processor.h | 4 ----
    arch/x86/kernel/cpu/common.c | 3 ---
    arch/x86/kernel/dumpstack_32.c | 4 ++--
    arch/x86/kernel/dumpstack_64.c | 2 +-
    arch/x86/kernel/irq_32.c | 13 +++++--------
    arch/x86/kernel/irq_64.c | 6 +++---
    arch/x86/kernel/process_64.c | 2 +-
    9 files changed, 24 insertions(+), 28 deletions(-)

    --- a/arch/x86/include/asm/current.h
    +++ b/arch/x86/include/asm/current.h
    @@ -18,6 +18,12 @@ struct pcpu_hot {
    int preempt_count;
    int cpu_number;
    unsigned long top_of_stack;
    + void *hardirq_stack_ptr;
    +#ifdef CONFIG_X86_64
    + bool hardirq_stack_inuse;
    +#else
    + void *softirq_stack_ptr;
    +#endif
    };
    u8 pad[64];
    };
    --- a/arch/x86/include/asm/irq_stack.h
    +++ b/arch/x86/include/asm/irq_stack.h
    @@ -116,7 +116,7 @@
    ASM_CALL_ARG2

    #define call_on_irqstack(func, asm_call, argconstr...) \
    - call_on_stack(__this_cpu_read(hardirq_stack_ptr), \
    + call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr), \
    func, asm_call, argconstr)

    /* Macros to assert type correctness for run_*_on_irqstack macros */
    @@ -135,7 +135,7 @@
    * User mode entry and interrupt on the irq stack do not \
    * switch stacks. If from user mode the task stack is empty. \
    */ \
    - if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \
    + if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
    irq_enter_rcu(); \
    func(c_args); \
    irq_exit_rcu(); \
    @@ -146,9 +146,9 @@
    * places. Invoke the stack switch macro with the call \
    * sequence which matches the above direct invocation. \
    */ \
    - __this_cpu_write(hardirq_stack_inuse, true); \
    + __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
    call_on_irqstack(func, asm_call, constr); \
    - __this_cpu_write(hardirq_stack_inuse, false); \
    + __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
    } \
    }

    @@ -212,9 +212,9 @@
    */
    #define do_softirq_own_stack() \
    { \
    - __this_cpu_write(hardirq_stack_inuse, true); \
    + __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
    call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \
    - __this_cpu_write(hardirq_stack_inuse, false); \
    + __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
    }

    #endif
    --- a/arch/x86/include/asm/processor.h
    +++ b/arch/x86/include/asm/processor.h
    @@ -448,8 +448,6 @@ static inline unsigned long cpu_kernelmo
    return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
    }

    -DECLARE_PER_CPU(void *, hardirq_stack_ptr);
    -DECLARE_PER_CPU(bool, hardirq_stack_inuse);
    extern asmlinkage void ignore_sysret(void);

    /* Save actual FS/GS selectors and bases to current->thread */
    @@ -458,8 +456,6 @@ void current_save_fsgs(void);
    #ifdef CONFIG_STACKPROTECTOR
    DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
    #endif
    -DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
    -DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
    #endif /* !X86_64 */

    struct perf_event;
    --- a/arch/x86/kernel/cpu/common.c
    +++ b/arch/x86/kernel/cpu/common.c
    @@ -2012,9 +2012,6 @@ DEFINE_PER_CPU_FIRST(struct fixed_percpu
    fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
    EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);

    -DEFINE_PER_CPU(void *, hardirq_stack_ptr);
    -DEFINE_PER_CPU(bool, hardirq_stack_inuse);
    -
    static void wrmsrl_cstar(unsigned long val)
    {
    /*
    --- a/arch/x86/kernel/dumpstack_32.c
    +++ b/arch/x86/kernel/dumpstack_32.c
    @@ -37,7 +37,7 @@ const char *stack_type_name(enum stack_t

    static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)
    {
    - unsigned long *begin = (unsigned long *)this_cpu_read(hardirq_stack_ptr);
    + unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);
    unsigned long *end = begin + (THREAD_SIZE / sizeof(long));

    /*
    @@ -62,7 +62,7 @@ static bool in_hardirq_stack(unsigned lo

    static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)
    {
    - unsigned long *begin = (unsigned long *)this_cpu_read(softirq_stack_ptr);
    + unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.softirq_stack_ptr);
    unsigned long *end = begin + (THREAD_SIZE / sizeof(long));

    /*
    --- a/arch/x86/kernel/dumpstack_64.c
    +++ b/arch/x86/kernel/dumpstack_64.c
    @@ -134,7 +134,7 @@ static __always_inline bool in_exception

    static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info)
    {
    - unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr);
    + unsigned long *end = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);
    unsigned long *begin;

    /*
    --- a/arch/x86/kernel/irq_32.c
    +++ b/arch/x86/kernel/irq_32.c
    @@ -52,9 +52,6 @@ static inline int check_stack_overflow(v
    static inline void print_stack_overflow(void) { }
    #endif

    -DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
    -DEFINE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
    -
    static void call_on_stack(void *func, void *stack)
    {
    asm volatile("xchgl %%ebx,%%esp \n"
    @@ -77,7 +74,7 @@ static inline int execute_on_irq_stack(i
    u32 *isp, *prev_esp, arg1;

    curstk = (struct irq_stack *) current_stack();
    - irqstk = __this_cpu_read(hardirq_stack_ptr);
    + irqstk = __this_cpu_read(pcpu_hot.hardirq_stack_ptr);

    /*
    * this is where we switch to the IRQ stack. However, if we are
    @@ -115,7 +112,7 @@ int irq_init_percpu_irqstack(unsigned in
    int node = cpu_to_node(cpu);
    struct page *ph, *ps;

    - if (per_cpu(hardirq_stack_ptr, cpu))
    + if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
    return 0;

    ph = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER);
    @@ -127,8 +124,8 @@ int irq_init_percpu_irqstack(unsigned in
    return -ENOMEM;
    }

    - per_cpu(hardirq_stack_ptr, cpu) = page_address(ph);
    - per_cpu(softirq_stack_ptr, cpu) = page_address(ps);
    + per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = page_address(ph);
    + per_cpu(pcpu_hot.softirq_stack_ptr, cpu) = page_address(ps);
    return 0;
    }

    @@ -138,7 +135,7 @@ void do_softirq_own_stack(void)
    struct irq_stack *irqstk;
    u32 *isp, *prev_esp;

    - irqstk = __this_cpu_read(softirq_stack_ptr);
    + irqstk = __this_cpu_read(pcpu_hot.softirq_stack_ptr);

    /* build the stack frame on the softirq stack */
    isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
    --- a/arch/x86/kernel/irq_64.c
    +++ b/arch/x86/kernel/irq_64.c
    @@ -50,7 +50,7 @@ static int map_irq_stack(unsigned int cp
    return -ENOMEM;

    /* Store actual TOS to avoid adjustment in the hotpath */
    - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
    + per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
    return 0;
    }
    #else
    @@ -63,14 +63,14 @@ static int map_irq_stack(unsigned int cp
    void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);

    /* Store actual TOS to avoid adjustment in the hotpath */
    - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
    + per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
    return 0;
    }
    #endif

    int irq_init_percpu_irqstack(unsigned int cpu)
    {
    - if (per_cpu(hardirq_stack_ptr, cpu))
    + if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
    return 0;
    return map_irq_stack(cpu);
    }
    --- a/arch/x86/kernel/process_64.c
    +++ b/arch/x86/kernel/process_64.c
    @@ -562,7 +562,7 @@ void compat_start_thread(struct pt_regs
    int cpu = smp_processor_id();

    WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
    - this_cpu_read(hardirq_stack_inuse));
    + this_cpu_read(pcpu_hot.hardirq_stack_inuse));

    if (!test_thread_flag(TIF_NEED_FPU_LOAD))
    switch_fpu_prepare(prev_fpu, cpu);

    \
     
     \ /
      Last update: 2022-09-02 16:29    [W:4.069 / U:0.376 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site