lkml.org 
[lkml]   [2022]   [Sep]   [15]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[PATCH v3 52/59] static_call: Add call depth tracking support
    From: Peter Zijlstra <peterz@infradead.org>

    When indirect calls are switched to direct calls then it has to be ensured
    that the call target is not the function, but the call thunk when call
    depth tracking is enabled. But static calls are available before call
    thunks have been set up.

    Ensure a second run through the static call patching code after call thunks
    have been created. When call thunks are not enabled this has no side
    effects.

    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    ---
    arch/x86/include/asm/alternative.h | 5 +++++
    arch/x86/kernel/callthunks.c | 18 ++++++++++++++++++
    arch/x86/kernel/static_call.c | 1 +
    include/linux/static_call.h | 2 ++
    kernel/static_call_inline.c | 23 ++++++++++++++++++-----
    5 files changed, 44 insertions(+), 5 deletions(-)

    --- a/arch/x86/include/asm/alternative.h
    +++ b/arch/x86/include/asm/alternative.h
    @@ -91,11 +91,16 @@ struct callthunk_sites {
    extern void callthunks_patch_builtin_calls(void);
    extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
    struct module *mod);
    +extern void *callthunks_translate_call_dest(void *dest);
    #else
    static __always_inline void callthunks_patch_builtin_calls(void) {}
    static __always_inline void
    callthunks_patch_module_calls(struct callthunk_sites *sites,
    struct module *mod) {}
    +static __always_inline void *callthunks_translate_call_dest(void *dest)
    +{
    + return dest;
    +}
    #endif

    #ifdef CONFIG_SMP
    --- a/arch/x86/kernel/callthunks.c
    +++ b/arch/x86/kernel/callthunks.c
    @@ -6,6 +6,7 @@
    #include <linux/kallsyms.h>
    #include <linux/memory.h>
    #include <linux/moduleloader.h>
    +#include <linux/static_call.h>

    #include <asm/alternative.h>
    #include <asm/asm-offsets.h>
    @@ -270,10 +271,27 @@ void __init callthunks_patch_builtin_cal
    pr_info("Setting up call depth tracking\n");
    mutex_lock(&text_mutex);
    callthunks_setup(&cs, &builtin_coretext);
    + static_call_force_reinit();
    thunks_initialized = true;
    mutex_unlock(&text_mutex);
    }

    +void *callthunks_translate_call_dest(void *dest)
    +{
    + void *target;
    +
    + lockdep_assert_held(&text_mutex);
    +
    + if (!thunks_initialized || skip_addr(dest))
    + return dest;
    +
    + if (!is_coretext(NULL, dest))
    + return dest;
    +
    + target = patch_dest(dest, false);
    + return target ? : dest;
    +}
    +
    #ifdef CONFIG_MODULES
    void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
    struct module *mod)
    --- a/arch/x86/kernel/static_call.c
    +++ b/arch/x86/kernel/static_call.c
    @@ -34,6 +34,7 @@ static void __ref __static_call_transfor

    switch (type) {
    case CALL:
    + func = callthunks_translate_call_dest(func);
    code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
    if (func == &__static_call_return0) {
    emulate = code;
    --- a/include/linux/static_call.h
    +++ b/include/linux/static_call.h
    @@ -162,6 +162,8 @@ extern void arch_static_call_transform(v

    extern int __init static_call_init(void);

    +extern void static_call_force_reinit(void);
    +
    struct static_call_mod {
    struct static_call_mod *next;
    struct module *mod; /* for vmlinux, mod == NULL */
    --- a/kernel/static_call_inline.c
    +++ b/kernel/static_call_inline.c
    @@ -15,7 +15,18 @@ extern struct static_call_site __start_s
    extern struct static_call_tramp_key __start_static_call_tramp_key[],
    __stop_static_call_tramp_key[];

    -static bool static_call_initialized;
    +static int static_call_initialized;
    +
    +/*
    + * Must be called before early_initcall() to be effective.
    + */
    +void static_call_force_reinit(void)
    +{
    + if (WARN_ON_ONCE(!static_call_initialized))
    + return;
    +
    + static_call_initialized++;
    +}

    /* mutex to protect key modules/sites */
    static DEFINE_MUTEX(static_call_mutex);
    @@ -475,7 +486,8 @@ int __init static_call_init(void)
    {
    int ret;

    - if (static_call_initialized)
    + /* See static_call_force_reinit(). */
    + if (static_call_initialized == 1)
    return 0;

    cpus_read_lock();
    @@ -490,11 +502,12 @@ int __init static_call_init(void)
    BUG();
    }

    - static_call_initialized = true;
    -
    #ifdef CONFIG_MODULES
    - register_module_notifier(&static_call_module_nb);
    + if (!static_call_initialized)
    + register_module_notifier(&static_call_module_nb);
    #endif
    +
    + static_call_initialized = 1;
    return 0;
    }
    early_initcall(static_call_init);

    \
     
     \ /
      Last update: 2022-09-15 13:42    [W:4.117 / U:0.672 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site