Messages in this thread Patch in this message | | | Date | Fri, 9 Apr 2021 15:13:19 +0200 | From | Peter Zijlstra <> | Subject | Re: static_branch/jump_label vs branch merging |
| |
On Fri, Apr 09, 2021 at 03:01:50PM +0200, Peter Zijlstra wrote: > On Fri, Apr 09, 2021 at 02:03:46PM +0200, Peter Zijlstra wrote: > > On Fri, Apr 09, 2021 at 07:55:42AM -0400, David Malcolm wrote: > > > > Sorry if this is a dumb question, but does the function attribute: > > > __attribute__ ((pure)) > > > help here? It's meant to allow multiple calls to a predicate to be > > > merged - though I'd be nervous of using it here, the predicate isn't > > > 100% pure, since AIUI the whole point of what you've built is for > > > predicates that very rarely change - but can change occasionally. > > > > I actually tried that, but it doesn't seem to work. Given the function > > arguments are all compile time constants it should DTRT AFAICT, but > > alas. > > FWIW, I tried the below patch and GCC-10.2.1 on current tip/master.
I also just tried __attribute__((__const__)), which is stronger still than __pure__ and that's also not working :/
I then also tried to replace the __buildin_types_compatible_p() magic in static_branch_unlikely() with _Generic(), but still no joy.
--- diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index 610a05374c02..f14c6863b911 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -14,7 +14,7 @@ #include <linux/stringify.h> #include <linux/types.h> -static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) +static __always_inline __attribute_const__ bool arch_static_branch(struct static_key * const key, const bool branch) { asm_volatile_goto("1:" ".byte " __stringify(BYTES_NOP5) "\n\t" @@ -30,7 +30,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, co return true; } -static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) +static __always_inline __attribute_const__ bool arch_static_branch_jump(struct static_key * const key, const bool branch) { asm_volatile_goto("1:" ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 05f5554d860f..2c250d8b9a02 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -195,12 +195,12 @@ struct module; #define JUMP_TYPE_LINKED 2UL #define JUMP_TYPE_MASK 3UL -static __always_inline bool static_key_false(struct static_key *key) +static __always_inline __attribute_const__ bool static_key_false(struct static_key * const key) { return arch_static_branch(key, false); } -static __always_inline bool static_key_true(struct static_key *key) +static __always_inline __attribute_const__ bool static_key_true(struct static_key * const key) { return !arch_static_branch(key, true); } @@ -466,6 +466,18 @@ extern bool ____wrong_branch_error(void); * See jump_label_type() / jump_label_init_type(). */ +#if 1 + +#define static_branch_likely(x) _Generic(*(x), \ + struct static_key_true: !arch_static_branch(&(x)->key, true), \ + struct static_key_false: !arch_static_branch_jump(&(x)->key, true)) + +#define static_branch_unlikely(x) _Generic(*(x), \ + struct static_key_true: arch_static_branch_jump(&(x)->key, false), \ + struct static_key_false: arch_static_branch(&(x)->key, false)) + +#else + #define static_branch_likely(x) \ ({ \ bool branch; \ @@ -490,6 +502,8 @@ extern bool ____wrong_branch_error(void); unlikely_notrace(branch); \ }) +#endif + #else /* !CONFIG_JUMP_LABEL */ #define static_branch_likely(x) likely_notrace(static_key_enabled(&(x)->key))
| |