Messages in this thread Patch in this message | | | From | Eric Dumazet <> | Subject | [PATCH 2/2] jump_label: refine placement of static_keys | Date | Tue, 9 Nov 2021 17:09:06 -0800 |
| |
From: Eric Dumazet <edumazet@google.com>
With CONFIG_JUMP_LABEL=y, "struct static_key" content is only used for the control path.
Marking them __read_mostly is only needed when CONFIG_JUMP_LABEL=n. Otherwise we place them out of the way to increase data locality.
This patch adds __static_key to centralize this new policy.
Signed-off-by: Eric Dumazet <edumazet@google.com> --- arch/x86/kvm/lapic.c | 4 ++-- arch/x86/kvm/x86.c | 2 +- include/linux/jump_label.h | 25 +++++++++++++++++-------- kernel/events/core.c | 2 +- kernel/sched/fair.c | 2 +- net/core/dev.c | 8 ++++---- net/netfilter/core.c | 2 +- net/netfilter/x_tables.c | 2 +- 8 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index d6ac32f3f650c066733ff9afe96ece36bca01523..a4e8ec38cd09aca2969e71242bced1d95e9e9c1f 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -91,8 +91,8 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap) return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); } -__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ); -__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ); +DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ); +DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ); static inline int apic_enabled(struct kvm_lapic *apic) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c1c4e2b05a63abd53e01d0671f740a3b900d98e6..051236fedee39e197d6795fc62bdb29849c7ada2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11204,7 +11204,7 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0; } -__read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); +DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu); void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 48b9b2a82767d74ec95f102ab1470f5fa79cc97d..13ad61a845a30a91c9f18eabff7ff93cac4028e2 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -106,10 +106,19 @@ struct static_key { }; }; +/* + * With CONFIG_JUMP_LABEL, "struct static_key" are not used in fast path. + * We can put them in a separate section to increase data locality. + */ +#define __static_key __section(".data.unlikely") + #else struct static_key { atomic_t enabled; }; + +#define __static_key __read_mostly + #endif /* CONFIG_JUMP_LABEL */ #endif /* __ASSEMBLY__ */ @@ -367,7 +376,7 @@ struct static_key_false { #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, } #define DEFINE_STATIC_KEY_TRUE(name) \ - struct static_key_true name = STATIC_KEY_TRUE_INIT + struct static_key_true __static_key name = STATIC_KEY_TRUE_INIT #define DEFINE_STATIC_KEY_TRUE_RO(name) \ struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT @@ -376,7 +385,7 @@ struct static_key_false { extern struct static_key_true name #define DEFINE_STATIC_KEY_FALSE(name) \ - struct static_key_false name = STATIC_KEY_FALSE_INIT + struct static_key_false __static_key name = STATIC_KEY_FALSE_INIT #define DEFINE_STATIC_KEY_FALSE_RO(name) \ struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT @@ -384,14 +393,14 @@ struct static_key_false { #define DECLARE_STATIC_KEY_FALSE(name) \ extern struct static_key_false name -#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \ - struct static_key_true name[count] = { \ - [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \ +#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \ + struct static_key_true __static_key name[count] = { \ + [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \ } -#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \ - struct static_key_false name[count] = { \ - [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \ +#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \ + struct static_key_false __static_key name[count] = { \ + [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \ } #define _DEFINE_STATIC_KEY_1(name) DEFINE_STATIC_KEY_TRUE(name) diff --git a/kernel/events/core.c b/kernel/events/core.c index f2253ea729a2732b0261f32280f77f1d1a2ebefb..5cd140b3a489bb51c2536bd5f869290bf8395e7b 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9600,7 +9600,7 @@ static int swevent_hlist_get(void) return err; } -struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; +struct static_key __static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; static void sw_perf_event_destroy(struct perf_event *event) { diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 13950beb01a251bfb65bf5b75952e3dea9cbc62b..de9b18a244cadd3b47a2ec4ed4c000a034a7fb96 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4586,7 +4586,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) #ifdef CONFIG_CFS_BANDWIDTH #ifdef CONFIG_JUMP_LABEL -static struct static_key __cfs_bandwidth_used; +static struct static_key __static_key __cfs_bandwidth_used; static inline bool cfs_bandwidth_used(void) { diff --git a/net/core/dev.c b/net/core/dev.c index edeb811c454e6caf4298aae723237aea2c363c3c..21d53ca2730d1abb5dc3103ccca053e1cdf676b7 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2390,8 +2390,8 @@ int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) EXPORT_SYMBOL(netdev_txq_to_tc); #ifdef CONFIG_XPS -static struct static_key xps_needed __read_mostly; -static struct static_key xps_rxqs_needed __read_mostly; +static struct static_key __static_key xps_needed; +static struct static_key __static_key xps_rxqs_needed; static DEFINE_MUTEX(xps_map_mutex); #define xmap_dereference(P) \ rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) @@ -4362,9 +4362,9 @@ EXPORT_SYMBOL(rps_sock_flow_table); u32 rps_cpu_mask __read_mostly; EXPORT_SYMBOL(rps_cpu_mask); -struct static_key_false rps_needed __read_mostly; +struct static_key_false __static_key rps_needed; EXPORT_SYMBOL(rps_needed); -struct static_key_false rfs_needed __read_mostly; +struct static_key_false __static_key rfs_needed; EXPORT_SYMBOL(rfs_needed); static struct rps_dev_flow * diff --git a/net/netfilter/core.c b/net/netfilter/core.c index 6dec9cd395f1570c845346b6956ba2586a4235d6..3fd26326a55d95c28ab634eb3caf5a4a851ad20c 100644 --- a/net/netfilter/core.c +++ b/net/netfilter/core.c @@ -35,7 +35,7 @@ DEFINE_PER_CPU(bool, nf_skb_duplicated); EXPORT_SYMBOL_GPL(nf_skb_duplicated); #ifdef CONFIG_JUMP_LABEL -struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; +struct static_key __static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS]; EXPORT_SYMBOL(nf_hooks_needed); #endif diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 25524e3933496098f68748286e56839454ee842c..aaf78481b04a0decaa09b03b9f69fa3737740cf4 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -1319,7 +1319,7 @@ EXPORT_SYMBOL_GPL(xt_compat_unlock); DEFINE_PER_CPU(seqcount_t, xt_recseq); EXPORT_PER_CPU_SYMBOL_GPL(xt_recseq); -struct static_key xt_tee_enabled __read_mostly; +struct static_key __static_key xt_tee_enabled; EXPORT_SYMBOL_GPL(xt_tee_enabled); static int xt_jumpstack_alloc(struct xt_table_info *i) -- 2.34.0.rc0.344.g81b53c2807-goog
| |