lkml.org 
[lkml]   [2021]   [Nov]   [9]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[PATCH 2/2] jump_label: refine placement of static_keys
Date
From: Eric Dumazet <edumazet@google.com>

With CONFIG_JUMP_LABEL=y, "struct static_key" content is only
used for the control path.

Marking them __read_mostly is only needed when CONFIG_JUMP_LABEL=n.
Otherwise we place them out of the way to increase data locality.

This patch adds __static_key to centralize this new policy.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
arch/x86/kvm/lapic.c | 4 ++--
arch/x86/kvm/x86.c | 2 +-
include/linux/jump_label.h | 25 +++++++++++++++++--------
kernel/events/core.c | 2 +-
kernel/sched/fair.c | 2 +-
net/core/dev.c | 8 ++++----
net/netfilter/core.c | 2 +-
net/netfilter/x_tables.c | 2 +-
8 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index d6ac32f3f650c066733ff9afe96ece36bca01523..a4e8ec38cd09aca2969e71242bced1d95e9e9c1f 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -91,8 +91,8 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}

-__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
-__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
+DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
+DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);

static inline int apic_enabled(struct kvm_lapic *apic)
{
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c1c4e2b05a63abd53e01d0671f740a3b900d98e6..051236fedee39e197d6795fc62bdb29849c7ada2 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11204,7 +11204,7 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
}

-__read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
+DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);

void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 48b9b2a82767d74ec95f102ab1470f5fa79cc97d..13ad61a845a30a91c9f18eabff7ff93cac4028e2 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -106,10 +106,19 @@ struct static_key {
};
};

+/*
+ * With CONFIG_JUMP_LABEL, "struct static_key" are not used in fast path.
+ * We can put them in a separate section to increase data locality.
+ */
+#define __static_key __section(".data.unlikely")
+
#else
struct static_key {
atomic_t enabled;
};
+
+#define __static_key __read_mostly
+
#endif /* CONFIG_JUMP_LABEL */
#endif /* __ASSEMBLY__ */

@@ -367,7 +376,7 @@ struct static_key_false {
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }

#define DEFINE_STATIC_KEY_TRUE(name) \
- struct static_key_true name = STATIC_KEY_TRUE_INIT
+ struct static_key_true __static_key name = STATIC_KEY_TRUE_INIT

#define DEFINE_STATIC_KEY_TRUE_RO(name) \
struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
@@ -376,7 +385,7 @@ struct static_key_false {
extern struct static_key_true name

#define DEFINE_STATIC_KEY_FALSE(name) \
- struct static_key_false name = STATIC_KEY_FALSE_INIT
+ struct static_key_false __static_key name = STATIC_KEY_FALSE_INIT

#define DEFINE_STATIC_KEY_FALSE_RO(name) \
struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
@@ -384,14 +393,14 @@ struct static_key_false {
#define DECLARE_STATIC_KEY_FALSE(name) \
extern struct static_key_false name

-#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
- struct static_key_true name[count] = { \
- [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
+#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
+ struct static_key_true __static_key name[count] = { \
+ [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
}

-#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
- struct static_key_false name[count] = { \
- [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
+#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
+ struct static_key_false __static_key name[count] = { \
+ [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
}

#define _DEFINE_STATIC_KEY_1(name) DEFINE_STATIC_KEY_TRUE(name)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index f2253ea729a2732b0261f32280f77f1d1a2ebefb..5cd140b3a489bb51c2536bd5f869290bf8395e7b 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -9600,7 +9600,7 @@ static int swevent_hlist_get(void)
return err;
}

-struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key __static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];

static void sw_perf_event_destroy(struct perf_event *event)
{
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 13950beb01a251bfb65bf5b75952e3dea9cbc62b..de9b18a244cadd3b47a2ec4ed4c000a034a7fb96 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4586,7 +4586,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH

#ifdef CONFIG_JUMP_LABEL
-static struct static_key __cfs_bandwidth_used;
+static struct static_key __static_key __cfs_bandwidth_used;

static inline bool cfs_bandwidth_used(void)
{
diff --git a/net/core/dev.c b/net/core/dev.c
index edeb811c454e6caf4298aae723237aea2c363c3c..21d53ca2730d1abb5dc3103ccca053e1cdf676b7 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2390,8 +2390,8 @@ int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
EXPORT_SYMBOL(netdev_txq_to_tc);

#ifdef CONFIG_XPS
-static struct static_key xps_needed __read_mostly;
-static struct static_key xps_rxqs_needed __read_mostly;
+static struct static_key __static_key xps_needed;
+static struct static_key __static_key xps_rxqs_needed;
static DEFINE_MUTEX(xps_map_mutex);
#define xmap_dereference(P) \
rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
@@ -4362,9 +4362,9 @@ EXPORT_SYMBOL(rps_sock_flow_table);
u32 rps_cpu_mask __read_mostly;
EXPORT_SYMBOL(rps_cpu_mask);

-struct static_key_false rps_needed __read_mostly;
+struct static_key_false __static_key rps_needed;
EXPORT_SYMBOL(rps_needed);
-struct static_key_false rfs_needed __read_mostly;
+struct static_key_false __static_key rfs_needed;
EXPORT_SYMBOL(rfs_needed);

static struct rps_dev_flow *
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index 6dec9cd395f1570c845346b6956ba2586a4235d6..3fd26326a55d95c28ab634eb3caf5a4a851ad20c 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -35,7 +35,7 @@ DEFINE_PER_CPU(bool, nf_skb_duplicated);
EXPORT_SYMBOL_GPL(nf_skb_duplicated);

#ifdef CONFIG_JUMP_LABEL
-struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key __static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif

diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index 25524e3933496098f68748286e56839454ee842c..aaf78481b04a0decaa09b03b9f69fa3737740cf4 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -1319,7 +1319,7 @@ EXPORT_SYMBOL_GPL(xt_compat_unlock);
DEFINE_PER_CPU(seqcount_t, xt_recseq);
EXPORT_PER_CPU_SYMBOL_GPL(xt_recseq);

-struct static_key xt_tee_enabled __read_mostly;
+struct static_key __static_key xt_tee_enabled;
EXPORT_SYMBOL_GPL(xt_tee_enabled);

static int xt_jumpstack_alloc(struct xt_table_info *i)
--
2.34.0.rc0.344.g81b53c2807-goog
\
 
 \ /
  Last update: 2021-11-10 02:23    [W:0.064 / U:0.068 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site