lkml.org 
[lkml]   [2019]   [Mar]   [7]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[PATCH 2/5] mm: prepare to premature release of per-node lruvec_stat_cpu
Date
Similar to the memcg's vmstats_percpu, per-memcg per-node stats
consists of percpu- and atomic counterparts, and we do expect
that both coexist during the whole life-cycle of the memcg.

To prepare for a premature release of percpu per-node data,
let's pretend that lruvec_stat_cpu is a rcu-protected pointer,
which can be NULL. This patch adds corresponding checks whenever
required.

Signed-off-by: Roman Gushchin <guro@fb.com>
---
include/linux/memcontrol.h | 21 +++++++++++++++------
mm/memcontrol.c | 11 +++++++++--
2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 05ca77767c6a..8ac04632002a 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -126,7 +126,7 @@ struct memcg_shrinker_map {
struct mem_cgroup_per_node {
struct lruvec lruvec;

- struct lruvec_stat __percpu *lruvec_stat_cpu;
+ struct lruvec_stat __rcu /* __percpu */ *lruvec_stat_cpu;
atomic_long_t lruvec_stat[NR_VM_NODE_STAT_ITEMS];

unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
@@ -682,6 +682,7 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
static inline void __mod_lruvec_state(struct lruvec *lruvec,
enum node_stat_item idx, int val)
{
+ struct lruvec_stat __percpu *lruvec_stat_cpu;
struct mem_cgroup_per_node *pn;
long x;

@@ -697,12 +698,20 @@ static inline void __mod_lruvec_state(struct lruvec *lruvec,
__mod_memcg_state(pn->memcg, idx, val);

/* Update lruvec */
- x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
- if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
- atomic_long_add(x, &pn->lruvec_stat[idx]);
- x = 0;
+ rcu_read_lock();
+ lruvec_stat_cpu = (struct lruvec_stat __percpu *)
+ rcu_dereference(pn->lruvec_stat_cpu);
+ if (likely(lruvec_stat_cpu)) {
+ x = val + __this_cpu_read(lruvec_stat_cpu->count[idx]);
+ if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
+ atomic_long_add(x, &pn->lruvec_stat[idx]);
+ x = 0;
+ }
+ __this_cpu_write(lruvec_stat_cpu->count[idx], x);
+ } else {
+ atomic_long_add(val, &pn->lruvec_stat[idx]);
}
- __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
+ rcu_read_unlock();
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 803c772f354b..8f3cac02221a 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2122,6 +2122,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
static int memcg_hotplug_cpu_dead(unsigned int cpu)
{
struct memcg_vmstats_percpu __percpu *vmstats_percpu;
+ struct lruvec_stat __percpu *lruvec_stat_cpu;
struct memcg_stock_pcp *stock;
struct mem_cgroup *memcg;

@@ -2152,7 +2153,12 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu)
struct mem_cgroup_per_node *pn;

pn = mem_cgroup_nodeinfo(memcg, nid);
- x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
+
+ lruvec_stat_cpu = (struct lruvec_stat __percpu*)
+ rcu_dereference(pn->lruvec_stat_cpu);
+ if (!lruvec_stat_cpu)
+ continue;
+ x = this_cpu_xchg(lruvec_stat_cpu->count[i], 0);
if (x)
atomic_long_add(x, &pn->lruvec_stat[i]);
}
@@ -4430,7 +4436,8 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
if (!pn)
return 1;

- pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
+ rcu_assign_pointer(pn->lruvec_stat_cpu,
+ alloc_percpu(struct lruvec_stat));
if (!pn->lruvec_stat_cpu) {
kfree(pn);
return 1;
--
2.20.1
\
 
 \ /
  Last update: 2019-03-08 00:01    [W:0.144 / U:0.128 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site