lkml.org 
[lkml]   [2022]   [Dec]   [20]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[PATCH RFC 1/2] mm: kmem: optimize get_obj_cgroup_from_current()
Date
Manually inline memcg_kmem_bypass() and active_memcg() to speed up
get_obj_cgroup_from_current() by avoiding duplicate in_task() checks
and active_memcg() readings.

Also add a likely() macro to __get_obj_cgroup_from_memcg():
obj_cgroup_tryget() should succeed at almost all times except
a very unlikely race with the memcg deletion path.

Signed-off-by: Roman Gushchin <roman.gushchin@linux.dev>
---
mm/memcontrol.c | 34 ++++++++++++++--------------------
1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index bafd3cde4507..82828c51d2ea 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1047,19 +1047,6 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
}
EXPORT_SYMBOL(get_mem_cgroup_from_mm);

-static __always_inline bool memcg_kmem_bypass(void)
-{
- /* Allow remote memcg charging from any context. */
- if (unlikely(active_memcg()))
- return false;
-
- /* Memcg to charge can't be determined. */
- if (!in_task() || !current->mm || (current->flags & PF_KTHREAD))
- return true;
-
- return false;
-}
-
/**
* mem_cgroup_iter - iterate over memory cgroup hierarchy
* @root: hierarchy root
@@ -3004,7 +2991,7 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)

for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
objcg = rcu_dereference(memcg->objcg);
- if (objcg && obj_cgroup_tryget(objcg))
+ if (likely(objcg && obj_cgroup_tryget(objcg)))
break;
objcg = NULL;
}
@@ -3013,16 +3000,23 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)

__always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
{
- struct obj_cgroup *objcg = NULL;
struct mem_cgroup *memcg;
+ struct obj_cgroup *objcg;

- if (memcg_kmem_bypass())
- return NULL;
+ if (in_task()) {
+ memcg = current->active_memcg;
+
+ /* Memcg to charge can't be determined. */
+ if (likely(!memcg) && (!current->mm || (current->flags & PF_KTHREAD)))
+ return NULL;
+ } else {
+ memcg = this_cpu_read(int_active_memcg);
+ if (likely(!memcg))
+ return NULL;
+ }

rcu_read_lock();
- if (unlikely(active_memcg()))
- memcg = active_memcg();
- else
+ if (!memcg)
memcg = mem_cgroup_from_task(current);
objcg = __get_obj_cgroup_from_memcg(memcg);
rcu_read_unlock();
--
2.39.0
\
 
 \ /
  Last update: 2023-03-26 23:16    [W:0.086 / U:1.416 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site