Messages in this thread Patch in this message | ![/](/images/icornerl.gif) | | Date | Thu, 11 Sep 2008 20:24:07 +0900 | From | KAMEZAWA Hiroyuki <> | Subject | [RFC] [PATCH 9/9] memcg: percpu page cgroup lookup cache |
| |
Use per-cpu cache for fast access to page_cgroup. This patch is for making fastpath faster.
Because page_cgroup is accessed when the page is allocated/freed, we can assume several of continuous page_cgroup will be accessed soon. (If not interleaved on NUMA...but in such case, alloc/free itself is slow.)
We cache some set of page_cgroup's base pointer on per-cpu area and use it when we hit.
TODO: - memory/cpu hotplug support.
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
--- mm/page_cgroup.c | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-)
Index: mmtom-2.6.27-rc5+/mm/page_cgroup.c =================================================================== --- mmtom-2.6.27-rc5+.orig/mm/page_cgroup.c +++ mmtom-2.6.27-rc5+/mm/page_cgroup.c @@ -57,14 +57,26 @@ static int pcg_hashmask __read_mostly; #define PCG_HASHMASK (pcg_hashmask) #define PCG_HASHSIZE (1 << pcg_hashshift) +#define PCG_CACHE_MAX_SLOT (32) +#define PCG_CACHE_MASK (PCG_CACHE_MAX_SLOT - 1) +struct percpu_page_cgroup_cache { + struct { + unsigned long index; + struct page_cgroup *base; + } slots[PCG_CACHE_MAX_SLOT]; +}; +DEFINE_PER_CPU(struct percpu_page_cgroup_cache, pcg_cache); + int pcg_hashfun(unsigned long index) { return hash_long(index, pcg_hashshift); } -struct page_cgroup *lookup_page_cgroup(unsigned long pfn) +noinline static struct page_cgroup * +__lookup_page_cgroup(struct percpu_page_cgroup_cache *pcc,unsigned long pfn) { unsigned long index = pfn >> ENTS_PER_CHUNK_SHIFT; + int s = index & PCG_CACHE_MASK; struct pcg_hash *ent; struct pcg_hash_head *head; struct hlist_node *node; @@ -77,6 +89,8 @@ struct page_cgroup *lookup_page_cgroup(u hlist_for_each_entry(ent, node, &head->head, node) { if (ent->index == index) { pc = ent->map + pfn; + pcc->slots[s].index = ent->index; + pcc->slots[s].base = ent->map; break; } } @@ -84,6 +98,22 @@ struct page_cgroup *lookup_page_cgroup(u return pc; } +struct page_cgroup *lookup_page_cgroup(unsigned long pfn) +{ + unsigned long index = pfn >> ENTS_PER_CHUNK_SHIFT; + int hnum = (pfn >> ENTS_PER_CHUNK_SHIFT) & PCG_CACHE_MASK; + struct percpu_page_cgroup_cache *pcc; + struct page_cgroup *ret; + + pcc = &get_cpu_var(pcg_cache); + if (likely(pcc->slots[hnum].index == index)) + ret = pcc->slots[hnum].base + pfn; + else + ret = __lookup_page_cgroup(pcc, pfn); + put_cpu_var(pcg_cache); + return ret; +} + static void __meminit alloc_page_cgroup(int node, unsigned long index) { struct pcg_hash *ent; @@ -124,12 +154,23 @@ static void __meminit alloc_page_cgroup( return; } +void clear_page_cgroup_cache_pcg(int cpu) +{ + struct percpu_page_cgroup_cache *pcc; + int i; + + pcc = &per_cpu(pcg_cache, cpu); + for (i = 0; i < PCG_CACHE_MAX_SLOT; i++) { + pcc->slots[i].index = -1; + pcc->slots[i].base = NULL; + } +} /* Called From mem_cgroup's initilization */ void __init page_cgroup_init(void) { struct pcg_hash_head *head; - int node, i; + int node, cpu, i; unsigned long start, pfn, end, index, offset; long default_pcg_hash_size; @@ -174,5 +215,7 @@ void __init page_cgroup_init(void) } } } + for_each_possible_cpu(cpu) + clear_page_cgroup_cache_pcg(cpu); return; }
| ![\](/images/icornerr.gif) |