lkml.org 
[lkml]   [2022]   [Dec]   [20]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
Patch in this message
/
From
Subject[PATCH 1/2] mm: vmalloc: Avoid a double lookup of freed VA in a tree
Date
When a VA is freed over a main path, for example by invoking
the vfree() function, a tree is accessed two times what is odd:

vfree():
__vunmap()
__find_vmap_area()
vm_remove_mappings()
remove_vm_area()
__find_vmap_area()

__find_vmap_area() are called two times. Fix it by introducing
a find_unlink_vmap_area() helper that finds and un-links a VA
from a tree.

Performance test results on a single CPU:

- fix_size_alloc_test loops: 1000000 avg: 476847 usec
- full_fit_alloc_test loops: 1000000 avg: 806746 usec
- long_busy_list_alloc_test loops: 1000000 avg: 13552093 usec
- random_size_alloc_test loops: 1000000 avg: 7441322 usec
- fix_align_alloc_test loops: 1000000 avg: 1411132 usec
All test took worker0=87650866284 cycles

- fix_size_alloc_test loops: 1000000 avg: 490713 usec
- full_fit_alloc_test loops: 1000000 avg: 579162 usec
- long_busy_list_alloc_test loops: 1000000 avg: 10485448 usec
- random_size_alloc_test loops: 1000000 avg: 5824449 usec
- fix_align_alloc_test loops: 1000000 avg: 984735 usec
All test took worker0=67952362802 cycles

Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
---
mm/vmalloc.c | 40 ++++++++++++++++++++++++++++------------
1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 9e30f0b39203..0fc38c36e0df 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -1825,9 +1825,14 @@ static void free_vmap_area_noflush(struct vmap_area *va)
unsigned long va_start = va->va_start;
unsigned long nr_lazy;

- spin_lock(&vmap_area_lock);
- unlink_va(va, &vmap_area_root);
- spin_unlock(&vmap_area_lock);
+ /*
+ * A free_vmap_block() is left. It is NOT a main free path.
+ */
+ if (!list_empty(&va->list)) {
+ spin_lock(&vmap_area_lock);
+ unlink_va(va, &vmap_area_root);
+ spin_unlock(&vmap_area_lock);
+ }

nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
PAGE_SHIFT, &vmap_lazy_nr);
@@ -1871,6 +1876,19 @@ struct vmap_area *find_vmap_area(unsigned long addr)
return va;
}

+static struct vmap_area *find_unlink_vmap_area(unsigned long addr)
+{
+ struct vmap_area *va;
+
+ spin_lock(&vmap_area_lock);
+ va = __find_vmap_area(addr, &vmap_area_root);
+ if (va)
+ unlink_va(va, &vmap_area_root);
+ spin_unlock(&vmap_area_lock);
+
+ return va;
+}
+
/*** Per cpu kva allocator ***/

/*
@@ -2236,7 +2254,7 @@ void vm_unmap_ram(const void *mem, unsigned int count)
return;
}

- va = find_vmap_area(addr);
+ va = find_unlink_vmap_area(addr);
BUG_ON(!va);
debug_check_no_locks_freed((void *)va->va_start,
(va->va_end - va->va_start));
@@ -2607,21 +2625,16 @@ struct vm_struct *remove_vm_area(const void *addr)

might_sleep();

- spin_lock(&vmap_area_lock);
- va = __find_vmap_area((unsigned long)addr, &vmap_area_root);
- if (va && va->vm) {
+ va = find_unlink_vmap_area((unsigned long) addr);
+ if (va) {
struct vm_struct *vm = va->vm;

- va->vm = NULL;
- spin_unlock(&vmap_area_lock);
-
kasan_free_module_shadow(vm);
free_unmap_vmap_area(va);

return vm;
}

- spin_unlock(&vmap_area_lock);
return NULL;
}

@@ -2690,6 +2703,7 @@ static void vm_remove_mappings(struct vm_struct *area, int deallocate_pages)
static void __vunmap(const void *addr, int deallocate_pages)
{
struct vm_struct *area;
+ struct vmap_area *va;

if (!addr)
return;
@@ -2698,7 +2712,9 @@ static void __vunmap(const void *addr, int deallocate_pages)
addr))
return;

- area = find_vm_area(addr);
+ va = find_unlink_vmap_area((unsigned long)addr);
+ area = va->vm;
+
if (unlikely(!area)) {
WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
addr);
--
2.30.2
\
 
 \ /
  Last update: 2023-03-26 23:16    [W:0.053 / U:0.252 seconds]
©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site