mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm/vma: extract the gathering of vmas from do_vmi_align_munmap()

Browse source 
Create vmi_gather_munmap_vmas() to handle the gathering of vmas into a
detached maple tree for removal later.  Part of the gathering is the
splitting of vmas that span the boundary.

Link: https://lkml.kernel.org/r/20240830040101.822209-5-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Bert Karwatzki <spasswolf@web.de>
Cc: Jeff Xu <jeffxu@chromium.org>
Cc: Jiri Olsa <olsajiri@gmail.com>
Cc: Kees Cook <kees@kernel.org>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Mark Brown <broonie@kernel.org>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: "Paul E. McKenney" <paulmck@kernel.org>
Cc: Paul Moore <paul@paul-moore.com>
Cc: Sidhartha Kumar <sidhartha.kumar@oracle.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Liam R. Howlett 2024-08-30 00:00:44 -04:00 committed by Andrew Morton
parent 01cf21e9e1
commit 6898c9039b
1 changed files with 62 additions and 33 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

95
mm/vma.c
View file

@ -737,32 +737,30 @@ vmi_complete_munmap_vmas(struct vma_iterator *vmi, struct vm_area_struct *vma,
} }
/* /*
* do_vmi_align_munmap() - munmap the aligned region from @start to @end. * vmi_gather_munmap_vmas() - Put all VMAs within a range into a maple tree
* for removal at a later date. Handles splitting first and last if necessary
* and marking the vmas as isolated.
*
* @vmi: The vma iterator * @vmi: The vma iterator
* @vma: The starting vm_area_struct * @vma: The starting vm_area_struct
* @mm: The mm_struct * @mm: The mm_struct
* @start: The aligned start address to munmap. * @start: The aligned start address to munmap.
* @end: The aligned end address to munmap. * @end: The aligned end address to munmap.
* @uf: The userfaultfd list_head * @uf: The userfaultfd list_head
* @unlock: Set to true to drop the mmap_lock. unlocking only happens on * @mas_detach: The maple state tracking the detached tree
* success. * @locked_vm: a pointer to store the VM_LOCKED pages count.
* *
* Return: 0 on success and drops the lock if so directed, error and leaves the * Return: 0 on success, -EPERM on mseal vmas, -ENOMEM otherwise
* lock held otherwise.
*/ */
int static int
do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, vmi_gather_munmap_vmas(struct vma_iterator *vmi, struct vm_area_struct *vma,
struct mm_struct *mm, unsigned long start, struct mm_struct *mm, unsigned long start,
unsigned long end, struct list_head *uf, bool unlock) unsigned long end, struct list_head *uf,
struct ma_state *mas_detach, unsigned long *locked_vm)
{ {
struct vm_area_struct *next = NULL; struct vm_area_struct *next = NULL;
struct maple_tree mt_detach;
int count = 0; int count = 0;
int error = -ENOMEM; int error = -ENOMEM;
unsigned long locked_vm = 0;
MA_STATE(mas_detach, &mt_detach, 0, 0);
mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
mt_on_stack(mt_detach);
/* /*
* If we need to split any vma, do it now to save pain later. * If we need to split any vma, do it now to save pain later.
@ -789,8 +787,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
goto start_split_failed; goto start_split_failed;
} }
error = __split_vma(vmi, vma, start, 1); if (__split_vma(vmi, vma, start, 1))
if (error)
goto start_split_failed; goto start_split_failed;
} }
@ -807,20 +804,18 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
/* Does it split the end? */ /* Does it split the end? */
if (next->vm_end > end) { if (next->vm_end > end) {
error = __split_vma(vmi, next, end, 0); if (__split_vma(vmi, next, end, 0))
if (error)
goto end_split_failed; goto end_split_failed;
} }
vma_start_write(next); vma_start_write(next);
mas_set(&mas_detach, count); mas_set(mas_detach, count++);
error = mas_store_gfp(&mas_detach, next, GFP_KERNEL); if (mas_store_gfp(mas_detach, next, GFP_KERNEL))
if (error)
goto munmap_gather_failed; goto munmap_gather_failed;
vma_mark_detached(next, true); vma_mark_detached(next, true);
if (next->vm_flags & VM_LOCKED) if (next->vm_flags & VM_LOCKED)
locked_vm += vma_pages(next); *locked_vm += vma_pages(next);
count++;
if (unlikely(uf)) { if (unlikely(uf)) {
/* /*
* If userfaultfd_unmap_prep returns an error the vmas * If userfaultfd_unmap_prep returns an error the vmas
@ -831,9 +826,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
* split, despite we could. This is unlikely enough * split, despite we could. This is unlikely enough
* failure that it's not worth optimizing it for. * failure that it's not worth optimizing it for.
*/ */
error = userfaultfd_unmap_prep(next, start, end, uf); if (userfaultfd_unmap_prep(next, start, end, uf))
if (error)
goto userfaultfd_error; goto userfaultfd_error;
} }
#ifdef CONFIG_DEBUG_VM_MAPLE_TREE #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
@ -845,7 +838,7 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
/* Make sure no VMAs are about to be lost. */ /* Make sure no VMAs are about to be lost. */
{ {
MA_STATE(test, &mt_detach, 0, 0); MA_STATE(test, mas_detach->tree, 0, 0);
struct vm_area_struct *vma_mas, *vma_test; struct vm_area_struct *vma_mas, *vma_test;
int test_count = 0; int test_count = 0;
@ -865,6 +858,48 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
while (vma_iter_addr(vmi) > start) while (vma_iter_addr(vmi) > start)
vma_iter_prev_range(vmi); vma_iter_prev_range(vmi);
return 0;
userfaultfd_error:
munmap_gather_failed:
end_split_failed:
modify_vma_failed:
abort_munmap_vmas(mas_detach);
start_split_failed:
map_count_exceeded:
return error;
}
/*
* do_vmi_align_munmap() - munmap the aligned region from @start to @end.
* @vmi: The vma iterator
* @vma: The starting vm_area_struct
* @mm: The mm_struct
* @start: The aligned start address to munmap.
* @end: The aligned end address to munmap.
* @uf: The userfaultfd list_head
* @unlock: Set to true to drop the mmap_lock. unlocking only happens on
* success.
*
* Return: 0 on success and drops the lock if so directed, error and leaves the
* lock held otherwise.
*/
int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
struct mm_struct *mm, unsigned long start, unsigned long end,
struct list_head *uf, bool unlock)
{
struct maple_tree mt_detach;
MA_STATE(mas_detach, &mt_detach, 0, 0);
mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
mt_on_stack(mt_detach);
int error;
unsigned long locked_vm = 0;
error = vmi_gather_munmap_vmas(vmi, vma, mm, start, end, uf,
&mas_detach, &locked_vm);
if (error)
goto gather_failed;
error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL); error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
if (error) if (error)
goto clear_tree_failed; goto clear_tree_failed;
@ -872,17 +907,11 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
/* Point of no return */ /* Point of no return */
vmi_complete_munmap_vmas(vmi, vma, mm, start, end, unlock, &mas_detach, vmi_complete_munmap_vmas(vmi, vma, mm, start, end, unlock, &mas_detach,
locked_vm); locked_vm);
return 0; return 0;
modify_vma_failed:
clear_tree_failed: clear_tree_failed:
userfaultfd_error:
munmap_gather_failed:
end_split_failed:
abort_munmap_vmas(&mas_detach); abort_munmap_vmas(&mas_detach);
start_split_failed: gather_failed:
map_count_exceeded:
validate_mm(mm); validate_mm(mm);
return error; return error;
} }