mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

madvise: convert madvise_cold_or_pageout_pte_range() to use folios

Browse source 
This change removes a number of calls to compound_head(), and saves
1729 bytes of kernel text.

Link: https://lkml.kernel.org/r/20221221180848.20774-3-vishal.moola@gmail.com
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Reviewed-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: SeongJae Park <sj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Vishal Moola (Oracle) 2022-12-21 10:08:46 -08:00 committed by Andrew Morton
parent 318e9342fb
commit 07e8c82b5e
1 changed files with 49 additions and 49 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

98
mm/madvise.c
View file

@ -345,8 +345,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
struct vm_area_struct *vma = walk->vma; struct vm_area_struct *vma = walk->vma;
pte_t *orig_pte, *pte, ptent; pte_t *orig_pte, *pte, ptent;
spinlock_t *ptl; spinlock_t *ptl;
struct page *page = NULL; struct folio *folio = NULL;
LIST_HEAD(page_list); LIST_HEAD(folio_list);
bool pageout_anon_only_filter; bool pageout_anon_only_filter;
if (fatal_signal_pending(current)) if (fatal_signal_pending(current))
@ -375,26 +375,26 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
goto huge_unlock; goto huge_unlock;
} }
page = pmd_page(orig_pmd); folio = pfn_folio(pmd_pfn(orig_pmd));
/* Do not interfere with other mappings of this page */ /* Do not interfere with other mappings of this folio */
if (page_mapcount(page) != 1) if (folio_mapcount(folio) != 1)
goto huge_unlock; goto huge_unlock;
if (pageout_anon_only_filter && !PageAnon(page)) if (pageout_anon_only_filter && !folio_test_anon(folio))
goto huge_unlock; goto huge_unlock;
if (next - addr != HPAGE_PMD_SIZE) { if (next - addr != HPAGE_PMD_SIZE) {
int err; int err;
get_page(page); folio_get(folio);
spin_unlock(ptl); spin_unlock(ptl);
lock_page(page); folio_lock(folio);
err = split_huge_page(page); err = split_folio(folio);
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
if (!err) if (!err)
goto regular_page; goto regular_folio;
return 0; return 0;
} }
@ -406,25 +406,25 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
tlb_remove_pmd_tlb_entry(tlb, pmd, addr); tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
} }
ClearPageReferenced(page); folio_clear_referenced(folio);
test_and_clear_page_young(page); folio_test_clear_young(folio);
if (pageout) { if (pageout) {
if (!isolate_lru_page(page)) { if (!folio_isolate_lru(folio)) {
if (PageUnevictable(page)) if (folio_test_unevictable(folio))
putback_lru_page(page); folio_putback_lru(folio);
else else
list_add(&page->lru, &page_list); list_add(&folio->lru, &folio_list);
} }
} else } else
deactivate_page(page); deactivate_page(&folio->page);
huge_unlock: huge_unlock:
spin_unlock(ptl); spin_unlock(ptl);
if (pageout) if (pageout)
reclaim_pages(&page_list); reclaim_pages(&folio_list);
return 0; return 0;
} }
regular_page: regular_folio:
if (pmd_trans_unstable(pmd)) if (pmd_trans_unstable(pmd))
return 0; return 0;
#endif #endif
@ -441,33 +441,33 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
if (!pte_present(ptent)) if (!pte_present(ptent))
continue; continue;
page = vm_normal_page(vma, addr, ptent); folio = vm_normal_folio(vma, addr, ptent);
if (!page || is_zone_device_page(page)) if (!folio || folio_is_zone_device(folio))
continue; continue;
/* /*
* Creating a THP page is expensive so split it only if we * Creating a THP page is expensive so split it only if we
* are sure it's worth. Split it if we are only owner. * are sure it's worth. Split it if we are only owner.
*/ */
if (PageTransCompound(page)) { if (folio_test_large(folio)) {
if (page_mapcount(page) != 1) if (folio_mapcount(folio) != 1)
break; break;
if (pageout_anon_only_filter && !PageAnon(page)) if (pageout_anon_only_filter && !folio_test_anon(folio))
break; break;
get_page(page); folio_get(folio);
if (!trylock_page(page)) { if (!folio_trylock(folio)) {
put_page(page); folio_put(folio);
break; break;
} }
pte_unmap_unlock(orig_pte, ptl); pte_unmap_unlock(orig_pte, ptl);
if (split_huge_page(page)) { if (split_folio(folio)) {
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
orig_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); orig_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
break; break;
} }
unlock_page(page); folio_unlock(folio);
put_page(page); folio_put(folio);
orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
pte--; pte--;
addr -= PAGE_SIZE; addr -= PAGE_SIZE;
@ -475,16 +475,16 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
} }
/* /*
* Do not interfere with other mappings of this page and * Do not interfere with other mappings of this folio and
* non-LRU page. * non-LRU folio.
*/ */
if (!PageLRU(page) || page_mapcount(page) != 1) if (!folio_test_lru(folio) || folio_mapcount(folio) != 1)
continue; continue;
if (pageout_anon_only_filter && !PageAnon(page)) if (pageout_anon_only_filter && !folio_test_anon(folio))
continue; continue;
VM_BUG_ON_PAGE(PageTransCompound(page), page); VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
if (pte_young(ptent)) { if (pte_young(ptent)) {
ptent = ptep_get_and_clear_full(mm, addr, pte, ptent = ptep_get_and_clear_full(mm, addr, pte,
@ -495,28 +495,28 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
} }
/* /*
* We are deactivating a page for accelerating reclaiming. * We are deactivating a folio for accelerating reclaiming.
* VM couldn't reclaim the page unless we clear PG_young. * VM couldn't reclaim the folio unless we clear PG_young.
* As a side effect, it makes confuse idle-page tracking * As a side effect, it makes confuse idle-page tracking
* because they will miss recent referenced history. * because they will miss recent referenced history.
*/ */
ClearPageReferenced(page); folio_clear_referenced(folio);
test_and_clear_page_young(page); folio_test_clear_young(folio);
if (pageout) { if (pageout) {
if (!isolate_lru_page(page)) { if (!folio_isolate_lru(folio)) {
if (PageUnevictable(page)) if (folio_test_unevictable(folio))
putback_lru_page(page); folio_putback_lru(folio);
else else
list_add(&page->lru, &page_list); list_add(&folio->lru, &folio_list);
} }
} else } else
deactivate_page(page); deactivate_page(&folio->page);
} }
arch_leave_lazy_mmu_mode(); arch_leave_lazy_mmu_mode();
pte_unmap_unlock(orig_pte, ptl); pte_unmap_unlock(orig_pte, ptl);
if (pageout) if (pageout)
reclaim_pages(&page_list); reclaim_pages(&folio_list);
cond_resched(); cond_resched();
return 0; return 0;