mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm/vmscan: fix a lot of comments

Browse source 
Patch series "MM folio changes for 6.1", v2.

My focus this round has been on shmem.  I believe it is now fully
converted to folios.  Of course, shmem interacts with a lot of the swap
cache and other parts of the kernel, so there are patches all over the MM.

This patch series survives a round of xfstests on tmpfs, which is nice,
but hardly an exhaustive test.  Hugh was nice enough to run a round of
tests on it and found a bug which is fixed in this edition.


This patch (of 57):

A lot of comments mention pages when they should say folios.
Fix them up.

[akpm@linux-foundation.org: fixups for mglru additions]
Link: https://lkml.kernel.org/r/20220902194653.1739778-1-willy@infradead.org
Link: https://lkml.kernel.org/r/20220902194653.1739778-2-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Matthew Wilcox (Oracle) 2022-09-02 20:45:57 +01:00 committed by Andrew Morton
parent 58730ab6c7
commit 49fd9b6df5
1 changed files with 129 additions and 132 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

261
mm/vmscan.c
View file

@ -90,7 +90,7 @@ struct scan_control {
unsigned long anon_cost; unsigned long anon_cost;
unsigned long file_cost; unsigned long file_cost;
/* Can active pages be deactivated as part of reclaim? */ /* Can active folios be deactivated as part of reclaim? */
#define DEACTIVATE_ANON 1 #define DEACTIVATE_ANON 1
#define DEACTIVATE_FILE 2 #define DEACTIVATE_FILE 2
unsigned int may_deactivate:2; unsigned int may_deactivate:2;
@ -100,10 +100,10 @@ struct scan_control {
/* Writepage batching in laptop mode; RECLAIM_WRITE */ /* Writepage batching in laptop mode; RECLAIM_WRITE */
unsigned int may_writepage:1; unsigned int may_writepage:1;
/* Can mapped pages be reclaimed? */ /* Can mapped folios be reclaimed? */
unsigned int may_unmap:1; unsigned int may_unmap:1;
/* Can pages be swapped as part of reclaim? */ /* Can folios be swapped as part of reclaim? */
unsigned int may_swap:1; unsigned int may_swap:1;
/* Proactive reclaim invoked by userspace through memory.reclaim */ /* Proactive reclaim invoked by userspace through memory.reclaim */
@ -128,7 +128,7 @@ struct scan_control {
/* There is easily reclaimable cold cache in the current node */ /* There is easily reclaimable cold cache in the current node */
unsigned int cache_trim_mode:1; unsigned int cache_trim_mode:1;
/* The file pages on the current node are dangerously low */ /* The file folios on the current node are dangerously low */
unsigned int file_is_tiny:1; unsigned int file_is_tiny:1;
/* Always discard instead of demoting to lower tier memory */ /* Always discard instead of demoting to lower tier memory */
@ -146,7 +146,7 @@ struct scan_control {
/* Scan (total_size >> priority) pages at once */ /* Scan (total_size >> priority) pages at once */
s8 priority; s8 priority;
/* The highest zone to isolate pages for reclaim from */ /* The highest zone to isolate folios for reclaim from */
s8 reclaim_idx; s8 reclaim_idx;
/* This context's GFP mask */ /* This context's GFP mask */
@ -454,7 +454,7 @@ static bool cgroup_reclaim(struct scan_control *sc)
* *
* The normal page dirty throttling mechanism in balance_dirty_pages() is * The normal page dirty throttling mechanism in balance_dirty_pages() is
* completely broken with the legacy memcg and direct stalling in * completely broken with the legacy memcg and direct stalling in
* shrink_page_list() is used for throttling instead, which lacks all the * shrink_folio_list() is used for throttling instead, which lacks all the
* niceties such as fairness, adaptive pausing, bandwidth proportional * niceties such as fairness, adaptive pausing, bandwidth proportional
* allocation and configurability. * allocation and configurability.
* *
@ -575,9 +575,9 @@ static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
} }
/* /*
* This misses isolated pages which are not accounted for to save counters. * This misses isolated folios which are not accounted for to save counters.
* As the data only determines if reclaim or compaction continues, it is * As the data only determines if reclaim or compaction continues, it is
* not expected that isolated pages will be a dominating factor. * not expected that isolated folios will be a dominating factor.
*/ */
unsigned long zone_reclaimable_pages(struct zone *zone) unsigned long zone_reclaimable_pages(struct zone *zone)
{ {
@ -1050,9 +1050,9 @@ void drop_slab(void)
static inline int is_page_cache_freeable(struct folio *folio) static inline int is_page_cache_freeable(struct folio *folio)
{ {
/* /*
* A freeable page cache page is referenced only by the caller * A freeable page cache folio is referenced only by the caller
* that isolated the page, the page cache and optional buffer * that isolated the folio, the page cache and optional filesystem
* heads at page->private. * private data at folio->private.
*/ */
return folio_ref_count(folio) - folio_test_private(folio) == return folio_ref_count(folio) - folio_test_private(folio) ==
1 + folio_nr_pages(folio); 1 + folio_nr_pages(folio);
@ -1092,8 +1092,8 @@ static bool skip_throttle_noprogress(pg_data_t *pgdat)
return true; return true;
/* /*
* If there are a lot of dirty/writeback pages then do not * If there are a lot of dirty/writeback folios then do not
* throttle as throttling will occur when the pages cycle * throttle as throttling will occur when the folios cycle
* towards the end of the LRU if still under writeback. * towards the end of the LRU if still under writeback.
*/ */
for (i = 0; i < MAX_NR_ZONES; i++) { for (i = 0; i < MAX_NR_ZONES; i++) {
@ -1136,7 +1136,7 @@ void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
* short. Failing to make progress or waiting on writeback are * short. Failing to make progress or waiting on writeback are
* potentially long-lived events so use a longer timeout. This is shaky * potentially long-lived events so use a longer timeout. This is shaky
* logic as a failure to make progress could be due to anything from * logic as a failure to make progress could be due to anything from
* writeback to a slow device to excessive references pages at the tail * writeback to a slow device to excessive referenced folios at the tail
* of the inactive LRU. * of the inactive LRU.
*/ */
switch(reason) { switch(reason) {
@ -1182,8 +1182,8 @@ void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
} }
/* /*
* Account for pages written if tasks are throttled waiting on dirty * Account for folios written if tasks are throttled waiting on dirty
* pages to clean. If enough pages have been cleaned since throttling * folios to clean. If enough folios have been cleaned since throttling
* started then wakeup the throttled tasks. * started then wakeup the throttled tasks.
*/ */
void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
@ -1209,18 +1209,18 @@ void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
/* possible outcome of pageout() */ /* possible outcome of pageout() */
typedef enum { typedef enum {
/* failed to write page out, page is locked */ /* failed to write folio out, folio is locked */
PAGE_KEEP, PAGE_KEEP,
/* move page to the active list, page is locked */ /* move folio to the active list, folio is locked */
PAGE_ACTIVATE, PAGE_ACTIVATE,
/* page has been sent to the disk successfully, page is unlocked */ /* folio has been sent to the disk successfully, folio is unlocked */
PAGE_SUCCESS, PAGE_SUCCESS,
/* page is clean and locked */ /* folio is clean and locked */
PAGE_CLEAN, PAGE_CLEAN,
} pageout_t; } pageout_t;
/* /*
* pageout is called by shrink_page_list() for each dirty page. * pageout is called by shrink_folio_list() for each dirty folio.
* Calls ->writepage(). * Calls ->writepage().
*/ */
static pageout_t pageout(struct folio *folio, struct address_space *mapping, static pageout_t pageout(struct folio *folio, struct address_space *mapping,
@ -1294,7 +1294,7 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,
} }
/* /*
* Same as remove_mapping, but if the page is removed from the mapping, it * Same as remove_mapping, but if the folio is removed from the mapping, it
* gets returned with a refcount of 0. * gets returned with a refcount of 0.
*/ */
static int __remove_mapping(struct address_space *mapping, struct folio *folio, static int __remove_mapping(struct address_space *mapping, struct folio *folio,
@ -1310,34 +1310,34 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
spin_lock(&mapping->host->i_lock); spin_lock(&mapping->host->i_lock);
xa_lock_irq(&mapping->i_pages); xa_lock_irq(&mapping->i_pages);
/* /*
* The non racy check for a busy page. * The non racy check for a busy folio.
* *
* Must be careful with the order of the tests. When someone has * Must be careful with the order of the tests. When someone has
* a ref to the page, it may be possible that they dirty it then * a ref to the folio, it may be possible that they dirty it then
* drop the reference. So if PageDirty is tested before page_count * drop the reference. So if the dirty flag is tested before the
* here, then the following race may occur: * refcount here, then the following race may occur:
* *
* get_user_pages(&page); * get_user_pages(&page);
* [user mapping goes away] * [user mapping goes away]
* write_to(page); * write_to(page);
* !PageDirty(page) [good] * !folio_test_dirty(folio) [good]
* SetPageDirty(page); * folio_set_dirty(folio);
* put_page(page); * folio_put(folio);
* !page_count(page) [good, discard it] * !refcount(folio) [good, discard it]
* *
* [oops, our write_to data is lost] * [oops, our write_to data is lost]
* *
* Reversing the order of the tests ensures such a situation cannot * Reversing the order of the tests ensures such a situation cannot
* escape unnoticed. The smp_rmb is needed to ensure the page->flags * escape unnoticed. The smp_rmb is needed to ensure the folio->flags
* load is not satisfied before that of page->_refcount. * load is not satisfied before that of folio->_refcount.
* *
* Note that if SetPageDirty is always performed via set_page_dirty, * Note that if the dirty flag is always set via folio_mark_dirty,
* and thus under the i_pages lock, then this ordering is not required. * and thus under the i_pages lock, then this ordering is not required.
*/ */
refcount = 1 + folio_nr_pages(folio); refcount = 1 + folio_nr_pages(folio);
if (!folio_ref_freeze(folio, refcount)) if (!folio_ref_freeze(folio, refcount))
goto cannot_free; goto cannot_free;
/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */ /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */
if (unlikely(folio_test_dirty(folio))) { if (unlikely(folio_test_dirty(folio))) {
folio_ref_unfreeze(folio, refcount); folio_ref_unfreeze(folio, refcount);
goto cannot_free; goto cannot_free;
@ -1368,7 +1368,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
* back. * back.
* *
* We also don't store shadows for DAX mappings because the * We also don't store shadows for DAX mappings because the
* only page cache pages found in these are zero pages * only page cache folios found in these are zero pages
* covering holes, and because we don't want to mix DAX * covering holes, and because we don't want to mix DAX
* exceptional entries and shadow exceptional entries in the * exceptional entries and shadow exceptional entries in the
* same address_space. * same address_space.
@ -1436,14 +1436,14 @@ void folio_putback_lru(struct folio *folio)
folio_put(folio); /* drop ref from isolate */ folio_put(folio); /* drop ref from isolate */
} }
enum page_references { enum folio_references {
PAGEREF_RECLAIM, FOLIOREF_RECLAIM,
PAGEREF_RECLAIM_CLEAN, FOLIOREF_RECLAIM_CLEAN,
PAGEREF_KEEP, FOLIOREF_KEEP,
PAGEREF_ACTIVATE, FOLIOREF_ACTIVATE,
}; };
static enum page_references folio_check_references(struct folio *folio, static enum folio_references folio_check_references(struct folio *folio,
struct scan_control *sc) struct scan_control *sc)
{ {
int referenced_ptes, referenced_folio; int referenced_ptes, referenced_folio;
@ -1458,11 +1458,11 @@ static enum page_references folio_check_references(struct folio *folio,
* Let the folio, now marked Mlocked, be moved to the unevictable list. * Let the folio, now marked Mlocked, be moved to the unevictable list.
*/ */
if (vm_flags & VM_LOCKED) if (vm_flags & VM_LOCKED)
return PAGEREF_ACTIVATE; return FOLIOREF_ACTIVATE;
/* rmap lock contention: rotate */ /* rmap lock contention: rotate */
if (referenced_ptes == -1) if (referenced_ptes == -1)
return PAGEREF_KEEP; return FOLIOREF_KEEP;
if (referenced_ptes) { if (referenced_ptes) {
/* /*
@ -1482,34 +1482,34 @@ static enum page_references folio_check_references(struct folio *folio,
folio_set_referenced(folio); folio_set_referenced(folio);
if (referenced_folio || referenced_ptes > 1) if (referenced_folio || referenced_ptes > 1)
return PAGEREF_ACTIVATE; return FOLIOREF_ACTIVATE;
/* /*
* Activate file-backed executable folios after first usage. * Activate file-backed executable folios after first usage.
*/ */
if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio))
return PAGEREF_ACTIVATE; return FOLIOREF_ACTIVATE;
return PAGEREF_KEEP; return FOLIOREF_KEEP;
} }
/* Reclaim if clean, defer dirty folios to writeback */ /* Reclaim if clean, defer dirty folios to writeback */
if (referenced_folio && folio_is_file_lru(folio)) if (referenced_folio && folio_is_file_lru(folio))
return PAGEREF_RECLAIM_CLEAN; return FOLIOREF_RECLAIM_CLEAN;
return PAGEREF_RECLAIM; return FOLIOREF_RECLAIM;
} }
/* Check if a page is dirty or under writeback */ /* Check if a folio is dirty or under writeback */
static void folio_check_dirty_writeback(struct folio *folio, static void folio_check_dirty_writeback(struct folio *folio,
bool *dirty, bool *writeback) bool *dirty, bool *writeback)
{ {
struct address_space *mapping; struct address_space *mapping;
/* /*
* Anonymous pages are not handled by flushers and must be written * Anonymous folios are not handled by flushers and must be written
* from reclaim context. Do not stall reclaim based on them. * from reclaim context. Do not stall reclaim based on them.
* MADV_FREE anonymous pages are put into inactive file list too. * MADV_FREE anonymous folios are put into inactive file list too.
* They could be mistakenly treated as file lru. So further anon * They could be mistakenly treated as file lru. So further anon
* test is needed. * test is needed.
*/ */
@ -1564,11 +1564,10 @@ static struct page *alloc_demote_page(struct page *page, unsigned long private)
} }
/* /*
* Take pages on @demote_list and attempt to demote them to * Take folios on @demote_folios and attempt to demote them to another node.
* another node. Pages which are not demoted are left on * Folios which are not demoted are left on @demote_folios.
* @demote_pages.
*/ */
static unsigned int demote_page_list(struct list_head *demote_pages, static unsigned int demote_folio_list(struct list_head *demote_folios,
struct pglist_data *pgdat) struct pglist_data *pgdat)
{ {
int target_nid = next_demotion_node(pgdat->node_id); int target_nid = next_demotion_node(pgdat->node_id);
@ -1587,7 +1586,7 @@ static unsigned int demote_page_list(struct list_head *demote_pages,
.nmask = &allowed_mask .nmask = &allowed_mask
}; };
if (list_empty(demote_pages)) if (list_empty(demote_folios))
return 0; return 0;
if (target_nid == NUMA_NO_NODE) if (target_nid == NUMA_NO_NODE)
@ -1596,7 +1595,7 @@ static unsigned int demote_page_list(struct list_head *demote_pages,
node_get_allowed_targets(pgdat, &allowed_mask); node_get_allowed_targets(pgdat, &allowed_mask);
/* Demotion ignores all cpuset and mempolicy settings */ /* Demotion ignores all cpuset and mempolicy settings */
migrate_pages(demote_pages, alloc_demote_page, NULL, migrate_pages(demote_folios, alloc_demote_page, NULL,
(unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
&nr_succeeded); &nr_succeeded);
@ -1625,17 +1624,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
} }
/* /*
* shrink_page_list() returns the number of reclaimed pages * shrink_folio_list() returns the number of reclaimed pages
*/ */
static unsigned int shrink_page_list(struct list_head *page_list, static unsigned int shrink_folio_list(struct list_head *folio_list,
struct pglist_data *pgdat, struct pglist_data *pgdat, struct scan_control *sc,
struct scan_control *sc, struct reclaim_stat *stat, bool ignore_references)
struct reclaim_stat *stat,
bool ignore_references)
{ {
LIST_HEAD(ret_pages); LIST_HEAD(ret_folios);
LIST_HEAD(free_pages); LIST_HEAD(free_folios);
LIST_HEAD(demote_pages); LIST_HEAD(demote_folios);
unsigned int nr_reclaimed = 0; unsigned int nr_reclaimed = 0;
unsigned int pgactivate = 0; unsigned int pgactivate = 0;
bool do_demote_pass; bool do_demote_pass;
@ -1646,16 +1643,16 @@ static unsigned int shrink_page_list(struct list_head *page_list,
do_demote_pass = can_demote(pgdat->node_id, sc); do_demote_pass = can_demote(pgdat->node_id, sc);
retry: retry:
while (!list_empty(page_list)) { while (!list_empty(folio_list)) {
struct address_space *mapping; struct address_space *mapping;
struct folio *folio; struct folio *folio;
enum page_references references = PAGEREF_RECLAIM; enum folio_references references = FOLIOREF_RECLAIM;
bool dirty, writeback; bool dirty, writeback;
unsigned int nr_pages; unsigned int nr_pages;
cond_resched(); cond_resched();
folio = lru_to_folio(page_list); folio = lru_to_folio(folio_list);
list_del(&folio->lru); list_del(&folio->lru);
if (!folio_trylock(folio)) if (!folio_trylock(folio))
@ -1779,7 +1776,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
folio_unlock(folio); folio_unlock(folio);
folio_wait_writeback(folio); folio_wait_writeback(folio);
/* then go back and try same folio again */ /* then go back and try same folio again */
list_add_tail(&folio->lru, page_list); list_add_tail(&folio->lru, folio_list);
continue; continue;
} }
} }
@ -1788,13 +1785,13 @@ static unsigned int shrink_page_list(struct list_head *page_list,
references = folio_check_references(folio, sc); references = folio_check_references(folio, sc);
switch (references) { switch (references) {
case PAGEREF_ACTIVATE: case FOLIOREF_ACTIVATE:
goto activate_locked; goto activate_locked;
case PAGEREF_KEEP: case FOLIOREF_KEEP:
stat->nr_ref_keep += nr_pages; stat->nr_ref_keep += nr_pages;
goto keep_locked; goto keep_locked;
case PAGEREF_RECLAIM: case FOLIOREF_RECLAIM:
case PAGEREF_RECLAIM_CLEAN: case FOLIOREF_RECLAIM_CLEAN:
; /* try to reclaim the folio below */ ; /* try to reclaim the folio below */
} }
@ -1804,7 +1801,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
*/ */
if (do_demote_pass && if (do_demote_pass &&
(thp_migration_supported() || !folio_test_large(folio))) { (thp_migration_supported() || !folio_test_large(folio))) {
list_add(&folio->lru, &demote_pages); list_add(&folio->lru, &demote_folios);
folio_unlock(folio); folio_unlock(folio);
continue; continue;
} }
@ -1831,7 +1828,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
*/ */
if (!folio_entire_mapcount(folio) && if (!folio_entire_mapcount(folio) &&
split_folio_to_list(folio, split_folio_to_list(folio,
page_list)) folio_list))
goto activate_locked; goto activate_locked;
} }
if (!add_to_swap(folio)) { if (!add_to_swap(folio)) {
@ -1839,7 +1836,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
goto activate_locked_split; goto activate_locked_split;
/* Fallback to swap normal pages */ /* Fallback to swap normal pages */
if (split_folio_to_list(folio, if (split_folio_to_list(folio,
page_list)) folio_list))
goto activate_locked; goto activate_locked;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
count_vm_event(THP_SWPOUT_FALLBACK); count_vm_event(THP_SWPOUT_FALLBACK);
@ -1851,7 +1848,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
} else if (folio_test_swapbacked(folio) && } else if (folio_test_swapbacked(folio) &&
folio_test_large(folio)) { folio_test_large(folio)) {
/* Split shmem folio */ /* Split shmem folio */
if (split_folio_to_list(folio, page_list)) if (split_folio_to_list(folio, folio_list))
goto keep_locked; goto keep_locked;
} }
@ -1916,7 +1913,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
goto activate_locked; goto activate_locked;
} }
if (references == PAGEREF_RECLAIM_CLEAN) if (references == FOLIOREF_RECLAIM_CLEAN)
goto keep_locked; goto keep_locked;
if (!may_enter_fs(folio, sc->gfp_mask)) if (!may_enter_fs(folio, sc->gfp_mask))
goto keep_locked; goto keep_locked;
@ -2029,13 +2026,13 @@ static unsigned int shrink_page_list(struct list_head *page_list,
nr_reclaimed += nr_pages; nr_reclaimed += nr_pages;
/* /*
* Is there need to periodically free_page_list? It would * Is there need to periodically free_folio_list? It would
* appear not as the counts should be low * appear not as the counts should be low
*/ */
if (unlikely(folio_test_large(folio))) if (unlikely(folio_test_large(folio)))
destroy_large_folio(folio); destroy_large_folio(folio);
else else
list_add(&folio->lru, &free_pages); list_add(&folio->lru, &free_folios);
continue; continue;
activate_locked_split: activate_locked_split:
@ -2063,29 +2060,29 @@ static unsigned int shrink_page_list(struct list_head *page_list,
keep_locked: keep_locked:
folio_unlock(folio); folio_unlock(folio);
keep: keep:
list_add(&folio->lru, &ret_pages); list_add(&folio->lru, &ret_folios);
VM_BUG_ON_FOLIO(folio_test_lru(folio) || VM_BUG_ON_FOLIO(folio_test_lru(folio) ||
folio_test_unevictable(folio), folio); folio_test_unevictable(folio), folio);
} }
/* 'page_list' is always empty here */ /* 'folio_list' is always empty here */
/* Migrate folios selected for demotion */ /* Migrate folios selected for demotion */
nr_reclaimed += demote_page_list(&demote_pages, pgdat); nr_reclaimed += demote_folio_list(&demote_folios, pgdat);
/* Folios that could not be demoted are still in @demote_pages */ /* Folios that could not be demoted are still in @demote_folios */
if (!list_empty(&demote_pages)) { if (!list_empty(&demote_folios)) {
/* Folios which weren't demoted go back on @page_list for retry: */ /* Folios which weren't demoted go back on @folio_list for retry: */
list_splice_init(&demote_pages, page_list); list_splice_init(&demote_folios, folio_list);
do_demote_pass = false; do_demote_pass = false;
goto retry; goto retry;
} }
pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
mem_cgroup_uncharge_list(&free_pages); mem_cgroup_uncharge_list(&free_folios);
try_to_unmap_flush(); try_to_unmap_flush();
free_unref_page_list(&free_pages); free_unref_page_list(&free_folios);
list_splice(&ret_pages, page_list); list_splice(&ret_folios, folio_list);
count_vm_events(PGACTIVATE, pgactivate); count_vm_events(PGACTIVATE, pgactivate);
if (plug) if (plug)
@ -2094,7 +2091,7 @@ static unsigned int shrink_page_list(struct list_head *page_list,
} }
unsigned int reclaim_clean_pages_from_list(struct zone *zone, unsigned int reclaim_clean_pages_from_list(struct zone *zone,
struct list_head *folio_list) struct list_head *folio_list)
{ {
struct scan_control sc = { struct scan_control sc = {
.gfp_mask = GFP_KERNEL, .gfp_mask = GFP_KERNEL,
@ -2122,7 +2119,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
* change in the future. * change in the future.
*/ */
noreclaim_flag = memalloc_noreclaim_save(); noreclaim_flag = memalloc_noreclaim_save();
nr_reclaimed = shrink_page_list(&clean_folios, zone->zone_pgdat, &sc, nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc,
&stat, true); &stat, true);
memalloc_noreclaim_restore(noreclaim_flag); memalloc_noreclaim_restore(noreclaim_flag);
@ -2181,7 +2178,7 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec,
* *
* returns how many pages were moved onto *@dst. * returns how many pages were moved onto *@dst.
*/ */
static unsigned long isolate_lru_pages(unsigned long nr_to_scan, static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
struct lruvec *lruvec, struct list_head *dst, struct lruvec *lruvec, struct list_head *dst,
unsigned long *nr_scanned, struct scan_control *sc, unsigned long *nr_scanned, struct scan_control *sc,
enum lru_list lru) enum lru_list lru)
@ -2288,8 +2285,8 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
* *
* Context: * Context:
* *
* (1) Must be called with an elevated refcount on the page. This is a * (1) Must be called with an elevated refcount on the folio. This is a
* fundamental difference from isolate_lru_pages() (which is called * fundamental difference from isolate_lru_folios() (which is called
* without a stable reference). * without a stable reference).
* (2) The lru_lock must not be held. * (2) The lru_lock must not be held.
* (3) Interrupts must be enabled. * (3) Interrupts must be enabled.
@ -2361,13 +2358,13 @@ static int too_many_isolated(struct pglist_data *pgdat, int file,
} }
/* /*
* move_pages_to_lru() moves folios from private @list to appropriate LRU list. * move_folios_to_lru() moves folios from private @list to appropriate LRU list.
* On return, @list is reused as a list of folios to be freed by the caller. * On return, @list is reused as a list of folios to be freed by the caller.
* *
* Returns the number of pages moved to the given lruvec. * Returns the number of pages moved to the given lruvec.
*/ */
static unsigned int move_pages_to_lru(struct lruvec *lruvec, static unsigned int move_folios_to_lru(struct lruvec *lruvec,
struct list_head *list) struct list_head *list)
{ {
int nr_pages, nr_moved = 0; int nr_pages, nr_moved = 0;
LIST_HEAD(folios_to_free); LIST_HEAD(folios_to_free);
@ -2387,7 +2384,7 @@ static unsigned int move_pages_to_lru(struct lruvec *lruvec,
/* /*
* The folio_set_lru needs to be kept here for list integrity. * The folio_set_lru needs to be kept here for list integrity.
* Otherwise: * Otherwise:
* #0 move_pages_to_lru #1 release_pages * #0 move_folios_to_lru #1 release_pages
* if (!folio_put_testzero()) * if (!folio_put_testzero())
* if (folio_put_testzero()) * if (folio_put_testzero())
* !lru //skip lru_lock * !lru //skip lru_lock
@ -2444,11 +2441,11 @@ static int current_may_throttle(void)
* shrink_inactive_list() is a helper for shrink_node(). It returns the number * shrink_inactive_list() is a helper for shrink_node(). It returns the number
* of reclaimed pages * of reclaimed pages
*/ */
static unsigned long static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, struct lruvec *lruvec, struct scan_control *sc,
struct scan_control *sc, enum lru_list lru) enum lru_list lru)
{ {
LIST_HEAD(page_list); LIST_HEAD(folio_list);
unsigned long nr_scanned; unsigned long nr_scanned;
unsigned int nr_reclaimed = 0; unsigned int nr_reclaimed = 0;
unsigned long nr_taken; unsigned long nr_taken;
@ -2475,7 +2472,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
spin_lock_irq(&lruvec->lru_lock); spin_lock_irq(&lruvec->lru_lock);
nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list, nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list,
&nr_scanned, sc, lru); &nr_scanned, sc, lru);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
@ -2490,10 +2487,10 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
if (nr_taken == 0) if (nr_taken == 0)
return 0; return 0;
nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false); nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false);
spin_lock_irq(&lruvec->lru_lock); spin_lock_irq(&lruvec->lru_lock);
move_pages_to_lru(lruvec, &page_list); move_folios_to_lru(lruvec, &folio_list);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
@ -2504,16 +2501,16 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
spin_unlock_irq(&lruvec->lru_lock); spin_unlock_irq(&lruvec->lru_lock);
lru_note_cost(lruvec, file, stat.nr_pageout); lru_note_cost(lruvec, file, stat.nr_pageout);
mem_cgroup_uncharge_list(&page_list); mem_cgroup_uncharge_list(&folio_list);
free_unref_page_list(&page_list); free_unref_page_list(&folio_list);
/* /*
* If dirty pages are scanned that are not queued for IO, it * If dirty folios are scanned that are not queued for IO, it
* implies that flushers are not doing their job. This can * implies that flushers are not doing their job. This can
* happen when memory pressure pushes dirty pages to the end of * happen when memory pressure pushes dirty folios to the end of
* the LRU before the dirty limits are breached and the dirty * the LRU before the dirty limits are breached and the dirty
* data has expired. It can also happen when the proportion of * data has expired. It can also happen when the proportion of
* dirty pages grows not through writes but through memory * dirty folios grows not through writes but through memory
* pressure reclaiming all the clean cache. And in some cases, * pressure reclaiming all the clean cache. And in some cases,
* the flushers simply cannot keep up with the allocation * the flushers simply cannot keep up with the allocation
* rate. Nudge the flusher threads in case they are asleep. * rate. Nudge the flusher threads in case they are asleep.
@ -2572,7 +2569,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
spin_lock_irq(&lruvec->lru_lock); spin_lock_irq(&lruvec->lru_lock);
nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold,
&nr_scanned, sc, lru); &nr_scanned, sc, lru);
__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
@ -2632,8 +2629,8 @@ static void shrink_active_list(unsigned long nr_to_scan,
*/ */
spin_lock_irq(&lruvec->lru_lock); spin_lock_irq(&lruvec->lru_lock);
nr_activate = move_pages_to_lru(lruvec, &l_active); nr_activate = move_folios_to_lru(lruvec, &l_active);
nr_deactivate = move_pages_to_lru(lruvec, &l_inactive); nr_deactivate = move_folios_to_lru(lruvec, &l_inactive);
/* Keep all free folios in l_active list */ /* Keep all free folios in l_active list */
list_splice(&l_inactive, &l_active); list_splice(&l_inactive, &l_active);
@ -2649,7 +2646,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
nr_deactivate, nr_rotated, sc->priority, file); nr_deactivate, nr_rotated, sc->priority, file);
} }
static unsigned int reclaim_page_list(struct list_head *page_list, static unsigned int reclaim_folio_list(struct list_head *folio_list,
struct pglist_data *pgdat) struct pglist_data *pgdat)
{ {
struct reclaim_stat dummy_stat; struct reclaim_stat dummy_stat;
@ -2663,9 +2660,9 @@ static unsigned int reclaim_page_list(struct list_head *page_list,
.no_demotion = 1, .no_demotion = 1,
}; };
nr_reclaimed = shrink_page_list(page_list, pgdat, &sc, &dummy_stat, false); nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false);
while (!list_empty(page_list)) { while (!list_empty(folio_list)) {
folio = lru_to_folio(page_list); folio = lru_to_folio(folio_list);
list_del(&folio->lru); list_del(&folio->lru);
folio_putback_lru(folio); folio_putback_lru(folio);
} }
@ -2695,11 +2692,11 @@ unsigned long reclaim_pages(struct list_head *folio_list)
continue; continue;
} }
nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid)); nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
nid = folio_nid(lru_to_folio(folio_list)); nid = folio_nid(lru_to_folio(folio_list));
} while (!list_empty(folio_list)); } while (!list_empty(folio_list));
nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid)); nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid));
memalloc_noreclaim_restore(noreclaim_flag); memalloc_noreclaim_restore(noreclaim_flag);
@ -2729,13 +2726,13 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
* but large enough to avoid thrashing the aggregate readahead window. * but large enough to avoid thrashing the aggregate readahead window.
* *
* Both inactive lists should also be large enough that each inactive * Both inactive lists should also be large enough that each inactive
* page has a chance to be referenced again before it is reclaimed. * folio has a chance to be referenced again before it is reclaimed.
* *
* If that fails and refaulting is observed, the inactive list grows. * If that fails and refaulting is observed, the inactive list grows.
* *
* The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios
* on this LRU, maintained by the pageout code. An inactive_ratio * on this LRU, maintained by the pageout code. An inactive_ratio
* of 3 means 3:1 or 25% of the pages are kept on the inactive list. * of 3 means 3:1 or 25% of the folios are kept on the inactive list.
* *
* total target max * total target max
* memory ratio inactive * memory ratio inactive
@ -2884,8 +2881,8 @@ static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc)
* Determine how aggressively the anon and file LRU lists should be * Determine how aggressively the anon and file LRU lists should be
* scanned. * scanned.
* *
* nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan
* nr[2] = file inactive pages to scan; nr[3] = file active pages to scan * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan
*/ */
static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
unsigned long *nr) unsigned long *nr)
@ -2900,7 +2897,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
unsigned long ap, fp; unsigned long ap, fp;
enum lru_list lru; enum lru_list lru;
/* If we have no swap space, do not bother scanning anon pages. */ /* If we have no swap space, do not bother scanning anon folios. */
if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
scan_balance = SCAN_FILE; scan_balance = SCAN_FILE;
goto out; goto out;
@ -3647,7 +3644,7 @@ static int folio_update_gen(struct folio *folio, int gen)
do { do {
/* lru_gen_del_folio() has isolated this page? */ /* lru_gen_del_folio() has isolated this page? */
if (!(old_flags & LRU_GEN_MASK)) { if (!(old_flags & LRU_GEN_MASK)) {
/* for shrink_page_list() */ /* for shrink_folio_list() */
new_flags = old_flags | BIT(PG_referenced); new_flags = old_flags | BIT(PG_referenced);
continue; continue;
} }
@ -4574,7 +4571,7 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc)
} }
/* /*
* This function exploits spatial locality when shrink_page_list() walks the * This function exploits spatial locality when shrink_folio_list() walks the
* rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If
* the scan was done cacheline efficiently, it adds the PMD entry pointing to * the scan was done cacheline efficiently, it adds the PMD entry pointing to
* the PTE table to the Bloom filter. This forms a feedback loop between the * the PTE table to the Bloom filter. This forms a feedback loop between the
@ -4795,7 +4792,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
if (!folio_test_referenced(folio)) if (!folio_test_referenced(folio))
set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0);
/* for shrink_page_list() */ /* for shrink_folio_list() */
folio_clear_reclaim(folio); folio_clear_reclaim(folio);
folio_clear_referenced(folio); folio_clear_referenced(folio);
@ -4998,7 +4995,7 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
if (list_empty(&list)) if (list_empty(&list))
return scanned; return scanned;
reclaimed = shrink_page_list(&list, pgdat, sc, &stat, false); reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false);
list_for_each_entry(folio, &list, lru) { list_for_each_entry(folio, &list, lru) {
/* restore LRU_REFS_FLAGS cleared by isolate_folio() */ /* restore LRU_REFS_FLAGS cleared by isolate_folio() */
@ -5015,7 +5012,7 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap
spin_lock_irq(&lruvec->lru_lock); spin_lock_irq(&lruvec->lru_lock);
move_pages_to_lru(lruvec, &list); move_folios_to_lru(lruvec, &list);
walk = current->reclaim_state->mm_walk; walk = current->reclaim_state->mm_walk;
if (walk && walk->batched) if (walk && walk->batched)