mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm: support order-1 folios in the page cache

Browse source 
Folios of order 1 have no space to store the deferred list.  This is not a
problem for the page cache as file-backed folios are never placed on the
deferred list.  All we need to do is prevent the core MM from touching the
deferred list for order 1 folios and remove the code which prevented us
from allocating order 1 folios.

Link: https://lore.kernel.org/linux-mm/90344ea7-4eec-47ee-5996-0c22f42d6a6a@google.com/
Link: https://lkml.kernel.org/r/20240226205534.1603748-3-zi.yan@sent.com
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Zi Yan <ziy@nvidia.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Luis Chamberlain <mcgrof@kernel.org>
Cc: Michal Koutny <mkoutny@suse.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Yu Zhao <yuzhao@google.com>
Cc: Zach O'Keefe <zokeefe@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Matthew Wilcox (Oracle) 2024-02-26 15:55:28 -05:00 committed by Andrew Morton
parent 319a624ec2
commit 8897277acf
4 changed files with 16 additions and 11 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

2
mm/filemap.c
View file

@ -1912,8 +1912,6 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
gfp_t alloc_gfp = gfp; gfp_t alloc_gfp = gfp;
err = -ENOMEM; err = -ENOMEM;
if (order == 1)
order = 0;
if (order > 0) if (order > 0)
alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN; alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
folio = filemap_alloc_folio(alloc_gfp, order); folio = filemap_alloc_folio(alloc_gfp, order);

19
mm/huge_memory.c
View file

@ -790,8 +790,10 @@ struct deferred_split *get_deferred_split_queue(struct folio *folio)
void folio_prep_large_rmappable(struct folio *folio) void folio_prep_large_rmappable(struct folio *folio)
{ {
VM_BUG_ON_FOLIO(folio_order(folio) < 2, folio); if (!folio || !folio_test_large(folio))
INIT_LIST_HEAD(&folio->_deferred_list); return;
if (folio_order(folio) > 1)
INIT_LIST_HEAD(&folio->_deferred_list);
folio_set_large_rmappable(folio); folio_set_large_rmappable(folio);
} }
@ -3114,7 +3116,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
/* Prevent deferred_split_scan() touching ->_refcount */ /* Prevent deferred_split_scan() touching ->_refcount */
spin_lock(&ds_queue->split_queue_lock); spin_lock(&ds_queue->split_queue_lock);
if (folio_ref_freeze(folio, 1 + extra_pins)) { if (folio_ref_freeze(folio, 1 + extra_pins)) {
if (!list_empty(&folio->_deferred_list)) { if (folio_order(folio) > 1 &&
!list_empty(&folio->_deferred_list)) {
ds_queue->split_queue_len--; ds_queue->split_queue_len--;
list_del(&folio->_deferred_list); list_del(&folio->_deferred_list);
} }
@ -3165,6 +3168,9 @@ void folio_undo_large_rmappable(struct folio *folio)
struct deferred_split *ds_queue; struct deferred_split *ds_queue;
unsigned long flags; unsigned long flags;
if (folio_order(folio) <= 1)
return;
/* /*
* At this point, there is no one trying to add the folio to * At this point, there is no one trying to add the folio to
* deferred_list. If folio is not in deferred_list, it's safe * deferred_list. If folio is not in deferred_list, it's safe
@ -3190,7 +3196,12 @@ void deferred_split_folio(struct folio *folio)
#endif #endif
unsigned long flags; unsigned long flags;
VM_BUG_ON_FOLIO(folio_order(folio) < 2, folio); /*
* Order 1 folios have no space for a deferred list, but we also
* won't waste much memory by not adding them to the deferred list.
*/
if (folio_order(folio) <= 1)
return;
/* /*
* The try_to_unmap() in page reclaim path might reach here too, * The try_to_unmap() in page reclaim path might reach here too,

3
mm/internal.h
View file

@ -420,8 +420,7 @@ static inline struct folio *page_rmappable_folio(struct page *page)
{ {
struct folio *folio = (struct folio *)page; struct folio *folio = (struct folio *)page;
if (folio && folio_order(folio) > 1) folio_prep_large_rmappable(folio);
folio_prep_large_rmappable(folio);
return folio; return folio;
} }

3
mm/readahead.c
View file

@ -514,9 +514,6 @@ void page_cache_ra_order(struct readahead_control *ractl,
/* Don't allocate pages past EOF */ /* Don't allocate pages past EOF */
while (index + (1UL << order) - 1 > limit) while (index + (1UL << order) - 1 > limit)
order--; order--;
/* THP machinery does not support order-1 */
if (order == 1)
order = 0;
err = ra_alloc_folio(ractl, index, mark, order, gfp); err = ra_alloc_folio(ractl, index, mark, order, gfp);
if (err) if (err)
break; break;