nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

mm/page_alloc: simplify how many pages are selected per pcp list during bulk free

Browse source 
free_pcppages_bulk() selects pages to free by round-robining between
lists.  Originally this was to evenly shrink pages by migratetype but
uneven freeing is inevitable due to high pages.  Simplify list selection
by starting with a list that definitely has pages on it in
free_unref_page_commit() and for drain, it does not matter where
draining starts as all pages are removed.

Link: https://lkml.kernel.org/r/20220217002227.5739-4-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Tested-by: Aaron Lu <aaron.lu@intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Mel Gorman 2022-03-22 14:43:36 -07:00 committed by Linus Torvalds
parent 35b6d770e6
commit fd56eef258
1 changed files with 11 additions and 23 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

34
mm/page_alloc.c
View file

@ -1444,13 +1444,11 @@ static inline void prefetch_buddy(struct page *page, unsigned int order)
* count is the number of pages to free. * count is the number of pages to free.
*/ */
static void free_pcppages_bulk(struct zone *zone, int count, static void free_pcppages_bulk(struct zone *zone, int count,
struct per_cpu_pages *pcp) struct per_cpu_pages *pcp,
int pindex)
{ {
int pindex = 0;
int min_pindex = 0; int min_pindex = 0;
int max_pindex = NR_PCP_LISTS - 1; int max_pindex = NR_PCP_LISTS - 1;
int batch_free = 0;
int nr_freed = 0;
unsigned int order; unsigned int order;
int prefetch_nr = READ_ONCE(pcp->batch); int prefetch_nr = READ_ONCE(pcp->batch);
bool isolated_pageblocks; bool isolated_pageblocks;
@ -1464,16 +1462,10 @@ static void free_pcppages_bulk(struct zone *zone, int count,
count = min(pcp->count, count); count = min(pcp->count, count);
while (count > 0) { while (count > 0) {
struct list_head *list; struct list_head *list;
int nr_pages;
/* /* Remove pages from lists in a round-robin fashion. */
* Remove pages from lists in a round-robin fashion. A
* batch_free count is maintained that is incremented when an
* empty list is encountered. This is so more pages are freed
* off fuller lists instead of spinning excessively around empty
* lists
*/
do { do {
batch_free++;
if (++pindex > max_pindex) if (++pindex > max_pindex)
pindex = min_pindex; pindex = min_pindex;
list = &pcp->lists[pindex]; list = &pcp->lists[pindex];
@ -1486,18 +1478,15 @@ static void free_pcppages_bulk(struct zone *zone, int count,
min_pindex++; min_pindex++;
} while (1); } while (1);
/* This is the only non-empty list. Free them all. */
if (batch_free >= max_pindex - min_pindex)
batch_free = count;
order = pindex_to_order(pindex); order = pindex_to_order(pindex);
nr_pages = 1 << order;
BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH)); BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH));
do { do {
page = list_last_entry(list, struct page, lru); page = list_last_entry(list, struct page, lru);
/* must delete to avoid corrupting pcp list */ /* must delete to avoid corrupting pcp list */
list_del(&page->lru); list_del(&page->lru);
nr_freed += 1 << order; count -= nr_pages;
count -= 1 << order; pcp->count -= nr_pages;
if (bulkfree_pcp_prepare(page)) if (bulkfree_pcp_prepare(page))
continue; continue;
@ -1521,9 +1510,8 @@ static void free_pcppages_bulk(struct zone *zone, int count,
prefetch_buddy(page, order); prefetch_buddy(page, order);
prefetch_nr--; prefetch_nr--;
} }
} while (count > 0 && --batch_free && !list_empty(list)); } while (count > 0 && !list_empty(list));
} }
pcp->count -= nr_freed;
/* /*
* local_lock_irq held so equivalent to spin_lock_irqsave for * local_lock_irq held so equivalent to spin_lock_irqsave for
@ -3077,7 +3065,7 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
batch = READ_ONCE(pcp->batch); batch = READ_ONCE(pcp->batch);
to_drain = min(pcp->count, batch); to_drain = min(pcp->count, batch);
if (to_drain > 0) if (to_drain > 0)
free_pcppages_bulk(zone, to_drain, pcp); free_pcppages_bulk(zone, to_drain, pcp, 0);
local_unlock_irqrestore(&pagesets.lock, flags); local_unlock_irqrestore(&pagesets.lock, flags);
} }
#endif #endif
@ -3098,7 +3086,7 @@ static void drain_pages_zone(unsigned int cpu, struct zone *zone)
pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu); pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
if (pcp->count) if (pcp->count)
free_pcppages_bulk(zone, pcp->count, pcp); free_pcppages_bulk(zone, pcp->count, pcp, 0);
local_unlock_irqrestore(&pagesets.lock, flags); local_unlock_irqrestore(&pagesets.lock, flags);
} }
@ -3379,7 +3367,7 @@ static void free_unref_page_commit(struct page *page, int migratetype,
if (pcp->count >= high) { if (pcp->count >= high) {
int batch = READ_ONCE(pcp->batch); int batch = READ_ONCE(pcp->batch);
free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp); free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp, pindex);
} }
} }