nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

mm, page_alloc: avoid looking up the first zone in a zonelist twice

Browse source 
The allocator fast path looks up the first usable zone in a zonelist and
then get_page_from_freelist does the same job in the zonelist iterator.
This patch preserves the necessary information.

                                             4.6.0-rc2                  4.6.0-rc2
                                        fastmark-v1r20             initonce-v1r20
  Min      alloc-odr0-1               364.00 (  0.00%)           359.00 (  1.37%)
  Min      alloc-odr0-2               262.00 (  0.00%)           260.00 (  0.76%)
  Min      alloc-odr0-4               214.00 (  0.00%)           214.00 (  0.00%)
  Min      alloc-odr0-8               186.00 (  0.00%)           186.00 (  0.00%)
  Min      alloc-odr0-16              173.00 (  0.00%)           173.00 (  0.00%)
  Min      alloc-odr0-32              165.00 (  0.00%)           165.00 (  0.00%)
  Min      alloc-odr0-64              161.00 (  0.00%)           162.00 ( -0.62%)
  Min      alloc-odr0-128             159.00 (  0.00%)           161.00 ( -1.26%)
  Min      alloc-odr0-256             168.00 (  0.00%)           170.00 ( -1.19%)
  Min      alloc-odr0-512             180.00 (  0.00%)           181.00 ( -0.56%)
  Min      alloc-odr0-1024            190.00 (  0.00%)           190.00 (  0.00%)
  Min      alloc-odr0-2048            196.00 (  0.00%)           196.00 (  0.00%)
  Min      alloc-odr0-4096            202.00 (  0.00%)           202.00 (  0.00%)
  Min      alloc-odr0-8192            206.00 (  0.00%)           205.00 (  0.49%)
  Min      alloc-odr0-16384           206.00 (  0.00%)           205.00 (  0.49%)

The benefit is negligible and the results are within the noise but each
cycle counts.

Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Mel Gorman 2016-05-19 17:14:10 -07:00 committed by Linus Torvalds
parent 48ee5f3696
commit c33d6c06f6
5 changed files with 43 additions and 40 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

10
fs/buffer.c
View file

@ -255,17 +255,17 @@ __find_get_block_slow(struct block_device *bdev, sector_t block)
*/ */
static void free_more_memory(void) static void free_more_memory(void)
{ {
struct zone *zone; struct zoneref *z;
int nid; int nid;
wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM); wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
yield(); yield();
for_each_online_node(nid) { for_each_online_node(nid) {
(void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
gfp_zone(GFP_NOFS), NULL, z = first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
&zone); gfp_zone(GFP_NOFS), NULL);
if (zone) if (z->zone)
try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0, try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
GFP_NOFS, NULL); GFP_NOFS, NULL);
} }

18
include/linux/mmzone.h
View file

@ -959,13 +959,10 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
*/ */
static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
enum zone_type highest_zoneidx, enum zone_type highest_zoneidx,
nodemask_t *nodes, nodemask_t *nodes)
struct zone **zone)
{ {
struct zoneref *z = next_zones_zonelist(zonelist->_zonerefs, return next_zones_zonelist(zonelist->_zonerefs,
highest_zoneidx, nodes); highest_zoneidx, nodes);
*zone = zonelist_zone(z);
return z;
} }
/** /**
@ -980,10 +977,17 @@ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
* within a given nodemask * within a given nodemask
*/ */
#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \ #define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
for (z = first_zones_zonelist(zlist, highidx, nodemask, &zone); \ for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \
zone; \ zone; \
z = next_zones_zonelist(++z, highidx, nodemask), \ z = next_zones_zonelist(++z, highidx, nodemask), \
zone = zonelist_zone(z)) \ zone = zonelist_zone(z))
#define for_next_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
for (zone = z->zone; \
zone; \
z = next_zones_zonelist(++z, highidx, nodemask), \
zone = zonelist_zone(z))
/** /**
* for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index

2
mm/internal.h
View file

@ -102,7 +102,7 @@ extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
struct alloc_context { struct alloc_context {
struct zonelist *zonelist; struct zonelist *zonelist;
nodemask_t *nodemask; nodemask_t *nodemask;
struct zone *preferred_zone; struct zoneref *preferred_zoneref;
int classzone_idx; int classzone_idx;
int migratetype; int migratetype;
enum zone_type high_zoneidx; enum zone_type high_zoneidx;

19
mm/mempolicy.c
View file

@ -1739,18 +1739,18 @@ unsigned int mempolicy_slab_node(void)
return interleave_nodes(policy); return interleave_nodes(policy);
case MPOL_BIND: { case MPOL_BIND: {
struct zoneref *z;
/* /*
* Follow bind policy behavior and start allocation at the * Follow bind policy behavior and start allocation at the
* first node. * first node.
*/ */
struct zonelist *zonelist; struct zonelist *zonelist;
struct zone *zone;
enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL); enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
zonelist = &NODE_DATA(node)->node_zonelists[0]; zonelist = &NODE_DATA(node)->node_zonelists[0];
(void)first_zones_zonelist(zonelist, highest_zoneidx, z = first_zones_zonelist(zonelist, highest_zoneidx,
&policy->v.nodes, &policy->v.nodes);
&zone); return z->zone ? z->zone->node : node;
return zone ? zone->node : node;
} }
default: default:
@ -2266,7 +2266,7 @@ static void sp_free(struct sp_node *n)
int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr) int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
{ {
struct mempolicy *pol; struct mempolicy *pol;
struct zone *zone; struct zoneref *z;
int curnid = page_to_nid(page); int curnid = page_to_nid(page);
unsigned long pgoff; unsigned long pgoff;
int thiscpu = raw_smp_processor_id(); int thiscpu = raw_smp_processor_id();
@ -2298,6 +2298,7 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
break; break;
case MPOL_BIND: case MPOL_BIND:
/* /*
* allows binding to multiple nodes. * allows binding to multiple nodes.
* use current page if in policy nodemask, * use current page if in policy nodemask,
@ -2306,11 +2307,11 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
*/ */
if (node_isset(curnid, pol->v.nodes)) if (node_isset(curnid, pol->v.nodes))
goto out; goto out;
(void)first_zones_zonelist( z = first_zones_zonelist(
node_zonelist(numa_node_id(), GFP_HIGHUSER), node_zonelist(numa_node_id(), GFP_HIGHUSER),
gfp_zone(GFP_HIGHUSER), gfp_zone(GFP_HIGHUSER),
&pol->v.nodes, &zone); &pol->v.nodes);
polnid = zone->node; polnid = z->zone->node;
break; break;
default: default:

34
mm/page_alloc.c
View file

@ -2704,7 +2704,7 @@ static struct page *
get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
const struct alloc_context *ac) const struct alloc_context *ac)
{ {
struct zoneref *z; struct zoneref *z = ac->preferred_zoneref;
struct zone *zone; struct zone *zone;
bool fair_skipped = false; bool fair_skipped = false;
bool apply_fair = (alloc_flags & ALLOC_FAIR); bool apply_fair = (alloc_flags & ALLOC_FAIR);
@ -2714,7 +2714,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
* Scan zonelist, looking for a zone with enough free. * Scan zonelist, looking for a zone with enough free.
* See also __cpuset_node_allowed() comment in kernel/cpuset.c. * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
*/ */
for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx, for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
ac->nodemask) { ac->nodemask) {
struct page *page; struct page *page;
unsigned long mark; unsigned long mark;
@ -2734,7 +2734,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
fair_skipped = true; fair_skipped = true;
continue; continue;
} }
if (!zone_local(ac->preferred_zone, zone)) { if (!zone_local(ac->preferred_zoneref->zone, zone)) {
if (fair_skipped) if (fair_skipped)
goto reset_fair; goto reset_fair;
apply_fair = false; apply_fair = false;
@ -2780,7 +2780,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
goto try_this_zone; goto try_this_zone;
if (zone_reclaim_mode == 0 || if (zone_reclaim_mode == 0 ||
!zone_allows_reclaim(ac->preferred_zone, zone)) !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
continue; continue;
ret = zone_reclaim(zone, gfp_mask, order); ret = zone_reclaim(zone, gfp_mask, order);
@ -2802,7 +2802,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
} }
try_this_zone: try_this_zone:
page = buffered_rmqueue(ac->preferred_zone, zone, order, page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order,
gfp_mask, alloc_flags, ac->migratetype); gfp_mask, alloc_flags, ac->migratetype);
if (page) { if (page) {
if (prep_new_page(page, order, gfp_mask, alloc_flags)) if (prep_new_page(page, order, gfp_mask, alloc_flags))
@ -2831,7 +2831,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
reset_fair: reset_fair:
apply_fair = false; apply_fair = false;
fair_skipped = false; fair_skipped = false;
reset_alloc_batches(ac->preferred_zone); reset_alloc_batches(ac->preferred_zoneref->zone);
goto zonelist_scan; goto zonelist_scan;
} }
@ -3114,7 +3114,7 @@ static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
ac->high_zoneidx, ac->nodemask) ac->high_zoneidx, ac->nodemask)
wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone)); wakeup_kswapd(zone, order, zonelist_zone_idx(ac->preferred_zoneref));
} }
static inline unsigned int static inline unsigned int
@ -3332,7 +3332,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
if ((did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) || if ((did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) ||
((gfp_mask & __GFP_REPEAT) && pages_reclaimed < (1 << order))) { ((gfp_mask & __GFP_REPEAT) && pages_reclaimed < (1 << order))) {
/* Wait for some write requests to complete then retry */ /* Wait for some write requests to complete then retry */
wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50); wait_iff_congested(ac->preferred_zoneref->zone, BLK_RW_ASYNC, HZ/50);
goto retry; goto retry;
} }
@ -3370,7 +3370,6 @@ struct page *
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
struct zonelist *zonelist, nodemask_t *nodemask) struct zonelist *zonelist, nodemask_t *nodemask)
{ {
struct zoneref *preferred_zoneref;
struct page *page; struct page *page;
unsigned int cpuset_mems_cookie; unsigned int cpuset_mems_cookie;
unsigned int alloc_flags = ALLOC_WMARK_LOW|ALLOC_FAIR; unsigned int alloc_flags = ALLOC_WMARK_LOW|ALLOC_FAIR;
@ -3416,14 +3415,14 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE); ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE);
/* The preferred zone is used for statistics later */ /* The preferred zone is used for statistics later */
preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx, ac.preferred_zoneref = first_zones_zonelist(ac.zonelist,
ac.nodemask, &ac.preferred_zone); ac.high_zoneidx, ac.nodemask);
if (!ac.preferred_zone) { if (!ac.preferred_zoneref) {
page = NULL; page = NULL;
goto no_zone; goto no_zone;
} }
ac.classzone_idx = zonelist_zone_idx(preferred_zoneref); ac.classzone_idx = zonelist_zone_idx(ac.preferred_zoneref);
/* First allocation attempt */ /* First allocation attempt */
page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
@ -4462,13 +4461,12 @@ static void build_zonelists(pg_data_t *pgdat)
*/ */
int local_memory_node(int node) int local_memory_node(int node)
{ {
struct zone *zone; struct zoneref *z;
(void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL), z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
gfp_zone(GFP_KERNEL), gfp_zone(GFP_KERNEL),
NULL, NULL);
&zone); return z->zone->node;
return zone->node;
} }
#endif #endif