nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

mm/slab: make cache_grow() handle the page allocated on arbitrary node

Browse source 
Currently, cache_grow() assumes that allocated page's nodeid would be
same with parameter nodeid which is used for allocation request.  If we
discard this assumption, we can handle fallback_alloc() case gracefully.
So, this patch makes cache_grow() handle the page allocated on arbitrary
node and clean-up relevant code.

Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Joonsoo Kim 2016-05-19 17:10:23 -07:00 committed by Linus Torvalds
parent 03d1d43a12
commit 511e3a0588
1 changed files with 21 additions and 39 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

60
mm/slab.c
View file

@ -2556,13 +2556,14 @@ static void slab_map_pages(struct kmem_cache *cache, struct page *page,
* Grow (by 1) the number of slabs within a cache. This is called by * Grow (by 1) the number of slabs within a cache. This is called by
* kmem_cache_alloc() when there are no active objs left in a cache. * kmem_cache_alloc() when there are no active objs left in a cache.
*/ */
static int cache_grow(struct kmem_cache *cachep, static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid)
gfp_t flags, int nodeid, struct page *page)
{ {
void *freelist; void *freelist;
size_t offset; size_t offset;
gfp_t local_flags; gfp_t local_flags;
int page_node;
struct kmem_cache_node *n; struct kmem_cache_node *n;
struct page *page;
/* /*
* Be lazy and only check for valid flags here, keeping it out of the * Be lazy and only check for valid flags here, keeping it out of the
@ -2590,12 +2591,12 @@ static int cache_grow(struct kmem_cache *cachep,
* Get mem for the objs. Attempt to allocate a physical page from * Get mem for the objs. Attempt to allocate a physical page from
* 'nodeid'. * 'nodeid'.
*/ */
if (!page) page = kmem_getpages(cachep, local_flags, nodeid);
page = kmem_getpages(cachep, local_flags, nodeid);
if (!page) if (!page)
goto failed; goto failed;
n = get_node(cachep, nodeid); page_node = page_to_nid(page);
n = get_node(cachep, page_node);
/* Get colour for the slab, and cal the next value. */ /* Get colour for the slab, and cal the next value. */
n->colour_next++; n->colour_next++;
@ -2610,7 +2611,7 @@ static int cache_grow(struct kmem_cache *cachep,
/* Get slab management. */ /* Get slab management. */
freelist = alloc_slabmgmt(cachep, page, offset, freelist = alloc_slabmgmt(cachep, page, offset,
local_flags & ~GFP_CONSTRAINT_MASK, nodeid); local_flags & ~GFP_CONSTRAINT_MASK, page_node);
if (OFF_SLAB(cachep) && !freelist) if (OFF_SLAB(cachep) && !freelist)
goto opps1; goto opps1;
@ -2629,13 +2630,13 @@ static int cache_grow(struct kmem_cache *cachep,
STATS_INC_GROWN(cachep); STATS_INC_GROWN(cachep);
n->free_objects += cachep->num; n->free_objects += cachep->num;
spin_unlock(&n->list_lock); spin_unlock(&n->list_lock);
return 1; return page_node;
opps1: opps1:
kmem_freepages(cachep, page); kmem_freepages(cachep, page);
failed: failed:
if (gfpflags_allow_blocking(local_flags)) if (gfpflags_allow_blocking(local_flags))
local_irq_disable(); local_irq_disable();
return 0; return -1;
} }
#if DEBUG #if DEBUG
@ -2916,14 +2917,14 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
return obj; return obj;
} }
x = cache_grow(cachep, gfp_exact_node(flags), node, NULL); x = cache_grow(cachep, gfp_exact_node(flags), node);
/* cache_grow can reenable interrupts, then ac could change. */ /* cache_grow can reenable interrupts, then ac could change. */
ac = cpu_cache_get(cachep); ac = cpu_cache_get(cachep);
node = numa_mem_id(); node = numa_mem_id();
/* no objects in sight? abort */ /* no objects in sight? abort */
if (!x && ac->avail == 0) if (x < 0 && ac->avail == 0)
return NULL; return NULL;
if (!ac->avail) /* objects refilled by interrupt? */ if (!ac->avail) /* objects refilled by interrupt? */
@ -3052,7 +3053,6 @@ static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
{ {
struct zonelist *zonelist; struct zonelist *zonelist;
gfp_t local_flags;
struct zoneref *z; struct zoneref *z;
struct zone *zone; struct zone *zone;
enum zone_type high_zoneidx = gfp_zone(flags); enum zone_type high_zoneidx = gfp_zone(flags);
@ -3063,8 +3063,6 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
if (flags & __GFP_THISNODE) if (flags & __GFP_THISNODE)
return NULL; return NULL;
local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
retry_cpuset: retry_cpuset:
cpuset_mems_cookie = read_mems_allowed_begin(); cpuset_mems_cookie = read_mems_allowed_begin();
zonelist = node_zonelist(mempolicy_slab_node(), flags); zonelist = node_zonelist(mempolicy_slab_node(), flags);
@ -3094,33 +3092,17 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
* We may trigger various forms of reclaim on the allowed * We may trigger various forms of reclaim on the allowed
* set and go into memory reserves if necessary. * set and go into memory reserves if necessary.
*/ */
struct page *page; nid = cache_grow(cache, flags, numa_mem_id());
if (nid >= 0) {
obj = ____cache_alloc_node(cache,
gfp_exact_node(flags), nid);
if (gfpflags_allow_blocking(local_flags))
local_irq_enable();
kmem_flagcheck(cache, flags);
page = kmem_getpages(cache, local_flags, numa_mem_id());
if (gfpflags_allow_blocking(local_flags))
local_irq_disable();
if (page) {
/* /*
* Insert into the appropriate per node queues * Another processor may allocate the objects in
* the slab since we are not holding any locks.
*/ */
nid = page_to_nid(page); if (!obj)
if (cache_grow(cache, flags, nid, page)) { goto retry;
obj = ____cache_alloc_node(cache,
gfp_exact_node(flags), nid);
if (!obj)
/*
* Another processor may allocate the
* objects in the slab since we are
* not holding any locks.
*/
goto retry;
} else {
/* cache_grow already freed obj */
obj = NULL;
}
} }
} }
@ -3171,8 +3153,8 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
must_grow: must_grow:
spin_unlock(&n->list_lock); spin_unlock(&n->list_lock);
x = cache_grow(cachep, gfp_exact_node(flags), nodeid, NULL); x = cache_grow(cachep, gfp_exact_node(flags), nodeid);
if (x) if (x >= 0)
goto retry; goto retry;
return fallback_alloc(cachep, flags); return fallback_alloc(cachep, flags);