mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

page_pool: disable direct recycling based on pool->cpuid on destroy

Browse source 
Now that direct recycling is performed basing on pool->cpuid when set,
memory leaks are possible:

1. A pool is destroyed.
2. Alloc cache is emptied (it's done only once).
3. pool->cpuid is still set.
4. napi_pp_put_page() does direct recycling basing on pool->cpuid.
5. Now alloc cache is not empty, but it won't ever be freed.

In order to avoid that, rewrite pool->cpuid to -1 when unlinking NAPI to
make sure no direct recycling will be possible after emptying the cache.
This involves a bit of overhead as pool->cpuid now must be accessed
via READ_ONCE() to avoid partial reads.
Rename page_pool_unlink_napi() -> page_pool_disable_direct_recycling()
to reflect what it actually does and unexport it.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Link: https://lore.kernel.org/r/20240215113905.96817-1-aleksander.lobakin@intel.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Alexander Lobakin 2024-02-15 12:39:05 +01:00 committed by Jakub Kicinski
parent 5983e5df86
commit 56ef27e3ab
3 changed files with 8 additions and 9 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

5
include/net/page_pool/types.h
View file

@ -210,17 +210,12 @@ struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,
struct xdp_mem_info; struct xdp_mem_info;
#ifdef CONFIG_PAGE_POOL #ifdef CONFIG_PAGE_POOL
void page_pool_unlink_napi(struct page_pool *pool);
void page_pool_destroy(struct page_pool *pool); void page_pool_destroy(struct page_pool *pool);
void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
struct xdp_mem_info *mem); struct xdp_mem_info *mem);
void page_pool_put_page_bulk(struct page_pool *pool, void **data, void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count); int count);
#else #else
static inline void page_pool_unlink_napi(struct page_pool *pool)
{
}
static inline void page_pool_destroy(struct page_pool *pool) static inline void page_pool_destroy(struct page_pool *pool)
{ {
} }

10
net/core/page_pool.c
View file

@ -949,8 +949,13 @@ void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
pool->xdp_mem_id = mem->id; pool->xdp_mem_id = mem->id;
} }
void page_pool_unlink_napi(struct page_pool *pool) static void page_pool_disable_direct_recycling(struct page_pool *pool)
{ {
/* Disable direct recycling based on pool->cpuid.
* Paired with READ_ONCE() in napi_pp_put_page().
*/
WRITE_ONCE(pool->cpuid, -1);
if (!pool->p.napi) if (!pool->p.napi)
return; return;
@ -962,7 +967,6 @@ void page_pool_unlink_napi(struct page_pool *pool)
WRITE_ONCE(pool->p.napi, NULL); WRITE_ONCE(pool->p.napi, NULL);
} }
EXPORT_SYMBOL(page_pool_unlink_napi);
void page_pool_destroy(struct page_pool *pool) void page_pool_destroy(struct page_pool *pool)
{ {
@ -972,7 +976,7 @@ void page_pool_destroy(struct page_pool *pool)
if (!page_pool_put(pool)) if (!page_pool_put(pool))
return; return;
page_pool_unlink_napi(pool); page_pool_disable_direct_recycling(pool);
page_pool_free_frag(pool); page_pool_free_frag(pool);
if (!page_pool_release(pool)) if (!page_pool_release(pool))

2
net/core/skbuff.c
View file

@ -1018,7 +1018,7 @@ bool napi_pp_put_page(struct page *page, bool napi_safe)
unsigned int cpuid = smp_processor_id(); unsigned int cpuid = smp_processor_id();
allow_direct = napi && READ_ONCE(napi->list_owner) == cpuid; allow_direct = napi && READ_ONCE(napi->list_owner) == cpuid;
allow_direct |= (pp->cpuid == cpuid); allow_direct |= READ_ONCE(pp->cpuid) == cpuid;
} }
/* Driver set this to memory recycling info. Reset it on recycle. /* Driver set this to memory recycling info. Reset it on recycle.