nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

mempool: hook up to memory allocation profiling

Browse source 
This adds hooks to mempools for correctly annotating mempool-backed
allocations at the correct source line, so they show up correctly in
/sys/kernel/debug/allocations.

Various inline functions are converted to wrappers so that we can invoke
alloc_hooks() in fewer places.

[surenb@google.com: undo _noprof additions in the documentation]
  Link: https://lkml.kernel.org/r/20240326231453.1206227-4-surenb@google.com
[surenb@google.com: add missing mempool_create_node documentation]
  Link: https://lkml.kernel.org/r/20240402180835.1661905-1-surenb@google.com
Link: https://lkml.kernel.org/r/20240321163705.3067592-27-surenb@google.com
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Tested-by: Kees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kent Overstreet 2024-03-21 09:36:48 -07:00 committed by Andrew Morton
parent 7bd230a266
commit e26d8769da
2 changed files with 50 additions and 59 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

73
include/linux/mempool.h
View file

@ -5,6 +5,8 @@
#ifndef _LINUX_MEMPOOL_H #ifndef _LINUX_MEMPOOL_H
#define _LINUX_MEMPOOL_H #define _LINUX_MEMPOOL_H
#include <linux/sched.h>
#include <linux/alloc_tag.h>
#include <linux/wait.h> #include <linux/wait.h>
#include <linux/compiler.h> #include <linux/compiler.h>
@ -39,18 +41,32 @@ void mempool_exit(mempool_t *pool);
int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data, mempool_free_t *free_fn, void *pool_data,
gfp_t gfp_mask, int node_id); gfp_t gfp_mask, int node_id);
int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data); mempool_free_t *free_fn, void *pool_data);
#define mempool_init(...) \
alloc_hooks(mempool_init_noprof(__VA_ARGS__))
extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data); mempool_free_t *free_fn, void *pool_data);
extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
extern mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data, mempool_free_t *free_fn, void *pool_data,
gfp_t gfp_mask, int nid); gfp_t gfp_mask, int nid);
#define mempool_create_node(...) \
alloc_hooks(mempool_create_node_noprof(__VA_ARGS__))
#define mempool_create(_min_nr, _alloc_fn, _free_fn, _pool_data) \
mempool_create_node(_min_nr, _alloc_fn, _free_fn, _pool_data, \
GFP_KERNEL, NUMA_NO_NODE)
extern int mempool_resize(mempool_t *pool, int new_min_nr); extern int mempool_resize(mempool_t *pool, int new_min_nr);
extern void mempool_destroy(mempool_t *pool); extern void mempool_destroy(mempool_t *pool);
extern void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) __malloc;
extern void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask) __malloc;
#define mempool_alloc(...) \
alloc_hooks(mempool_alloc_noprof(__VA_ARGS__))
extern void *mempool_alloc_preallocated(mempool_t *pool) __malloc; extern void *mempool_alloc_preallocated(mempool_t *pool) __malloc;
extern void mempool_free(void *element, mempool_t *pool); extern void mempool_free(void *element, mempool_t *pool);
@ -62,19 +78,10 @@ extern void mempool_free(void *element, mempool_t *pool);
void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data); void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data);
void mempool_free_slab(void *element, void *pool_data); void mempool_free_slab(void *element, void *pool_data);
static inline int #define mempool_init_slab_pool(_pool, _min_nr, _kc) \
mempool_init_slab_pool(mempool_t *pool, int min_nr, struct kmem_cache *kc) mempool_init(_pool, (_min_nr), mempool_alloc_slab, mempool_free_slab, (void *)(_kc))
{ #define mempool_create_slab_pool(_min_nr, _kc) \
return mempool_init(pool, min_nr, mempool_alloc_slab, mempool_create((_min_nr), mempool_alloc_slab, mempool_free_slab, (void *)(_kc))
mempool_free_slab, (void *) kc);
}
static inline mempool_t *
mempool_create_slab_pool(int min_nr, struct kmem_cache *kc)
{
return mempool_create(min_nr, mempool_alloc_slab, mempool_free_slab,
(void *) kc);
}
/* /*
* a mempool_alloc_t and a mempool_free_t to kmalloc and kfree the * a mempool_alloc_t and a mempool_free_t to kmalloc and kfree the
@ -83,17 +90,12 @@ mempool_create_slab_pool(int min_nr, struct kmem_cache *kc)
void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data); void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data);
void mempool_kfree(void *element, void *pool_data); void mempool_kfree(void *element, void *pool_data);
static inline int mempool_init_kmalloc_pool(mempool_t *pool, int min_nr, size_t size) #define mempool_init_kmalloc_pool(_pool, _min_nr, _size) \
{ mempool_init(_pool, (_min_nr), mempool_kmalloc, mempool_kfree, \
return mempool_init(pool, min_nr, mempool_kmalloc, (void *)(unsigned long)(_size))
mempool_kfree, (void *) size); #define mempool_create_kmalloc_pool(_min_nr, _size) \
} mempool_create((_min_nr), mempool_kmalloc, mempool_kfree, \
(void *)(unsigned long)(_size))
static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size)
{
return mempool_create(min_nr, mempool_kmalloc, mempool_kfree,
(void *) size);
}
void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data); void *mempool_kvmalloc(gfp_t gfp_mask, void *pool_data);
void mempool_kvfree(void *element, void *pool_data); void mempool_kvfree(void *element, void *pool_data);
@ -115,16 +117,11 @@ static inline mempool_t *mempool_create_kvmalloc_pool(int min_nr, size_t size)
void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data); void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data);
void mempool_free_pages(void *element, void *pool_data); void mempool_free_pages(void *element, void *pool_data);
static inline int mempool_init_page_pool(mempool_t *pool, int min_nr, int order) #define mempool_init_page_pool(_pool, _min_nr, _order) \
{ mempool_init(_pool, (_min_nr), mempool_alloc_pages, \
return mempool_init(pool, min_nr, mempool_alloc_pages, mempool_free_pages, (void *)(long)(_order))
mempool_free_pages, (void *)(long)order); #define mempool_create_page_pool(_min_nr, _order) \
} mempool_create((_min_nr), mempool_alloc_pages, \
mempool_free_pages, (void *)(long)(_order))
static inline mempool_t *mempool_create_page_pool(int min_nr, int order)
{
return mempool_create(min_nr, mempool_alloc_pages, mempool_free_pages,
(void *)(long)order);
}
#endif /* _LINUX_MEMPOOL_H */ #endif /* _LINUX_MEMPOOL_H */

36
mm/mempool.c
View file

@ -240,22 +240,24 @@ EXPORT_SYMBOL(mempool_init_node);
* *
* Return: %0 on success, negative error code otherwise. * Return: %0 on success, negative error code otherwise.
*/ */
int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn, int mempool_init_noprof(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data) mempool_free_t *free_fn, void *pool_data)
{ {
return mempool_init_node(pool, min_nr, alloc_fn, free_fn, return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
pool_data, GFP_KERNEL, NUMA_NO_NODE); pool_data, GFP_KERNEL, NUMA_NO_NODE);
} }
EXPORT_SYMBOL(mempool_init); EXPORT_SYMBOL(mempool_init_noprof);
/** /**
* mempool_create - create a memory pool * mempool_create_node - create a memory pool
* @min_nr: the minimum number of elements guaranteed to be * @min_nr: the minimum number of elements guaranteed to be
* allocated for this pool. * allocated for this pool.
* @alloc_fn: user-defined element-allocation function. * @alloc_fn: user-defined element-allocation function.
* @free_fn: user-defined element-freeing function. * @free_fn: user-defined element-freeing function.
* @pool_data: optional private data available to the user-defined functions. * @pool_data: optional private data available to the user-defined functions.
* @gfp_mask: memory allocation flags
* @node_id: numa node to allocate on
* *
* this function creates and allocates a guaranteed size, preallocated * this function creates and allocates a guaranteed size, preallocated
* memory pool. The pool can be used from the mempool_alloc() and mempool_free() * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
@ -265,17 +267,9 @@ EXPORT_SYMBOL(mempool_init);
* *
* Return: pointer to the created memory pool object or %NULL on error. * Return: pointer to the created memory pool object or %NULL on error.
*/ */
mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, mempool_t *mempool_create_node_noprof(int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data) mempool_free_t *free_fn, void *pool_data,
{ gfp_t gfp_mask, int node_id)
return mempool_create_node(min_nr, alloc_fn, free_fn, pool_data,
GFP_KERNEL, NUMA_NO_NODE);
}
EXPORT_SYMBOL(mempool_create);
mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data,
gfp_t gfp_mask, int node_id)
{ {
mempool_t *pool; mempool_t *pool;
@ -291,7 +285,7 @@ mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
return pool; return pool;
} }
EXPORT_SYMBOL(mempool_create_node); EXPORT_SYMBOL(mempool_create_node_noprof);
/** /**
* mempool_resize - resize an existing memory pool * mempool_resize - resize an existing memory pool
@ -387,7 +381,7 @@ EXPORT_SYMBOL(mempool_resize);
* *
* Return: pointer to the allocated element or %NULL on error. * Return: pointer to the allocated element or %NULL on error.
*/ */
void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask) void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)
{ {
void *element; void *element;
unsigned long flags; unsigned long flags;
@ -454,7 +448,7 @@ void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
finish_wait(&pool->wait, &wait); finish_wait(&pool->wait, &wait);
goto repeat_alloc; goto repeat_alloc;
} }
EXPORT_SYMBOL(mempool_alloc); EXPORT_SYMBOL(mempool_alloc_noprof);
/** /**
* mempool_alloc_preallocated - allocate an element from preallocated elements * mempool_alloc_preallocated - allocate an element from preallocated elements
@ -562,7 +556,7 @@ void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
{ {
struct kmem_cache *mem = pool_data; struct kmem_cache *mem = pool_data;
VM_BUG_ON(mem->ctor); VM_BUG_ON(mem->ctor);
return kmem_cache_alloc(mem, gfp_mask); return kmem_cache_alloc_noprof(mem, gfp_mask);
} }
EXPORT_SYMBOL(mempool_alloc_slab); EXPORT_SYMBOL(mempool_alloc_slab);
@ -580,7 +574,7 @@ EXPORT_SYMBOL(mempool_free_slab);
void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
{ {
size_t size = (size_t)pool_data; size_t size = (size_t)pool_data;
return kmalloc(size, gfp_mask); return kmalloc_noprof(size, gfp_mask);
} }
EXPORT_SYMBOL(mempool_kmalloc); EXPORT_SYMBOL(mempool_kmalloc);
@ -610,7 +604,7 @@ EXPORT_SYMBOL(mempool_kvfree);
void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
{ {
int order = (int)(long)pool_data; int order = (int)(long)pool_data;
return alloc_pages(gfp_mask, order); return alloc_pages_noprof(gfp_mask, order);
} }
EXPORT_SYMBOL(mempool_alloc_pages); EXPORT_SYMBOL(mempool_alloc_pages);