mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

mm: kmemleak: use the memory pool for early allocations

Browse source 
Currently kmemleak uses a static early_log buffer to trace all memory
allocation/freeing before the slab allocator is initialised.  Such early
log is replayed during kmemleak_init() to properly initialise the kmemleak
metadata for objects allocated up that point.  With a memory pool that
does not rely on the slab allocator, it is possible to skip this early log
entirely.

In order to remove the early logging, consider kmemleak_enabled == 1 by
default while the kmem_cache availability is checked directly on the
object_cache and scan_area_cache variables.  The RCU callback is only
invoked after object_cache has been initialised as we wouldn't have any
concurrent list traversal before this.

In order to reduce the number of callbacks before kmemleak is fully
initialised, move the kmemleak_init() call to mm_init().

[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: remove WARN_ON(), per Catalin]
Link: http://lkml.kernel.org/r/20190812160642.52134-4-catalin.marinas@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Qian Cai <cai@lca.pw>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Catalin Marinas 2019-09-23 15:34:05 -07:00 committed by Linus Torvalds
parent 0647398a8c
commit c566586818
3 changed files with 33 additions and 245 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

2
init/main.c
View file

@ -556,6 +556,7 @@ static void __init mm_init(void)
report_meminit();
mem_init();
kmem_cache_init();
kmemleak_init();
pgtable_init();
debug_objects_mem_init();
vmalloc_init();
@ -740,7 +741,6 @@ asmlinkage __visible void __init start_kernel(void)
initrd_start = 0;
}
#endif
kmemleak_init();
setup_per_cpu_pageset();
numa_policy_init();
acpi_early_init();

11
lib/Kconfig.debug
View file

@ -576,17 +576,18 @@ config DEBUG_KMEMLEAK
In order to access the kmemleak file, debugfs needs to be
mounted (usually at /sys/kernel/debug).
config DEBUG_KMEMLEAK_EARLY_LOG_SIZE
int "Maximum kmemleak early log entries"
config DEBUG_KMEMLEAK_MEM_POOL_SIZE
int "Kmemleak memory pool size"
depends on DEBUG_KMEMLEAK
range 200 40000
default 16000
help
Kmemleak must track all the memory allocations to avoid
reporting false positives. Since memory may be allocated or
freed before kmemleak is initialised, an early log buffer is
used to store these actions. If kmemleak reports "early log
buffer exceeded", please increase this value.
freed before kmemleak is fully initialised, use a static pool
of metadata objects to track such callbacks. After kmemleak is
fully initialised, this memory pool acts as an emergency one
if slab allocations fail.
config DEBUG_KMEMLEAK_TEST
tristate "Simple test for the kernel memory leak detector"

265
mm/kmemleak.c
View file

@ -180,15 +180,13 @@ struct kmemleak_object {
#define HEX_ASCII 1
/* max number of lines to be printed */
#define HEX_MAX_LINES 2
/* memory pool size */
#define MEM_POOL_SIZE 16000
/* the list of all allocated objects */
static LIST_HEAD(object_list);
/* the list of gray-colored objects (see color_gray comment below) */
static LIST_HEAD(gray_list);
/* memory pool allocation */
static struct kmemleak_object mem_pool[MEM_POOL_SIZE];
static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
static LIST_HEAD(mem_pool_free_list);
/* search tree for object boundaries */
@ -201,13 +199,11 @@ static struct kmem_cache *object_cache;
static struct kmem_cache *scan_area_cache;
/* set if tracing memory operations is enabled */
static int kmemleak_enabled;
static int kmemleak_enabled = 1;
/* same as above but only for the kmemleak_free() callback */
static int kmemleak_free_enabled;
static int kmemleak_free_enabled = 1;
/* set in the late_initcall if there were no errors */
static int kmemleak_initialized;
/* enables or disables early logging of the memory operations */
static int kmemleak_early_log = 1;
/* set if a kmemleak warning was issued */
static int kmemleak_warning;
/* set if a fatal kmemleak error has occurred */
@ -235,49 +231,6 @@ static bool kmemleak_found_leaks;
static bool kmemleak_verbose;
module_param_named(verbose, kmemleak_verbose, bool, 0600);
/*
* Early object allocation/freeing logging. Kmemleak is initialized after the
* kernel allocator. However, both the kernel allocator and kmemleak may
* allocate memory blocks which need to be tracked. Kmemleak defines an
* arbitrary buffer to hold the allocation/freeing information before it is
* fully initialized.
*/
/* kmemleak operation type for early logging */
enum {
KMEMLEAK_ALLOC,
KMEMLEAK_ALLOC_PERCPU,
KMEMLEAK_FREE,
KMEMLEAK_FREE_PART,
KMEMLEAK_FREE_PERCPU,
KMEMLEAK_NOT_LEAK,
KMEMLEAK_IGNORE,
KMEMLEAK_SCAN_AREA,
KMEMLEAK_NO_SCAN,
KMEMLEAK_SET_EXCESS_REF
};
/*
* Structure holding the information passed to kmemleak callbacks during the
* early logging.
*/
struct early_log {
int op_type; /* kmemleak operation type */
int min_count; /* minimum reference count */
const void *ptr; /* allocated/freed memory block */
union {
size_t size; /* memory block size */
unsigned long excess_ref; /* surplus reference passing */
};
unsigned long trace[MAX_TRACE]; /* stack trace */
unsigned int trace_len; /* stack trace length */
};
/* early logging buffer and current position */
static struct early_log
early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
static int crt_early_log __initdata;
static void kmemleak_disable(void);
/*
@ -466,9 +419,11 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
struct kmemleak_object *object;
/* try the slab allocator first */
object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
if (object)
return object;
if (object_cache) {
object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
if (object)
return object;
}
/* slab allocation failed, try the memory pool */
write_lock_irqsave(&kmemleak_lock, flags);
@ -478,6 +433,8 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
list_del(&object->object_list);
else if (mem_pool_free_count)
object = &mem_pool[--mem_pool_free_count];
else
pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
write_unlock_irqrestore(&kmemleak_lock, flags);
return object;
@ -537,7 +494,15 @@ static void put_object(struct kmemleak_object *object)
/* should only get here after delete_object was called */
WARN_ON(object->flags & OBJECT_ALLOCATED);
call_rcu(&object->rcu, free_object_rcu);
/*
* It may be too early for the RCU callbacks, however, there is no
* concurrent object_list traversal when !object_cache and all objects
* came from the memory pool. Free the object directly.
*/
if (object_cache)
call_rcu(&object->rcu, free_object_rcu);
else
free_object_rcu(&object->rcu);
}
/*
@ -741,9 +706,7 @@ static void delete_object_part(unsigned long ptr, size_t size)
/*
* Create one or two objects that may result from the memory block
* split. Note that partial freeing is only done by free_bootmem() and
* this happens before kmemleak_init() is called. The path below is
* only executed during early log recording in kmemleak_init(), so
* GFP_KERNEL is enough.
* this happens before kmemleak_init() is called.
*/
start = object->pointer;
end = object->pointer + object->size;
@ -815,7 +778,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
{
unsigned long flags;
struct kmemleak_object *object;
struct kmemleak_scan_area *area;
struct kmemleak_scan_area *area = NULL;
object = find_and_get_object(ptr, 1);
if (!object) {
@ -824,7 +787,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
return;
}
area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
if (scan_area_cache)
area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
spin_lock_irqsave(&object->lock, flags);
if (!area) {
@ -898,86 +862,6 @@ static void object_no_scan(unsigned long ptr)
put_object(object);
}
/*
* Log an early kmemleak_* call to the early_log buffer. These calls will be
* processed later once kmemleak is fully initialized.
*/
static void __init log_early(int op_type, const void *ptr, size_t size,
int min_count)
{
unsigned long flags;
struct early_log *log;
if (kmemleak_error) {
/* kmemleak stopped recording, just count the requests */
crt_early_log++;
return;
}
if (crt_early_log >= ARRAY_SIZE(early_log)) {
crt_early_log++;
kmemleak_disable();
return;
}
/*
* There is no need for locking since the kernel is still in UP mode
* at this stage. Disabling the IRQs is enough.
*/
local_irq_save(flags);
log = &early_log[crt_early_log];
log->op_type = op_type;
log->ptr = ptr;
log->size = size;
log->min_count = min_count;
log->trace_len = __save_stack_trace(log->trace);
crt_early_log++;
local_irq_restore(flags);
}
/*
* Log an early allocated block and populate the stack trace.
*/
static void early_alloc(struct early_log *log)
{
struct kmemleak_object *object;
unsigned long flags;
int i;
if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
return;
/*
* RCU locking needed to ensure object is not freed via put_object().
*/
rcu_read_lock();
object = create_object((unsigned long)log->ptr, log->size,
log->min_count, GFP_ATOMIC);
if (!object)
goto out;
spin_lock_irqsave(&object->lock, flags);
for (i = 0; i < log->trace_len; i++)
object->trace[i] = log->trace[i];
object->trace_len = log->trace_len;
spin_unlock_irqrestore(&object->lock, flags);
out:
rcu_read_unlock();
}
/*
* Log an early allocated block and populate the stack trace.
*/
static void early_alloc_percpu(struct early_log *log)
{
unsigned int cpu;
const void __percpu *ptr = log->ptr;
for_each_possible_cpu(cpu) {
log->ptr = per_cpu_ptr(ptr, cpu);
early_alloc(log);
}
}
/**
* kmemleak_alloc - register a newly allocated object
* @ptr: pointer to beginning of the object
@ -999,8 +883,6 @@ void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
create_object((unsigned long)ptr, size, min_count, gfp);
else if (kmemleak_early_log)
log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
}
EXPORT_SYMBOL_GPL(kmemleak_alloc);
@ -1028,8 +910,6 @@ void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
for_each_possible_cpu(cpu)
create_object((unsigned long)per_cpu_ptr(ptr, cpu),
size, 0, gfp);
else if (kmemleak_early_log)
log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
}
EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
@ -1054,11 +934,6 @@ void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp
create_object((unsigned long)area->addr, size, 2, gfp);
object_set_excess_ref((unsigned long)area,
(unsigned long)area->addr);
} else if (kmemleak_early_log) {
log_early(KMEMLEAK_ALLOC, area->addr, size, 2);
/* reusing early_log.size for storing area->addr */
log_early(KMEMLEAK_SET_EXCESS_REF,
area, (unsigned long)area->addr, 0);
}
}
EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
@ -1076,8 +951,6 @@ void __ref kmemleak_free(const void *ptr)
if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
delete_object_full((unsigned long)ptr);
else if (kmemleak_early_log)
log_early(KMEMLEAK_FREE, ptr, 0, 0);
}
EXPORT_SYMBOL_GPL(kmemleak_free);
@ -1096,8 +969,6 @@ void __ref kmemleak_free_part(const void *ptr, size_t size)
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
delete_object_part((unsigned long)ptr, size);
else if (kmemleak_early_log)
log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
}
EXPORT_SYMBOL_GPL(kmemleak_free_part);
@ -1118,8 +989,6 @@ void __ref kmemleak_free_percpu(const void __percpu *ptr)
for_each_possible_cpu(cpu)
delete_object_full((unsigned long)per_cpu_ptr(ptr,
cpu));
else if (kmemleak_early_log)
log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
}
EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
@ -1170,8 +1039,6 @@ void __ref kmemleak_not_leak(const void *ptr)
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
make_gray_object((unsigned long)ptr);
else if (kmemleak_early_log)
log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
}
EXPORT_SYMBOL(kmemleak_not_leak);
@ -1190,8 +1057,6 @@ void __ref kmemleak_ignore(const void *ptr)
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
make_black_object((unsigned long)ptr);
else if (kmemleak_early_log)
log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
}
EXPORT_SYMBOL(kmemleak_ignore);
@ -1212,8 +1077,6 @@ void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
add_scan_area((unsigned long)ptr, size, gfp);
else if (kmemleak_early_log)
log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
}
EXPORT_SYMBOL(kmemleak_scan_area);
@ -1232,8 +1095,6 @@ void __ref kmemleak_no_scan(const void *ptr)
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
object_no_scan((unsigned long)ptr);
else if (kmemleak_early_log)
log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
}
EXPORT_SYMBOL(kmemleak_no_scan);
@ -2020,7 +1881,6 @@ static void kmemleak_disable(void)
/* stop any memory operation tracing */
kmemleak_enabled = 0;
kmemleak_early_log = 0;
/* check whether it is too early for a kernel thread */
if (kmemleak_initialized)
@ -2048,20 +1908,11 @@ static int __init kmemleak_boot_config(char *str)
}
early_param("kmemleak", kmemleak_boot_config);
static void __init print_log_trace(struct early_log *log)
{
pr_notice("Early log backtrace:\n");
stack_trace_print(log->trace, log->trace_len, 2);
}
/*
* Kmemleak initialization.
*/
void __init kmemleak_init(void)
{
int i;
unsigned long flags;
#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
if (!kmemleak_skip_disable) {
kmemleak_disable();
@ -2069,28 +1920,15 @@ void __init kmemleak_init(void)
}
#endif
if (kmemleak_error)
return;
jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
if (crt_early_log > ARRAY_SIZE(early_log))
pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
crt_early_log);
/* the kernel is still in UP mode, so disabling the IRQs is enough */
local_irq_save(flags);
kmemleak_early_log = 0;
if (kmemleak_error) {
local_irq_restore(flags);
return;
} else {
kmemleak_enabled = 1;
kmemleak_free_enabled = 1;
}
local_irq_restore(flags);
/* register the data/bss sections */
create_object((unsigned long)_sdata, _edata - _sdata,
KMEMLEAK_GREY, GFP_ATOMIC);
@ -2101,57 +1939,6 @@ void __init kmemleak_init(void)
create_object((unsigned long)__start_ro_after_init,
__end_ro_after_init - __start_ro_after_init,
KMEMLEAK_GREY, GFP_ATOMIC);
/*
* This is the point where tracking allocations is safe. Automatic
* scanning is started during the late initcall. Add the early logged
* callbacks to the kmemleak infrastructure.
*/
for (i = 0; i < crt_early_log; i++) {
struct early_log *log = &early_log[i];
switch (log->op_type) {
case KMEMLEAK_ALLOC:
early_alloc(log);
break;
case KMEMLEAK_ALLOC_PERCPU:
early_alloc_percpu(log);
break;
case KMEMLEAK_FREE:
kmemleak_free(log->ptr);
break;
case KMEMLEAK_FREE_PART:
kmemleak_free_part(log->ptr, log->size);
break;
case KMEMLEAK_FREE_PERCPU:
kmemleak_free_percpu(log->ptr);
break;
case KMEMLEAK_NOT_LEAK:
kmemleak_not_leak(log->ptr);
break;
case KMEMLEAK_IGNORE:
kmemleak_ignore(log->ptr);
break;
case KMEMLEAK_SCAN_AREA:
kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
break;
case KMEMLEAK_NO_SCAN:
kmemleak_no_scan(log->ptr);
break;
case KMEMLEAK_SET_EXCESS_REF:
object_set_excess_ref((unsigned long)log->ptr,
log->excess_ref);
break;
default:
kmemleak_warn("Unknown early log operation: %d\n",
log->op_type);
}
if (kmemleak_warning) {
print_log_trace(log);
kmemleak_warning = 0;
}
}
}
/*