mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

tracing: make the function profiler per cpu

Browse source 
Impact: speed enhancement

By making the function profiler record in per cpu data we not only
get better readings, avoid races, we also do not have to take any
locks.

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
This commit is contained in:
Steven Rostedt 2009-03-24 20:50:39 -04:00
parent 0706f1c48c
commit cafb168a1c
1 changed files with 130 additions and 69 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

199
kernel/trace/ftrace.c
View file

@ -257,28 +257,28 @@ struct ftrace_profile_page {
struct ftrace_profile records[]; struct ftrace_profile records[];
}; };
struct ftrace_profile_stat {
atomic_t disabled;
struct hlist_head *hash;
struct ftrace_profile_page *pages;
struct ftrace_profile_page *start;
struct tracer_stat stat;
};
#define PROFILE_RECORDS_SIZE \ #define PROFILE_RECORDS_SIZE \
(PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
#define PROFILES_PER_PAGE \ #define PROFILES_PER_PAGE \
(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
/* TODO: make these percpu, to prevent cache line bouncing */
static struct ftrace_profile_page *profile_pages_start;
static struct ftrace_profile_page *profile_pages;
static struct hlist_head *ftrace_profile_hash;
static int ftrace_profile_bits; static int ftrace_profile_bits;
static int ftrace_profile_enabled; static int ftrace_profile_enabled;
static DEFINE_MUTEX(ftrace_profile_lock); static DEFINE_MUTEX(ftrace_profile_lock);
static DEFINE_PER_CPU(atomic_t, ftrace_profile_disable); static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
static raw_spinlock_t ftrace_profile_rec_lock =
(raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
static void * static void *
function_stat_next(void *v, int idx) function_stat_next(void *v, int idx)
{ {
@ -303,7 +303,13 @@ function_stat_next(void *v, int idx)
static void *function_stat_start(struct tracer_stat *trace) static void *function_stat_start(struct tracer_stat *trace)
{ {
return function_stat_next(&profile_pages_start->records[0], 0); struct ftrace_profile_stat *stat =
container_of(trace, struct ftrace_profile_stat, stat);
if (!stat || !stat->start)
return NULL;
return function_stat_next(&stat->start->records[0], 0);
} }
#ifdef CONFIG_FUNCTION_GRAPH_TRACER #ifdef CONFIG_FUNCTION_GRAPH_TRACER
@ -374,20 +380,11 @@ static int function_stat_show(struct seq_file *m, void *v)
return 0; return 0;
} }
static struct tracer_stat function_stats = { static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
.name = "functions",
.stat_start = function_stat_start,
.stat_next = function_stat_next,
.stat_cmp = function_stat_cmp,
.stat_headers = function_stat_headers,
.stat_show = function_stat_show
};
static void ftrace_profile_reset(void)
{ {
struct ftrace_profile_page *pg; struct ftrace_profile_page *pg;
pg = profile_pages = profile_pages_start; pg = stat->pages = stat->start;
while (pg) { while (pg) {
memset(pg->records, 0, PROFILE_RECORDS_SIZE); memset(pg->records, 0, PROFILE_RECORDS_SIZE);
@ -395,24 +392,24 @@ static void ftrace_profile_reset(void)
pg = pg->next; pg = pg->next;
} }
memset(ftrace_profile_hash, 0, memset(stat->hash, 0,
FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
} }
int ftrace_profile_pages_init(void) int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
{ {
struct ftrace_profile_page *pg; struct ftrace_profile_page *pg;
int i; int i;
/* If we already allocated, do nothing */ /* If we already allocated, do nothing */
if (profile_pages) if (stat->pages)
return 0; return 0;
profile_pages = (void *)get_zeroed_page(GFP_KERNEL); stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
if (!profile_pages) if (!stat->pages)
return -ENOMEM; return -ENOMEM;
pg = profile_pages_start = profile_pages; pg = stat->start = stat->pages;
/* allocate 10 more pages to start */ /* allocate 10 more pages to start */
for (i = 0; i < 10; i++) { for (i = 0; i < 10; i++) {
@ -430,13 +427,16 @@ int ftrace_profile_pages_init(void)
return 0; return 0;
} }
static int ftrace_profile_init(void) static int ftrace_profile_init_cpu(int cpu)
{ {
struct ftrace_profile_stat *stat;
int size; int size;
if (ftrace_profile_hash) { stat = &per_cpu(ftrace_profile_stats, cpu);
if (stat->hash) {
/* If the profile is already created, simply reset it */ /* If the profile is already created, simply reset it */
ftrace_profile_reset(); ftrace_profile_reset(stat);
return 0; return 0;
} }
@ -446,29 +446,45 @@ static int ftrace_profile_init(void)
*/ */
size = FTRACE_PROFILE_HASH_SIZE; size = FTRACE_PROFILE_HASH_SIZE;
ftrace_profile_hash = stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
if (!ftrace_profile_hash) if (!stat->hash)
return -ENOMEM; return -ENOMEM;
size--; if (!ftrace_profile_bits) {
size--;
for (; size; size >>= 1) for (; size; size >>= 1)
ftrace_profile_bits++; ftrace_profile_bits++;
}
/* Preallocate a few pages */ /* Preallocate a few pages */
if (ftrace_profile_pages_init() < 0) { if (ftrace_profile_pages_init(stat) < 0) {
kfree(ftrace_profile_hash); kfree(stat->hash);
ftrace_profile_hash = NULL; stat->hash = NULL;
return -ENOMEM; return -ENOMEM;
} }
return 0; return 0;
} }
static int ftrace_profile_init(void)
{
int cpu;
int ret = 0;
for_each_online_cpu(cpu) {
ret = ftrace_profile_init_cpu(cpu);
if (ret)
break;
}
return ret;
}
/* interrupts must be disabled */ /* interrupts must be disabled */
static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip) static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
{ {
struct ftrace_profile *rec; struct ftrace_profile *rec;
struct hlist_head *hhd; struct hlist_head *hhd;
@ -476,7 +492,7 @@ static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
unsigned long key; unsigned long key;
key = hash_long(ip, ftrace_profile_bits); key = hash_long(ip, ftrace_profile_bits);
hhd = &ftrace_profile_hash[key]; hhd = &stat->hash[key];
if (hlist_empty(hhd)) if (hlist_empty(hhd))
return NULL; return NULL;
@ -489,52 +505,50 @@ static struct ftrace_profile *ftrace_find_profiled_func(unsigned long ip)
return NULL; return NULL;
} }
static void ftrace_add_profile(struct ftrace_profile *rec) static void ftrace_add_profile(struct ftrace_profile_stat *stat,
struct ftrace_profile *rec)
{ {
unsigned long key; unsigned long key;
key = hash_long(rec->ip, ftrace_profile_bits); key = hash_long(rec->ip, ftrace_profile_bits);
hlist_add_head_rcu(&rec->node, &ftrace_profile_hash[key]); hlist_add_head_rcu(&rec->node, &stat->hash[key]);
} }
/* Interrupts must be disabled calling this */ /* Interrupts must be disabled calling this */
static struct ftrace_profile * static struct ftrace_profile *
ftrace_profile_alloc(unsigned long ip, bool alloc_safe) ftrace_profile_alloc(struct ftrace_profile_stat *stat,
unsigned long ip, bool alloc_safe)
{ {
struct ftrace_profile *rec = NULL; struct ftrace_profile *rec = NULL;
/* prevent recursion */ /* prevent recursion */
if (atomic_inc_return(&__get_cpu_var(ftrace_profile_disable)) != 1) if (atomic_inc_return(&stat->disabled) != 1)
goto out; goto out;
__raw_spin_lock(&ftrace_profile_rec_lock);
/* Try to always keep another page available */ /* Try to always keep another page available */
if (!profile_pages->next && alloc_safe) if (!stat->pages->next && alloc_safe)
profile_pages->next = (void *)get_zeroed_page(GFP_ATOMIC); stat->pages->next = (void *)get_zeroed_page(GFP_ATOMIC);
/* /*
* Try to find the function again since another * Try to find the function again since another
* task on another CPU could have added it * task on another CPU could have added it
*/ */
rec = ftrace_find_profiled_func(ip); rec = ftrace_find_profiled_func(stat, ip);
if (rec) if (rec)
goto out_unlock; goto out;
if (profile_pages->index == PROFILES_PER_PAGE) { if (stat->pages->index == PROFILES_PER_PAGE) {
if (!profile_pages->next) if (!stat->pages->next)
goto out_unlock; goto out;
profile_pages = profile_pages->next; stat->pages = stat->pages->next;
} }
rec = &profile_pages->records[profile_pages->index++]; rec = &stat->pages->records[stat->pages->index++];
rec->ip = ip; rec->ip = ip;
ftrace_add_profile(rec); ftrace_add_profile(stat, rec);
out_unlock:
__raw_spin_unlock(&ftrace_profile_rec_lock);
out: out:
atomic_dec(&__get_cpu_var(ftrace_profile_disable)); atomic_dec(&stat->disabled);
return rec; return rec;
} }
@ -552,6 +566,7 @@ static bool ftrace_safe_to_allocate(void)
static void static void
function_profile_call(unsigned long ip, unsigned long parent_ip) function_profile_call(unsigned long ip, unsigned long parent_ip)
{ {
struct ftrace_profile_stat *stat;
struct ftrace_profile *rec; struct ftrace_profile *rec;
unsigned long flags; unsigned long flags;
bool alloc_safe; bool alloc_safe;
@ -562,9 +577,14 @@ function_profile_call(unsigned long ip, unsigned long parent_ip)
alloc_safe = ftrace_safe_to_allocate(); alloc_safe = ftrace_safe_to_allocate();
local_irq_save(flags); local_irq_save(flags);
rec = ftrace_find_profiled_func(ip);
stat = &__get_cpu_var(ftrace_profile_stats);
if (!stat->hash)
goto out;
rec = ftrace_find_profiled_func(stat, ip);
if (!rec) { if (!rec) {
rec = ftrace_profile_alloc(ip, alloc_safe); rec = ftrace_profile_alloc(stat, ip, alloc_safe);
if (!rec) if (!rec)
goto out; goto out;
} }
@ -583,13 +603,19 @@ static int profile_graph_entry(struct ftrace_graph_ent *trace)
static void profile_graph_return(struct ftrace_graph_ret *trace) static void profile_graph_return(struct ftrace_graph_ret *trace)
{ {
unsigned long flags; struct ftrace_profile_stat *stat;
struct ftrace_profile *rec; struct ftrace_profile *rec;
unsigned long flags;
local_irq_save(flags); local_irq_save(flags);
rec = ftrace_find_profiled_func(trace->func); stat = &__get_cpu_var(ftrace_profile_stats);
if (!stat->hash)
goto out;
rec = ftrace_find_profiled_func(stat, trace->func);
if (rec) if (rec)
rec->time += trace->rettime - trace->calltime; rec->time += trace->rettime - trace->calltime;
out:
local_irq_restore(flags); local_irq_restore(flags);
} }
@ -687,16 +713,51 @@ static const struct file_operations ftrace_profile_fops = {
.write = ftrace_profile_write, .write = ftrace_profile_write,
}; };
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
.name = "functions",
.stat_start = function_stat_start,
.stat_next = function_stat_next,
.stat_cmp = function_stat_cmp,
.stat_headers = function_stat_headers,
.stat_show = function_stat_show
};
static void ftrace_profile_debugfs(struct dentry *d_tracer) static void ftrace_profile_debugfs(struct dentry *d_tracer)
{ {
struct ftrace_profile_stat *stat;
struct dentry *entry; struct dentry *entry;
char *name;
int ret; int ret;
int cpu;
ret = register_stat_tracer(&function_stats); for_each_possible_cpu(cpu) {
if (ret) { stat = &per_cpu(ftrace_profile_stats, cpu);
pr_warning("Warning: could not register "
"function stats\n"); /* allocate enough for function name + cpu number */
return; name = kmalloc(32, GFP_KERNEL);
if (!name) {
/*
* The files created are permanent, if something happens
* we still do not free memory.
*/
kfree(stat);
WARN(1,
"Could not allocate stat file for cpu %d\n",
cpu);
return;
}
stat->stat = function_stats;
snprintf(name, 32, "function%d", cpu);
stat->stat.name = name;
ret = register_stat_tracer(&stat->stat);
if (ret) {
WARN(1,
"Could not register function stat for cpu %d\n",
cpu);
kfree(name);
return;
}
} }
entry = debugfs_create_file("function_profile_enabled", 0644, entry = debugfs_create_file("function_profile_enabled", 0644,