mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction

Browse source 
Improves performance.

Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Steven Rostedt <rostedt@goodmis.org>
LKML-Reference: <1274259525.5605.10352.camel@twins>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Peter Zijlstra 2010-05-19 10:52:27 +02:00 committed by Ingo Molnar
parent 1c34bde13a
commit b7e2ecef92
5 changed files with 48 additions and 73 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

9
include/linux/ftrace_event.h
View file

@ -197,20 +197,17 @@ extern void perf_trace_disable(int event_id);
extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
char *filter_str); char *filter_str);
extern void ftrace_profile_free_filter(struct perf_event *event); extern void ftrace_profile_free_filter(struct perf_event *event);
extern void * extern void *perf_trace_buf_prepare(int size, unsigned short type,
perf_trace_buf_prepare(int size, unsigned short type, int *rctxp, struct pt_regs *regs, int *rctxp);
unsigned long *irq_flags);
static inline void static inline void
perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr, perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
u64 count, unsigned long irq_flags, struct pt_regs *regs, u64 count, struct pt_regs *regs, void *event)
void *event)
{ {
struct trace_entry *entry = raw_data; struct trace_entry *entry = raw_data;
perf_tp_event(entry->type, addr, count, raw_data, size, regs, event); perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
perf_swevent_put_recursion_context(rctx); perf_swevent_put_recursion_context(rctx);
local_irq_restore(irq_flags);
} }
#endif #endif

17
include/trace/ftrace.h
View file

@ -768,7 +768,6 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \
struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
struct ftrace_raw_##call *entry; \ struct ftrace_raw_##call *entry; \
u64 __addr = 0, __count = 1; \ u64 __addr = 0, __count = 1; \
unsigned long irq_flags; \
int __entry_size; \ int __entry_size; \
int __data_size; \ int __data_size; \
int rctx; \ int rctx; \
@ -781,17 +780,18 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \
if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE, \ if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE, \
"profile buffer not large enough")) \ "profile buffer not large enough")) \
return; \ return; \
\
entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare( \ entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare( \
__entry_size, event_call->id, &rctx, &irq_flags); \ __entry_size, event_call->id, __regs, &rctx); \
if (!entry) \ if (!entry) \
return; \ return; \
\
tstruct \ tstruct \
\ \
{ assign; } \ { assign; } \
\ \
perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \ perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \
__count, irq_flags, __regs, \ __count, __regs, event_call->perf_data); \
event_call->perf_data); \
} }
#undef DEFINE_EVENT #undef DEFINE_EVENT
@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \
static notrace void perf_trace_##call(proto) \ static notrace void perf_trace_##call(proto) \
{ \ { \
struct ftrace_event_call *event_call = &event_##call; \ struct ftrace_event_call *event_call = &event_##call; \
struct pt_regs *__regs = &get_cpu_var(perf_trace_regs); \ struct pt_regs __regs; \
\ \
perf_fetch_caller_regs(__regs, 1); \ perf_fetch_caller_regs(&__regs, 1); \
\ perf_trace_templ_##template(event_call, &__regs, args); \
perf_trace_templ_##template(event_call, __regs, args); \
\
put_cpu_var(perf_trace_regs); \
} }
#undef DEFINE_EVENT_PRINT #undef DEFINE_EVENT_PRINT

71
kernel/trace/trace_event_perf.c
View file

@ -9,13 +9,9 @@
#include <linux/kprobes.h> #include <linux/kprobes.h>
#include "trace.h" #include "trace.h"
DEFINE_PER_CPU(struct pt_regs, perf_trace_regs);
EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs);
EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);
static char *perf_trace_buf; static char *perf_trace_buf[4];
static char *perf_trace_buf_nmi;
/* /*
* Force it to be aligned to unsigned long to avoid misaligned accesses * Force it to be aligned to unsigned long to avoid misaligned accesses
@ -29,7 +25,6 @@ static int total_ref_count;
static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
{ {
char *buf;
int ret = -ENOMEM; int ret = -ENOMEM;
if (event->perf_refcount++ > 0) { if (event->perf_refcount++ > 0) {
@ -38,17 +33,16 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
} }
if (!total_ref_count) { if (!total_ref_count) {
char *buf;
int i;
for (i = 0; i < 4; i++) {
buf = (char *)alloc_percpu(perf_trace_t); buf = (char *)alloc_percpu(perf_trace_t);
if (!buf) if (!buf)
goto fail_buf; goto fail_buf;
rcu_assign_pointer(perf_trace_buf, buf); rcu_assign_pointer(perf_trace_buf[i], buf);
}
buf = (char *)alloc_percpu(perf_trace_t);
if (!buf)
goto fail_buf_nmi;
rcu_assign_pointer(perf_trace_buf_nmi, buf);
} }
ret = event->perf_event_enable(event); ret = event->perf_event_enable(event);
@ -58,14 +52,15 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
return 0; return 0;
} }
fail_buf_nmi:
if (!total_ref_count) {
free_percpu(perf_trace_buf_nmi);
free_percpu(perf_trace_buf);
perf_trace_buf_nmi = NULL;
perf_trace_buf = NULL;
}
fail_buf: fail_buf:
if (!total_ref_count) {
int i;
for (i = 0; i < 4; i++) {
free_percpu(perf_trace_buf[i]);
perf_trace_buf[i] = NULL;
}
}
event->perf_refcount--; event->perf_refcount--;
return ret; return ret;
@ -91,19 +86,19 @@ int perf_trace_enable(int event_id, void *data)
static void perf_trace_event_disable(struct ftrace_event_call *event) static void perf_trace_event_disable(struct ftrace_event_call *event)
{ {
char *buf, *nmi_buf;
if (--event->perf_refcount > 0) if (--event->perf_refcount > 0)
return; return;
event->perf_event_disable(event); event->perf_event_disable(event);
if (!--total_ref_count) { if (!--total_ref_count) {
buf = perf_trace_buf; char *buf[4];
rcu_assign_pointer(perf_trace_buf, NULL); int i;
nmi_buf = perf_trace_buf_nmi; for (i = 0; i < 4; i++) {
rcu_assign_pointer(perf_trace_buf_nmi, NULL); buf[i] = perf_trace_buf[i];
rcu_assign_pointer(perf_trace_buf[i], NULL);
}
/* /*
* Ensure every events in profiling have finished before * Ensure every events in profiling have finished before
@ -111,8 +106,8 @@ static void perf_trace_event_disable(struct ftrace_event_call *event)
*/ */
synchronize_sched(); synchronize_sched();
free_percpu(buf); for (i = 0; i < 4; i++)
free_percpu(nmi_buf); free_percpu(buf[i]);
} }
} }
@ -132,47 +127,37 @@ void perf_trace_disable(int event_id)
} }
__kprobes void *perf_trace_buf_prepare(int size, unsigned short type, __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
int *rctxp, unsigned long *irq_flags) struct pt_regs *regs, int *rctxp)
{ {
struct trace_entry *entry; struct trace_entry *entry;
char *trace_buf, *raw_data; char *trace_buf, *raw_data;
int pc, cpu; int pc;
BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long)); BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
pc = preempt_count(); pc = preempt_count();
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(*irq_flags);
*rctxp = perf_swevent_get_recursion_context(); *rctxp = perf_swevent_get_recursion_context();
if (*rctxp < 0) if (*rctxp < 0)
goto err_recursion; goto err_recursion;
cpu = smp_processor_id(); trace_buf = rcu_dereference_sched(perf_trace_buf[*rctxp]);
if (in_nmi())
trace_buf = rcu_dereference_sched(perf_trace_buf_nmi);
else
trace_buf = rcu_dereference_sched(perf_trace_buf);
if (!trace_buf) if (!trace_buf)
goto err; goto err;
raw_data = per_cpu_ptr(trace_buf, cpu); raw_data = per_cpu_ptr(trace_buf, smp_processor_id());
/* zero the dead bytes from align to not leak stack to user */ /* zero the dead bytes from align to not leak stack to user */
memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64)); memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
entry = (struct trace_entry *)raw_data; entry = (struct trace_entry *)raw_data;
tracing_generic_entry_update(entry, *irq_flags, pc); tracing_generic_entry_update(entry, regs->flags, pc);
entry->type = type; entry->type = type;
return raw_data; return raw_data;
err: err:
perf_swevent_put_recursion_context(*rctxp); perf_swevent_put_recursion_context(*rctxp);
err_recursion: err_recursion:
local_irq_restore(*irq_flags);
return NULL; return NULL;
} }
EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);

10
kernel/trace/trace_kprobe.c
View file

@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
struct kprobe_trace_entry_head *entry; struct kprobe_trace_entry_head *entry;
u8 *data; u8 *data;
int size, __size, i; int size, __size, i;
unsigned long irq_flags;
int rctx; int rctx;
__size = sizeof(*entry) + tp->size; __size = sizeof(*entry) + tp->size;
@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
"profile buffer not large enough")) "profile buffer not large enough"))
return; return;
entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
if (!entry) if (!entry)
return; return;
@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp,
for (i = 0; i < tp->nr_args; i++) for (i = 0; i < tp->nr_args; i++)
call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data); perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
} }
/* Kretprobe profile handler */ /* Kretprobe profile handler */
@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
struct kretprobe_trace_entry_head *entry; struct kretprobe_trace_entry_head *entry;
u8 *data; u8 *data;
int size, __size, i; int size, __size, i;
unsigned long irq_flags;
int rctx; int rctx;
__size = sizeof(*entry) + tp->size; __size = sizeof(*entry) + tp->size;
@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
"profile buffer not large enough")) "profile buffer not large enough"))
return; return;
entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
if (!entry) if (!entry)
return; return;
@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri,
call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
irq_flags, regs, call->perf_data); regs, call->perf_data);
} }
static int probe_perf_enable(struct ftrace_event_call *call) static int probe_perf_enable(struct ftrace_event_call *call)

10
kernel/trace/trace_syscalls.c
View file

@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
{ {
struct syscall_metadata *sys_data; struct syscall_metadata *sys_data;
struct syscall_trace_enter *rec; struct syscall_trace_enter *rec;
unsigned long flags;
int syscall_nr; int syscall_nr;
int rctx; int rctx;
int size; int size;
@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt_regs *regs, long id)
return; return;
rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size, rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
sys_data->enter_event->id, &rctx, &flags); sys_data->enter_event->id, regs, &rctx);
if (!rec) if (!rec)
return; return;
rec->nr = syscall_nr; rec->nr = syscall_nr;
syscall_get_arguments(current, regs, 0, sys_data->nb_args, syscall_get_arguments(current, regs, 0, sys_data->nb_args,
(unsigned long *)&rec->args); (unsigned long *)&rec->args);
perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs, perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
sys_data->enter_event->perf_data); sys_data->enter_event->perf_data);
} }
@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
{ {
struct syscall_metadata *sys_data; struct syscall_metadata *sys_data;
struct syscall_trace_exit *rec; struct syscall_trace_exit *rec;
unsigned long flags;
int syscall_nr; int syscall_nr;
int rctx; int rctx;
int size; int size;
@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret)
return; return;
rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size, rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
sys_data->exit_event->id, &rctx, &flags); sys_data->exit_event->id, regs, &rctx);
if (!rec) if (!rec)
return; return;
rec->nr = syscall_nr; rec->nr = syscall_nr;
rec->ret = syscall_get_return_value(current, regs); rec->ret = syscall_get_return_value(current, regs);
perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs, perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
sys_data->exit_event->perf_data); sys_data->exit_event->perf_data);
} }