nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

kprobes, x86: Use NOKPROBE_SYMBOL() instead of __kprobes annotation

Browse source 
Use NOKPROBE_SYMBOL macro for protecting functions
from kprobes instead of __kprobes annotation under
arch/x86.

This applies nokprobe_inline annotation for some cases,
because NOKPROBE_SYMBOL() will inhibit inlining by
referring the symbol address.

This just folds a bunch of previous NOKPROBE_SYMBOL()
cleanup patches for x86 to one patch.

Signed-off-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Link: http://lkml.kernel.org/r/20140417081814.26341.51656.stgit@ltc230.yrl.intra.hitachi.co.jp
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Borislav Petkov <bp@suse.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fernando Luis Vázquez Cao <fernando_b1@lab.ntt.co.jp>
Cc: Gleb Natapov <gleb@redhat.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: Jiri Slaby <jslaby@suse.cz>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Jonathan Lebon <jlebon@redhat.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Matt Fleming <matt.fleming@intel.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Seiji Aguchi <seiji.aguchi@hds.com>
Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vineet Gupta <vgupta@synopsys.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Masami Hiramatsu 2014-04-17 17:18:14 +09:00 committed by Ingo Molnar
parent 9c54b6164e
commit 9326638cbe
12 changed files with 122 additions and 69 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

2
arch/x86/include/asm/traps.h
View file

@ -68,7 +68,7 @@ dotraplinkage void do_segment_not_present(struct pt_regs *, long);
dotraplinkage void do_stack_segment(struct pt_regs *, long); dotraplinkage void do_stack_segment(struct pt_regs *, long);
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
dotraplinkage void do_double_fault(struct pt_regs *, long); dotraplinkage void do_double_fault(struct pt_regs *, long);
asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *); asmlinkage struct pt_regs *sync_regs(struct pt_regs *);
#endif #endif
dotraplinkage void do_general_protection(struct pt_regs *, long); dotraplinkage void do_general_protection(struct pt_regs *, long);
dotraplinkage void do_page_fault(struct pt_regs *, unsigned long); dotraplinkage void do_page_fault(struct pt_regs *, unsigned long);

3
arch/x86/kernel/apic/hw_nmi.c
View file

@ -60,7 +60,7 @@ void arch_trigger_all_cpu_backtrace(void)
smp_mb__after_clear_bit(); smp_mb__after_clear_bit();
} }
static int __kprobes static int
arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs)
{ {
int cpu; int cpu;
@ -80,6 +80,7 @@ arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs)
return NMI_DONE; return NMI_DONE;
} }
NOKPROBE_SYMBOL(arch_trigger_all_cpu_backtrace_handler);
static int __init register_trigger_all_cpu_backtrace(void) static int __init register_trigger_all_cpu_backtrace(void)
{ {

3
arch/x86/kernel/cpu/perf_event.c
View file

@ -1292,7 +1292,7 @@ void perf_events_lapic_init(void)
apic_write(APIC_LVTPC, APIC_DM_NMI); apic_write(APIC_LVTPC, APIC_DM_NMI);
} }
static int __kprobes static int
perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{ {
u64 start_clock; u64 start_clock;
@ -1310,6 +1310,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
return ret; return ret;
} }
NOKPROBE_SYMBOL(perf_event_nmi_handler);
struct event_constraint emptyconstraint; struct event_constraint emptyconstraint;
struct event_constraint unconstrained; struct event_constraint unconstrained;

3
arch/x86/kernel/cpu/perf_event_amd_ibs.c
View file

@ -593,7 +593,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
return 1; return 1;
} }
static int __kprobes static int
perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{ {
int handled = 0; int handled = 0;
@ -606,6 +606,7 @@ perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
return handled; return handled;
} }
NOKPROBE_SYMBOL(perf_ibs_nmi_handler);
static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
{ {

9
arch/x86/kernel/dumpstack.c
View file

@ -200,7 +200,7 @@ static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;
static int die_owner = -1; static int die_owner = -1;
static unsigned int die_nest_count; static unsigned int die_nest_count;
unsigned __kprobes long oops_begin(void) unsigned long oops_begin(void)
{ {
int cpu; int cpu;
unsigned long flags; unsigned long flags;
@ -223,8 +223,9 @@ unsigned __kprobes long oops_begin(void)
return flags; return flags;
} }
EXPORT_SYMBOL_GPL(oops_begin); EXPORT_SYMBOL_GPL(oops_begin);
NOKPROBE_SYMBOL(oops_begin);
void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr) void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
{ {
if (regs && kexec_should_crash(current)) if (regs && kexec_should_crash(current))
crash_kexec(regs); crash_kexec(regs);
@ -247,8 +248,9 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
panic("Fatal exception"); panic("Fatal exception");
do_exit(signr); do_exit(signr);
} }
NOKPROBE_SYMBOL(oops_end);
int __kprobes __die(const char *str, struct pt_regs *regs, long err) int __die(const char *str, struct pt_regs *regs, long err)
{ {
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
unsigned short ss; unsigned short ss;
@ -291,6 +293,7 @@ int __kprobes __die(const char *str, struct pt_regs *regs, long err)
#endif #endif
return 0; return 0;
} }
NOKPROBE_SYMBOL(__die);
/* /*
* This is gone through when something in the kernel has done something bad * This is gone through when something in the kernel has done something bad

77
arch/x86/kernel/kprobes/core.c
View file

@ -112,7 +112,8 @@ struct kretprobe_blackpoint kretprobe_blacklist[] = {
const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op) static nokprobe_inline void
__synthesize_relative_insn(void *from, void *to, u8 op)
{ {
struct __arch_relative_insn { struct __arch_relative_insn {
u8 op; u8 op;
@ -125,21 +126,23 @@ static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
} }
/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
void __kprobes synthesize_reljump(void *from, void *to) void synthesize_reljump(void *from, void *to)
{ {
__synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE); __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
} }
NOKPROBE_SYMBOL(synthesize_reljump);
/* Insert a call instruction at address 'from', which calls address 'to'.*/ /* Insert a call instruction at address 'from', which calls address 'to'.*/
void __kprobes synthesize_relcall(void *from, void *to) void synthesize_relcall(void *from, void *to)
{ {
__synthesize_relative_insn(from, to, RELATIVECALL_OPCODE); __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
} }
NOKPROBE_SYMBOL(synthesize_relcall);
/* /*
* Skip the prefixes of the instruction. * Skip the prefixes of the instruction.
*/ */
static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn) static kprobe_opcode_t *skip_prefixes(kprobe_opcode_t *insn)
{ {
insn_attr_t attr; insn_attr_t attr;
@ -154,6 +157,7 @@ static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn)
#endif #endif
return insn; return insn;
} }
NOKPROBE_SYMBOL(skip_prefixes);
/* /*
* Returns non-zero if opcode is boostable. * Returns non-zero if opcode is boostable.
@ -425,7 +429,8 @@ void arch_remove_kprobe(struct kprobe *p)
} }
} }
static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) static nokprobe_inline void
save_previous_kprobe(struct kprobe_ctlblk *kcb)
{ {
kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.kp = kprobe_running();
kcb->prev_kprobe.status = kcb->kprobe_status; kcb->prev_kprobe.status = kcb->kprobe_status;
@ -433,7 +438,8 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
} }
static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) static nokprobe_inline void
restore_previous_kprobe(struct kprobe_ctlblk *kcb)
{ {
__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_status = kcb->prev_kprobe.status;
@ -441,8 +447,9 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
} }
static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, static nokprobe_inline void
struct kprobe_ctlblk *kcb) set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{ {
__this_cpu_write(current_kprobe, p); __this_cpu_write(current_kprobe, p);
kcb->kprobe_saved_flags = kcb->kprobe_old_flags kcb->kprobe_saved_flags = kcb->kprobe_old_flags
@ -451,7 +458,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF; kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
} }
static void __kprobes clear_btf(void) static nokprobe_inline void clear_btf(void)
{ {
if (test_thread_flag(TIF_BLOCKSTEP)) { if (test_thread_flag(TIF_BLOCKSTEP)) {
unsigned long debugctl = get_debugctlmsr(); unsigned long debugctl = get_debugctlmsr();
@ -461,7 +468,7 @@ static void __kprobes clear_btf(void)
} }
} }
static void __kprobes restore_btf(void) static nokprobe_inline void restore_btf(void)
{ {
if (test_thread_flag(TIF_BLOCKSTEP)) { if (test_thread_flag(TIF_BLOCKSTEP)) {
unsigned long debugctl = get_debugctlmsr(); unsigned long debugctl = get_debugctlmsr();
@ -471,8 +478,7 @@ static void __kprobes restore_btf(void)
} }
} }
void __kprobes void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
{ {
unsigned long *sara = stack_addr(regs); unsigned long *sara = stack_addr(regs);
@ -481,9 +487,10 @@ arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
/* Replace the return addr with trampoline addr */ /* Replace the return addr with trampoline addr */
*sara = (unsigned long) &kretprobe_trampoline; *sara = (unsigned long) &kretprobe_trampoline;
} }
NOKPROBE_SYMBOL(arch_prepare_kretprobe);
static void __kprobes static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb, int reenter) struct kprobe_ctlblk *kcb, int reenter)
{ {
if (setup_detour_execution(p, regs, reenter)) if (setup_detour_execution(p, regs, reenter))
return; return;
@ -519,14 +526,15 @@ setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *k
else else
regs->ip = (unsigned long)p->ainsn.insn; regs->ip = (unsigned long)p->ainsn.insn;
} }
NOKPROBE_SYMBOL(setup_singlestep);
/* /*
* We have reentered the kprobe_handler(), since another probe was hit while * We have reentered the kprobe_handler(), since another probe was hit while
* within the handler. We save the original kprobes variables and just single * within the handler. We save the original kprobes variables and just single
* step on the instruction of the new probe without calling any user handlers. * step on the instruction of the new probe without calling any user handlers.
*/ */
static int __kprobes static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) struct kprobe_ctlblk *kcb)
{ {
switch (kcb->kprobe_status) { switch (kcb->kprobe_status) {
case KPROBE_HIT_SSDONE: case KPROBE_HIT_SSDONE:
@ -554,12 +562,13 @@ reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb
return 1; return 1;
} }
NOKPROBE_SYMBOL(reenter_kprobe);
/* /*
* Interrupts are disabled on entry as trap3 is an interrupt gate and they * Interrupts are disabled on entry as trap3 is an interrupt gate and they
* remain disabled throughout this function. * remain disabled throughout this function.
*/ */
int __kprobes kprobe_int3_handler(struct pt_regs *regs) int kprobe_int3_handler(struct pt_regs *regs)
{ {
kprobe_opcode_t *addr; kprobe_opcode_t *addr;
struct kprobe *p; struct kprobe *p;
@ -622,12 +631,13 @@ int __kprobes kprobe_int3_handler(struct pt_regs *regs)
preempt_enable_no_resched(); preempt_enable_no_resched();
return 0; return 0;
} }
NOKPROBE_SYMBOL(kprobe_int3_handler);
/* /*
* When a retprobed function returns, this code saves registers and * When a retprobed function returns, this code saves registers and
* calls trampoline_handler() runs, which calls the kretprobe's handler. * calls trampoline_handler() runs, which calls the kretprobe's handler.
*/ */
static void __used __kprobes kretprobe_trampoline_holder(void) static void __used kretprobe_trampoline_holder(void)
{ {
asm volatile ( asm volatile (
".global kretprobe_trampoline\n" ".global kretprobe_trampoline\n"
@ -658,11 +668,13 @@ static void __used __kprobes kretprobe_trampoline_holder(void)
#endif #endif
" ret\n"); " ret\n");
} }
NOKPROBE_SYMBOL(kretprobe_trampoline_holder);
NOKPROBE_SYMBOL(kretprobe_trampoline);
/* /*
* Called from kretprobe_trampoline * Called from kretprobe_trampoline
*/ */
__visible __used __kprobes void *trampoline_handler(struct pt_regs *regs) __visible __used void *trampoline_handler(struct pt_regs *regs)
{ {
struct kretprobe_instance *ri = NULL; struct kretprobe_instance *ri = NULL;
struct hlist_head *head, empty_rp; struct hlist_head *head, empty_rp;
@ -748,6 +760,7 @@ __visible __used __kprobes void *trampoline_handler(struct pt_regs *regs)
} }
return (void *)orig_ret_address; return (void *)orig_ret_address;
} }
NOKPROBE_SYMBOL(trampoline_handler);
/* /*
* Called after single-stepping. p->addr is the address of the * Called after single-stepping. p->addr is the address of the
@ -776,8 +789,8 @@ __visible __used __kprobes void *trampoline_handler(struct pt_regs *regs)
* jump instruction after the copied instruction, that jumps to the next * jump instruction after the copied instruction, that jumps to the next
* instruction after the probepoint. * instruction after the probepoint.
*/ */
static void __kprobes static void resume_execution(struct kprobe *p, struct pt_regs *regs,
resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) struct kprobe_ctlblk *kcb)
{ {
unsigned long *tos = stack_addr(regs); unsigned long *tos = stack_addr(regs);
unsigned long copy_ip = (unsigned long)p->ainsn.insn; unsigned long copy_ip = (unsigned long)p->ainsn.insn;
@ -852,12 +865,13 @@ resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *k
no_change: no_change:
restore_btf(); restore_btf();
} }
NOKPROBE_SYMBOL(resume_execution);
/* /*
* Interrupts are disabled on entry as trap1 is an interrupt gate and they * Interrupts are disabled on entry as trap1 is an interrupt gate and they
* remain disabled throughout this function. * remain disabled throughout this function.
*/ */
int __kprobes kprobe_debug_handler(struct pt_regs *regs) int kprobe_debug_handler(struct pt_regs *regs)
{ {
struct kprobe *cur = kprobe_running(); struct kprobe *cur = kprobe_running();
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
@ -892,8 +906,9 @@ int __kprobes kprobe_debug_handler(struct pt_regs *regs)
return 1; return 1;
} }
NOKPROBE_SYMBOL(kprobe_debug_handler);
int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
{ {
struct kprobe *cur = kprobe_running(); struct kprobe *cur = kprobe_running();
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
@ -950,12 +965,13 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
return 0; return 0;
} }
NOKPROBE_SYMBOL(kprobe_fault_handler);
/* /*
* Wrapper routine for handling exceptions. * Wrapper routine for handling exceptions.
*/ */
int __kprobes int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val,
kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data) void *data)
{ {
struct die_args *args = data; struct die_args *args = data;
int ret = NOTIFY_DONE; int ret = NOTIFY_DONE;
@ -975,8 +991,9 @@ kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *d
} }
return ret; return ret;
} }
NOKPROBE_SYMBOL(kprobe_exceptions_notify);
int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
{ {
struct jprobe *jp = container_of(p, struct jprobe, kp); struct jprobe *jp = container_of(p, struct jprobe, kp);
unsigned long addr; unsigned long addr;
@ -1000,8 +1017,9 @@ int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
regs->ip = (unsigned long)(jp->entry); regs->ip = (unsigned long)(jp->entry);
return 1; return 1;
} }
NOKPROBE_SYMBOL(setjmp_pre_handler);
void __kprobes jprobe_return(void) void jprobe_return(void)
{ {
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
@ -1017,8 +1035,10 @@ void __kprobes jprobe_return(void)
" nop \n"::"b" " nop \n"::"b"
(kcb->jprobe_saved_sp):"memory"); (kcb->jprobe_saved_sp):"memory");
} }
NOKPROBE_SYMBOL(jprobe_return);
NOKPROBE_SYMBOL(jprobe_return_end);
int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
{ {
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
u8 *addr = (u8 *) (regs->ip - 1); u8 *addr = (u8 *) (regs->ip - 1);
@ -1046,6 +1066,7 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
} }
return 0; return 0;
} }
NOKPROBE_SYMBOL(longjmp_break_handler);
bool arch_within_kprobe_blacklist(unsigned long addr) bool arch_within_kprobe_blacklist(unsigned long addr)
{ {

15
arch/x86/kernel/kprobes/ftrace.c
View file

@ -25,8 +25,9 @@
#include "common.h" #include "common.h"
static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs, static nokprobe_inline
struct kprobe_ctlblk *kcb) int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb)
{ {
/* /*
* Emulate singlestep (and also recover regs->ip) * Emulate singlestep (and also recover regs->ip)
@ -41,18 +42,19 @@ static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
return 1; return 1;
} }
int __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs, int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
struct kprobe_ctlblk *kcb) struct kprobe_ctlblk *kcb)
{ {
if (kprobe_ftrace(p)) if (kprobe_ftrace(p))
return __skip_singlestep(p, regs, kcb); return __skip_singlestep(p, regs, kcb);
else else
return 0; return 0;
} }
NOKPROBE_SYMBOL(skip_singlestep);
/* Ftrace callback handler for kprobes */ /* Ftrace callback handler for kprobes */
void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ops, struct pt_regs *regs) struct ftrace_ops *ops, struct pt_regs *regs)
{ {
struct kprobe *p; struct kprobe *p;
struct kprobe_ctlblk *kcb; struct kprobe_ctlblk *kcb;
@ -84,6 +86,7 @@ void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
end: end:
local_irq_restore(flags); local_irq_restore(flags);
} }
NOKPROBE_SYMBOL(kprobe_ftrace_handler);
int arch_prepare_kprobe_ftrace(struct kprobe *p) int arch_prepare_kprobe_ftrace(struct kprobe *p)
{ {

8
arch/x86/kernel/kprobes/opt.c
View file

@ -138,7 +138,8 @@ asm (
#define INT3_SIZE sizeof(kprobe_opcode_t) #define INT3_SIZE sizeof(kprobe_opcode_t)
/* Optimized kprobe call back function: called from optinsn */ /* Optimized kprobe call back function: called from optinsn */
static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) static void
optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
{ {
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
unsigned long flags; unsigned long flags;
@ -168,6 +169,7 @@ static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_
} }
local_irq_restore(flags); local_irq_restore(flags);
} }
NOKPROBE_SYMBOL(optimized_callback);
static int copy_optimized_instructions(u8 *dest, u8 *src) static int copy_optimized_instructions(u8 *dest, u8 *src)
{ {
@ -424,8 +426,7 @@ extern void arch_unoptimize_kprobes(struct list_head *oplist,
} }
} }
int __kprobes int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
{ {
struct optimized_kprobe *op; struct optimized_kprobe *op;
@ -441,3 +442,4 @@ setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
} }
return 0; return 0;
} }
NOKPROBE_SYMBOL(setup_detour_execution);

4
arch/x86/kernel/kvm.c
View file

@ -251,8 +251,9 @@ u32 kvm_read_and_reset_pf_reason(void)
return reason; return reason;
} }
EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
dotraplinkage void __kprobes dotraplinkage void
do_async_page_fault(struct pt_regs *regs, unsigned long error_code) do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
enum ctx_state prev_state; enum ctx_state prev_state;
@ -276,6 +277,7 @@ do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
break; break;
} }
} }
NOKPROBE_SYMBOL(do_async_page_fault);
static void __init paravirt_ops_setup(void) static void __init paravirt_ops_setup(void)
{ {

18
arch/x86/kernel/nmi.c
View file

@ -110,7 +110,7 @@ static void nmi_max_handler(struct irq_work *w)
a->handler, whole_msecs, decimal_msecs); a->handler, whole_msecs, decimal_msecs);
} }
static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b) static int nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
{ {
struct nmi_desc *desc = nmi_to_desc(type); struct nmi_desc *desc = nmi_to_desc(type);
struct nmiaction *a; struct nmiaction *a;
@ -146,6 +146,7 @@ static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2
/* return total number of NMI events handled */ /* return total number of NMI events handled */
return handled; return handled;
} }
NOKPROBE_SYMBOL(nmi_handle);
int __register_nmi_handler(unsigned int type, struct nmiaction *action) int __register_nmi_handler(unsigned int type, struct nmiaction *action)
{ {
@ -208,7 +209,7 @@ void unregister_nmi_handler(unsigned int type, const char *name)
} }
EXPORT_SYMBOL_GPL(unregister_nmi_handler); EXPORT_SYMBOL_GPL(unregister_nmi_handler);
static __kprobes void static void
pci_serr_error(unsigned char reason, struct pt_regs *regs) pci_serr_error(unsigned char reason, struct pt_regs *regs)
{ {
/* check to see if anyone registered against these types of errors */ /* check to see if anyone registered against these types of errors */
@ -238,8 +239,9 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs)
reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR; reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
outb(reason, NMI_REASON_PORT); outb(reason, NMI_REASON_PORT);
} }
NOKPROBE_SYMBOL(pci_serr_error);
static __kprobes void static void
io_check_error(unsigned char reason, struct pt_regs *regs) io_check_error(unsigned char reason, struct pt_regs *regs)
{ {
unsigned long i; unsigned long i;
@ -269,8 +271,9 @@ io_check_error(unsigned char reason, struct pt_regs *regs)
reason &= ~NMI_REASON_CLEAR_IOCHK; reason &= ~NMI_REASON_CLEAR_IOCHK;
outb(reason, NMI_REASON_PORT); outb(reason, NMI_REASON_PORT);
} }
NOKPROBE_SYMBOL(io_check_error);
static __kprobes void static void
unknown_nmi_error(unsigned char reason, struct pt_regs *regs) unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
{ {
int handled; int handled;
@ -298,11 +301,12 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
pr_emerg("Dazed and confused, but trying to continue\n"); pr_emerg("Dazed and confused, but trying to continue\n");
} }
NOKPROBE_SYMBOL(unknown_nmi_error);
static DEFINE_PER_CPU(bool, swallow_nmi); static DEFINE_PER_CPU(bool, swallow_nmi);
static DEFINE_PER_CPU(unsigned long, last_nmi_rip); static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
static __kprobes void default_do_nmi(struct pt_regs *regs) static void default_do_nmi(struct pt_regs *regs)
{ {
unsigned char reason = 0; unsigned char reason = 0;
int handled; int handled;
@ -401,6 +405,7 @@ static __kprobes void default_do_nmi(struct pt_regs *regs)
else else
unknown_nmi_error(reason, regs); unknown_nmi_error(reason, regs);
} }
NOKPROBE_SYMBOL(default_do_nmi);
/* /*
* NMIs can hit breakpoints which will cause it to lose its * NMIs can hit breakpoints which will cause it to lose its
@ -520,7 +525,7 @@ static inline void nmi_nesting_postprocess(void)
} }
#endif #endif
dotraplinkage notrace __kprobes void dotraplinkage notrace void
do_nmi(struct pt_regs *regs, long error_code) do_nmi(struct pt_regs *regs, long error_code)
{ {
nmi_nesting_preprocess(regs); nmi_nesting_preprocess(regs);
@ -537,6 +542,7 @@ do_nmi(struct pt_regs *regs, long error_code)
/* On i386, may loop back to preprocess */ /* On i386, may loop back to preprocess */
nmi_nesting_postprocess(); nmi_nesting_postprocess();
} }
NOKPROBE_SYMBOL(do_nmi);
void stop_nmi(void) void stop_nmi(void)
{ {

20
arch/x86/kernel/traps.c
View file

@ -106,7 +106,7 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
preempt_count_dec(); preempt_count_dec();
} }
static int __kprobes static nokprobe_inline int
do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str, do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
struct pt_regs *regs, long error_code) struct pt_regs *regs, long error_code)
{ {
@ -136,7 +136,7 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
return -1; return -1;
} }
static void __kprobes static void
do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
long error_code, siginfo_t *info) long error_code, siginfo_t *info)
{ {
@ -173,6 +173,7 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
else else
force_sig(signr, tsk); force_sig(signr, tsk);
} }
NOKPROBE_SYMBOL(do_trap);
#define DO_ERROR(trapnr, signr, str, name) \ #define DO_ERROR(trapnr, signr, str, name) \
dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
@ -263,7 +264,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
} }
#endif #endif
dotraplinkage void __kprobes dotraplinkage void
do_general_protection(struct pt_regs *regs, long error_code) do_general_protection(struct pt_regs *regs, long error_code)
{ {
struct task_struct *tsk; struct task_struct *tsk;
@ -309,9 +310,10 @@ do_general_protection(struct pt_regs *regs, long error_code)
exit: exit:
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_general_protection);
/* May run on IST stack. */ /* May run on IST stack. */
dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code) dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
{ {
enum ctx_state prev_state; enum ctx_state prev_state;
@ -355,6 +357,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
exit: exit:
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_int3);
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
/* /*
@ -362,7 +365,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
* for scheduling or signal handling. The actual stack switch is done in * for scheduling or signal handling. The actual stack switch is done in
* entry.S * entry.S
*/ */
asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs) asmlinkage struct pt_regs *sync_regs(struct pt_regs *eregs)
{ {
struct pt_regs *regs = eregs; struct pt_regs *regs = eregs;
/* Did already sync */ /* Did already sync */
@ -381,6 +384,7 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
*regs = *eregs; *regs = *eregs;
return regs; return regs;
} }
NOKPROBE_SYMBOL(sync_regs);
#endif #endif
/* /*
@ -407,7 +411,7 @@ asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
* *
* May run on IST stack. * May run on IST stack.
*/ */
dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
{ {
struct task_struct *tsk = current; struct task_struct *tsk = current;
enum ctx_state prev_state; enum ctx_state prev_state;
@ -491,6 +495,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
exit: exit:
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_debug);
/* /*
* Note that we play around with the 'TS' bit in an attempt to get * Note that we play around with the 'TS' bit in an attempt to get
@ -662,7 +667,7 @@ void math_state_restore(void)
} }
EXPORT_SYMBOL_GPL(math_state_restore); EXPORT_SYMBOL_GPL(math_state_restore);
dotraplinkage void __kprobes dotraplinkage void
do_device_not_available(struct pt_regs *regs, long error_code) do_device_not_available(struct pt_regs *regs, long error_code)
{ {
enum ctx_state prev_state; enum ctx_state prev_state;
@ -688,6 +693,7 @@ do_device_not_available(struct pt_regs *regs, long error_code)
#endif #endif
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_device_not_available);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)

29
arch/x86/mm/fault.c
View file

@ -8,7 +8,7 @@
#include <linux/kdebug.h> /* oops_begin/end, ... */ #include <linux/kdebug.h> /* oops_begin/end, ... */
#include <linux/module.h> /* search_exception_table */ #include <linux/module.h> /* search_exception_table */
#include <linux/bootmem.h> /* max_low_pfn */ #include <linux/bootmem.h> /* max_low_pfn */
#include <linux/kprobes.h> /* __kprobes, ... */ #include <linux/kprobes.h> /* NOKPROBE_SYMBOL, ... */
#include <linux/mmiotrace.h> /* kmmio_handler, ... */ #include <linux/mmiotrace.h> /* kmmio_handler, ... */
#include <linux/perf_event.h> /* perf_sw_event */ #include <linux/perf_event.h> /* perf_sw_event */
#include <linux/hugetlb.h> /* hstate_index_to_shift */ #include <linux/hugetlb.h> /* hstate_index_to_shift */
@ -45,7 +45,7 @@ enum x86_pf_error_code {
* Returns 0 if mmiotrace is disabled, or if the fault is not * Returns 0 if mmiotrace is disabled, or if the fault is not
* handled by mmiotrace: * handled by mmiotrace:
*/ */
static inline int __kprobes static nokprobe_inline int
kmmio_fault(struct pt_regs *regs, unsigned long addr) kmmio_fault(struct pt_regs *regs, unsigned long addr)
{ {
if (unlikely(is_kmmio_active())) if (unlikely(is_kmmio_active()))
@ -54,7 +54,7 @@ kmmio_fault(struct pt_regs *regs, unsigned long addr)
return 0; return 0;
} }
static inline int __kprobes kprobes_fault(struct pt_regs *regs) static nokprobe_inline int kprobes_fault(struct pt_regs *regs)
{ {
int ret = 0; int ret = 0;
@ -261,7 +261,7 @@ void vmalloc_sync_all(void)
* *
* Handle a fault on the vmalloc or module mapping area * Handle a fault on the vmalloc or module mapping area
*/ */
static noinline __kprobes int vmalloc_fault(unsigned long address) static noinline int vmalloc_fault(unsigned long address)
{ {
unsigned long pgd_paddr; unsigned long pgd_paddr;
pmd_t *pmd_k; pmd_t *pmd_k;
@ -291,6 +291,7 @@ static noinline __kprobes int vmalloc_fault(unsigned long address)
return 0; return 0;
} }
NOKPROBE_SYMBOL(vmalloc_fault);
/* /*
* Did it hit the DOS screen memory VA from vm86 mode? * Did it hit the DOS screen memory VA from vm86 mode?
@ -358,7 +359,7 @@ void vmalloc_sync_all(void)
* *
* This assumes no large pages in there. * This assumes no large pages in there.
*/ */
static noinline __kprobes int vmalloc_fault(unsigned long address) static noinline int vmalloc_fault(unsigned long address)
{ {
pgd_t *pgd, *pgd_ref; pgd_t *pgd, *pgd_ref;
pud_t *pud, *pud_ref; pud_t *pud, *pud_ref;
@ -425,6 +426,7 @@ static noinline __kprobes int vmalloc_fault(unsigned long address)
return 0; return 0;
} }
NOKPROBE_SYMBOL(vmalloc_fault);
#ifdef CONFIG_CPU_SUP_AMD #ifdef CONFIG_CPU_SUP_AMD
static const char errata93_warning[] = static const char errata93_warning[] =
@ -927,7 +929,7 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
* There are no security implications to leaving a stale TLB when * There are no security implications to leaving a stale TLB when
* increasing the permissions on a page. * increasing the permissions on a page.
*/ */
static noinline __kprobes int static noinline int
spurious_fault(unsigned long error_code, unsigned long address) spurious_fault(unsigned long error_code, unsigned long address)
{ {
pgd_t *pgd; pgd_t *pgd;
@ -975,6 +977,7 @@ spurious_fault(unsigned long error_code, unsigned long address)
return ret; return ret;
} }
NOKPROBE_SYMBOL(spurious_fault);
int show_unhandled_signals = 1; int show_unhandled_signals = 1;
@ -1030,7 +1033,7 @@ static inline bool smap_violation(int error_code, struct pt_regs *regs)
* {,trace_}do_page_fault() have notrace on. Having this an actual function * {,trace_}do_page_fault() have notrace on. Having this an actual function
* guarantees there's a function trace entry. * guarantees there's a function trace entry.
*/ */
static void __kprobes noinline static noinline void
__do_page_fault(struct pt_regs *regs, unsigned long error_code, __do_page_fault(struct pt_regs *regs, unsigned long error_code,
unsigned long address) unsigned long address)
{ {
@ -1253,8 +1256,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
} }
NOKPROBE_SYMBOL(__do_page_fault);
dotraplinkage void __kprobes notrace dotraplinkage void notrace
do_page_fault(struct pt_regs *regs, unsigned long error_code) do_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
unsigned long address = read_cr2(); /* Get the faulting address */ unsigned long address = read_cr2(); /* Get the faulting address */
@ -1272,10 +1276,12 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
__do_page_fault(regs, error_code, address); __do_page_fault(regs, error_code, address);
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(do_page_fault);
#ifdef CONFIG_TRACING #ifdef CONFIG_TRACING
static void trace_page_fault_entries(unsigned long address, struct pt_regs *regs, static nokprobe_inline void
unsigned long error_code) trace_page_fault_entries(unsigned long address, struct pt_regs *regs,
unsigned long error_code)
{ {
if (user_mode(regs)) if (user_mode(regs))
trace_page_fault_user(address, regs, error_code); trace_page_fault_user(address, regs, error_code);
@ -1283,7 +1289,7 @@ static void trace_page_fault_entries(unsigned long address, struct pt_regs *regs
trace_page_fault_kernel(address, regs, error_code); trace_page_fault_kernel(address, regs, error_code);
} }
dotraplinkage void __kprobes notrace dotraplinkage void notrace
trace_do_page_fault(struct pt_regs *regs, unsigned long error_code) trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
{ {
/* /*
@ -1300,4 +1306,5 @@ trace_do_page_fault(struct pt_regs *regs, unsigned long error_code)
__do_page_fault(regs, error_code, address); __do_page_fault(regs, error_code, address);
exception_exit(prev_state); exception_exit(prev_state);
} }
NOKPROBE_SYMBOL(trace_do_page_fault);
#endif /* CONFIG_TRACING */ #endif /* CONFIG_TRACING */