mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

x86/entry: Consolidate 32/64 bit syscall entry

Browse source 
64bit and 32bit entry code have the same open coded syscall entry handling
after the bitwidth specific bits.

Move it to a helper function and share the code.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://lkml.kernel.org/r/20200722220520.051234096@linutronix.de
This commit is contained in:
Thomas Gleixner 2020-07-23 00:00:01 +02:00
parent 8d5ea35c5e
commit 0b085e68f4
1 changed files with 42 additions and 53 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

95
arch/x86/entry/common.c
View file

@ -366,8 +366,7 @@ __visible noinstr void syscall_return_slowpath(struct pt_regs *regs)
exit_to_user_mode();
}
#ifdef CONFIG_X86_64
__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
static noinstr long syscall_enter(struct pt_regs *regs, unsigned long nr)
{
struct thread_info *ti;
@ -379,6 +378,16 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
nr = syscall_trace_enter(regs);
instrumentation_end();
return nr;
}
#ifdef CONFIG_X86_64
__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
{
nr = syscall_enter(regs, nr);
instrumentation_begin();
if (likely(nr < NR_syscalls)) {
nr = array_index_nospec(nr, NR_syscalls);
regs->ax = sys_call_table[nr](regs);
@ -390,64 +399,53 @@ __visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs)
regs->ax = x32_sys_call_table[nr](regs);
#endif
}
__syscall_return_slowpath(regs);
instrumentation_end();
exit_to_user_mode();
syscall_return_slowpath(regs);
}
#endif
#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
/*
* Does a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL. Does
* all entry and exit work and returns with IRQs off. This function is
* extremely hot in workloads that use it, and it's usually called from
* do_fast_syscall_32, so forcibly inline it to improve performance.
*/
static void do_syscall_32_irqs_on(struct pt_regs *regs)
static __always_inline unsigned int syscall_32_enter(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
unsigned int nr = (unsigned int)regs->orig_ax;
#ifdef CONFIG_IA32_EMULATION
ti->status |= TS_COMPAT;
#endif
if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
/*
* Subtlety here: if ptrace pokes something larger than
* 2^32-1 into orig_ax, this truncates it. This may or
* may not be necessary, but it matches the old asm
* behavior.
*/
nr = syscall_trace_enter(regs);
}
if (IS_ENABLED(CONFIG_IA32_EMULATION))
current_thread_info()->status |= TS_COMPAT;
/*
* Subtlety here: if ptrace pokes something larger than 2^32-1 into
* orig_ax, the unsigned int return value truncates it. This may
* or may not be necessary, but it matches the old asm behavior.
*/
return syscall_enter(regs, (unsigned int)regs->orig_ax);
}
/*
* Invoke a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL.
*/
static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs,
unsigned int nr)
{
if (likely(nr < IA32_NR_syscalls)) {
instrumentation_begin();
nr = array_index_nospec(nr, IA32_NR_syscalls);
regs->ax = ia32_sys_call_table[nr](regs);
instrumentation_end();
}
__syscall_return_slowpath(regs);
}
/* Handles int $0x80 */
__visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
{
enter_from_user_mode(regs);
instrumentation_begin();
unsigned int nr = syscall_32_enter(regs);
local_irq_enable();
do_syscall_32_irqs_on(regs);
instrumentation_end();
exit_to_user_mode();
do_syscall_32_irqs_on(regs, nr);
syscall_return_slowpath(regs);
}
static bool __do_fast_syscall_32(struct pt_regs *regs)
static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
{
unsigned int nr = syscall_32_enter(regs);
int res;
instrumentation_begin();
/* Fetch EBP from where the vDSO stashed it. */
if (IS_ENABLED(CONFIG_X86_64)) {
/*
@ -460,17 +458,18 @@ static bool __do_fast_syscall_32(struct pt_regs *regs)
res = get_user(*(u32 *)&regs->bp,
(u32 __user __force *)(unsigned long)(u32)regs->sp);
}
instrumentation_end();
if (res) {
/* User code screwed up. */
regs->ax = -EFAULT;
local_irq_disable();
__prepare_exit_to_usermode(regs);
syscall_return_slowpath(regs);
return false;
}
/* Now this is just like a normal syscall. */
do_syscall_32_irqs_on(regs);
do_syscall_32_irqs_on(regs, nr);
syscall_return_slowpath(regs);
return true;
}
@ -483,7 +482,6 @@ __visible noinstr long do_fast_syscall_32(struct pt_regs *regs)
*/
unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
vdso_image_32.sym_int80_landing_pad;
bool success;
/*
* SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
@ -492,17 +490,8 @@ __visible noinstr long do_fast_syscall_32(struct pt_regs *regs)
*/
regs->ip = landing_pad;
enter_from_user_mode(regs);
instrumentation_begin();
local_irq_enable();
success = __do_fast_syscall_32(regs);
instrumentation_end();
exit_to_user_mode();
/* If it failed, keep it simple: use IRET. */
if (!success)
/* Invoke the syscall. If it failed, keep it simple: use IRET. */
if (!__do_fast_syscall_32(regs))
return 0;
#ifdef CONFIG_X86_64