mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-01 00:58:39 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

x86: Put hot per CPU variables into a struct

Browse source 
The layout of per-cpu variables is at the mercy of the compiler. This
can lead to random performance fluctuations from build to build.

Create a structure to hold some of the hottest per-cpu variables,
starting with current_task.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111145.179707194@infradead.org
This commit is contained in:
Thomas Gleixner 2022-09-15 13:11:01 +02:00 committed by Peter Zijlstra
parent fdc9ee7e97
commit e57ef2ed97
5 changed files with 24 additions and 15 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

19
arch/x86/include/asm/current.h
View file

@ -3,16 +3,29 @@
#define _ASM_X86_CURRENT_H #define _ASM_X86_CURRENT_H
#include <linux/compiler.h> #include <linux/compiler.h>
#include <asm/percpu.h>
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/cache.h>
#include <asm/percpu.h>
struct task_struct; struct task_struct;
DECLARE_PER_CPU(struct task_struct *, current_task); struct pcpu_hot {
union {
struct {
struct task_struct *current_task;
};
u8 pad[64];
};
};
static_assert(sizeof(struct pcpu_hot) == 64);
DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
static __always_inline struct task_struct *get_current(void) static __always_inline struct task_struct *get_current(void)
{ {
return this_cpu_read_stable(current_task); return this_cpu_read_stable(pcpu_hot.current_task);
} }
#define current get_current() #define current get_current()

14
arch/x86/kernel/cpu/common.c
View file

@ -2012,18 +2012,16 @@ static __init int setup_clearcpuid(char *arg)
} }
__setup("clearcpuid=", setup_clearcpuid); __setup("clearcpuid=", setup_clearcpuid);
DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
.current_task = &init_task,
};
EXPORT_PER_CPU_SYMBOL(pcpu_hot);
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
DEFINE_PER_CPU_FIRST(struct fixed_percpu_data, DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
fixed_percpu_data) __aligned(PAGE_SIZE) __visible; fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data); EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
/*
* The following percpu variables are hot. Align current_task to
* cacheline size such that they fall in the same cacheline.
*/
DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
&init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
DEFINE_PER_CPU(void *, hardirq_stack_ptr); DEFINE_PER_CPU(void *, hardirq_stack_ptr);
DEFINE_PER_CPU(bool, hardirq_stack_inuse); DEFINE_PER_CPU(bool, hardirq_stack_inuse);
@ -2083,8 +2081,6 @@ void syscall_init(void)
#else /* CONFIG_X86_64 */ #else /* CONFIG_X86_64 */
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT; DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count); EXPORT_PER_CPU_SYMBOL(__preempt_count);

2
arch/x86/kernel/process_32.c
View file

@ -207,7 +207,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs) if (prev->gs | next->gs)
loadsegment(gs, next->gs); loadsegment(gs, next->gs);
this_cpu_write(current_task, next_p); raw_cpu_write(pcpu_hot.current_task, next_p);
switch_fpu_finish(); switch_fpu_finish();

2
arch/x86/kernel/process_64.c
View file

@ -617,7 +617,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/* /*
* Switch the PDA and FPU contexts. * Switch the PDA and FPU contexts.
*/ */
this_cpu_write(current_task, next_p); raw_cpu_write(pcpu_hot.current_task, next_p);
this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p)); this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));
switch_fpu_finish(); switch_fpu_finish();

2
arch/x86/kernel/smpboot.c
View file

@ -1046,7 +1046,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle)
/* Just in case we booted with a single CPU. */ /* Just in case we booted with a single CPU. */
alternatives_enable_smp(); alternatives_enable_smp();
per_cpu(current_task, cpu) = idle; per_cpu(pcpu_hot.current_task, cpu) = idle;
cpu_init_stack_canary(cpu, idle); cpu_init_stack_canary(cpu, idle);
/* Initialize the interrupt stack(s) */ /* Initialize the interrupt stack(s) */