mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

sched: Fix balance vs hotplug race

Browse source 
Since (e761b77: cpu hotplug, sched: Introduce cpu_active_map and redo
sched domain managment) we have cpu_active_mask which is suppose to rule
scheduler migration and load-balancing, except it never (fully) did.

The particular problem being solved here is a crash in try_to_wake_up()
where select_task_rq() ends up selecting an offline cpu because
select_task_rq_fair() trusts the sched_domain tree to reflect the
current state of affairs, similarly select_task_rq_rt() trusts the
root_domain.

However, the sched_domains are updated from CPU_DEAD, which is after the
cpu is taken offline and after stop_machine is done. Therefore it can
race perfectly well with code assuming the domains are right.

Cure this by building the domains from cpu_active_mask on
CPU_DOWN_PREPARE.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
Peter Zijlstra 2009-11-25 13:31:39 +01:00 committed by Ingo Molnar
parent e1b8090bdf
commit 6ad4c18884
4 changed files with 41 additions and 27 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

2
include/linux/cpumask.h
View file

@ -84,6 +84,7 @@ extern const struct cpumask *const cpu_active_mask;
#define num_online_cpus() cpumask_weight(cpu_online_mask) #define num_online_cpus() cpumask_weight(cpu_online_mask)
#define num_possible_cpus() cpumask_weight(cpu_possible_mask) #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask) #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask) #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask) #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
@ -92,6 +93,7 @@ extern const struct cpumask *const cpu_active_mask;
#define num_online_cpus() 1 #define num_online_cpus() 1
#define num_possible_cpus() 1 #define num_possible_cpus() 1
#define num_present_cpus() 1 #define num_present_cpus() 1
#define num_active_cpus() 1
#define cpu_online(cpu) ((cpu) == 0) #define cpu_online(cpu) ((cpu) == 0)
#define cpu_possible(cpu) ((cpu) == 0) #define cpu_possible(cpu) ((cpu) == 0)
#define cpu_present(cpu) ((cpu) == 0) #define cpu_present(cpu) ((cpu) == 0)

18
kernel/cpu.c
View file

@ -212,6 +212,8 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod, err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
hcpu, -1, &nr_calls); hcpu, -1, &nr_calls);
if (err == NOTIFY_BAD) { if (err == NOTIFY_BAD) {
set_cpu_active(cpu, true);
nr_calls--; nr_calls--;
__raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod, __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
hcpu, nr_calls, NULL); hcpu, nr_calls, NULL);
@ -223,11 +225,11 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
/* Ensure that we are not runnable on dying cpu */ /* Ensure that we are not runnable on dying cpu */
cpumask_copy(old_allowed, &current->cpus_allowed); cpumask_copy(old_allowed, &current->cpus_allowed);
set_cpus_allowed_ptr(current, set_cpus_allowed_ptr(current, cpu_active_mask);
cpumask_of(cpumask_any_but(cpu_online_mask, cpu)));
err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
if (err) { if (err) {
set_cpu_active(cpu, true);
/* CPU didn't die: tell everyone. Can't complain. */ /* CPU didn't die: tell everyone. Can't complain. */
if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod, if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
hcpu) == NOTIFY_BAD) hcpu) == NOTIFY_BAD)
@ -292,9 +294,6 @@ int __ref cpu_down(unsigned int cpu)
err = _cpu_down(cpu, 0); err = _cpu_down(cpu, 0);
if (cpu_online(cpu))
set_cpu_active(cpu, true);
out: out:
cpu_maps_update_done(); cpu_maps_update_done();
stop_machine_destroy(); stop_machine_destroy();
@ -387,6 +386,15 @@ int disable_nonboot_cpus(void)
* with the userspace trying to use the CPU hotplug at the same time * with the userspace trying to use the CPU hotplug at the same time
*/ */
cpumask_clear(frozen_cpus); cpumask_clear(frozen_cpus);
for_each_online_cpu(cpu) {
if (cpu == first_cpu)
continue;
set_cpu_active(cpu, false);
}
synchronize_sched();
printk("Disabling non-boot CPUs ...\n"); printk("Disabling non-boot CPUs ...\n");
for_each_online_cpu(cpu) { for_each_online_cpu(cpu) {
if (cpu == first_cpu) if (cpu == first_cpu)

16
kernel/cpuset.c
View file

@ -872,7 +872,7 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
if (retval < 0) if (retval < 0)
return retval; return retval;
if (!cpumask_subset(trialcs->cpus_allowed, cpu_online_mask)) if (!cpumask_subset(trialcs->cpus_allowed, cpu_active_mask))
return -EINVAL; return -EINVAL;
} }
retval = validate_change(cs, trialcs); retval = validate_change(cs, trialcs);
@ -2010,7 +2010,7 @@ static void scan_for_empty_cpusets(struct cpuset *root)
} }
/* Continue past cpusets with all cpus, mems online */ /* Continue past cpusets with all cpus, mems online */
if (cpumask_subset(cp->cpus_allowed, cpu_online_mask) && if (cpumask_subset(cp->cpus_allowed, cpu_active_mask) &&
nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY])) nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY]))
continue; continue;
@ -2019,7 +2019,7 @@ static void scan_for_empty_cpusets(struct cpuset *root)
/* Remove offline cpus and mems from this cpuset. */ /* Remove offline cpus and mems from this cpuset. */
mutex_lock(&callback_mutex); mutex_lock(&callback_mutex);
cpumask_and(cp->cpus_allowed, cp->cpus_allowed, cpumask_and(cp->cpus_allowed, cp->cpus_allowed,
cpu_online_mask); cpu_active_mask);
nodes_and(cp->mems_allowed, cp->mems_allowed, nodes_and(cp->mems_allowed, cp->mems_allowed,
node_states[N_HIGH_MEMORY]); node_states[N_HIGH_MEMORY]);
mutex_unlock(&callback_mutex); mutex_unlock(&callback_mutex);
@ -2057,8 +2057,10 @@ static int cpuset_track_online_cpus(struct notifier_block *unused_nb,
switch (phase) { switch (phase) {
case CPU_ONLINE: case CPU_ONLINE:
case CPU_ONLINE_FROZEN: case CPU_ONLINE_FROZEN:
case CPU_DEAD: case CPU_DOWN_PREPARE:
case CPU_DEAD_FROZEN: case CPU_DOWN_PREPARE_FROZEN:
case CPU_DOWN_FAILED:
case CPU_DOWN_FAILED_FROZEN:
break; break;
default: default:
@ -2067,7 +2069,7 @@ static int cpuset_track_online_cpus(struct notifier_block *unused_nb,
cgroup_lock(); cgroup_lock();
mutex_lock(&callback_mutex); mutex_lock(&callback_mutex);
cpumask_copy(top_cpuset.cpus_allowed, cpu_online_mask); cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
mutex_unlock(&callback_mutex); mutex_unlock(&callback_mutex);
scan_for_empty_cpusets(&top_cpuset); scan_for_empty_cpusets(&top_cpuset);
ndoms = generate_sched_domains(&doms, &attr); ndoms = generate_sched_domains(&doms, &attr);
@ -2114,7 +2116,7 @@ static int cpuset_track_online_nodes(struct notifier_block *self,
void __init cpuset_init_smp(void) void __init cpuset_init_smp(void)
{ {
cpumask_copy(top_cpuset.cpus_allowed, cpu_online_mask); cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
hotcpu_notifier(cpuset_track_online_cpus, 0); hotcpu_notifier(cpuset_track_online_cpus, 0);

32
kernel/sched.c
View file

@ -4134,7 +4134,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
unsigned long flags; unsigned long flags;
struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
cpumask_copy(cpus, cpu_online_mask); cpumask_copy(cpus, cpu_active_mask);
/* /*
* When power savings policy is enabled for the parent domain, idle * When power savings policy is enabled for the parent domain, idle
@ -4297,7 +4297,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
int all_pinned = 0; int all_pinned = 0;
struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
cpumask_copy(cpus, cpu_online_mask); cpumask_copy(cpus, cpu_active_mask);
/* /*
* When power savings policy is enabled for the parent domain, idle * When power savings policy is enabled for the parent domain, idle
@ -4694,7 +4694,7 @@ int select_nohz_load_balancer(int stop_tick)
cpumask_set_cpu(cpu, nohz.cpu_mask); cpumask_set_cpu(cpu, nohz.cpu_mask);
/* time for ilb owner also to sleep */ /* time for ilb owner also to sleep */
if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) { if (cpumask_weight(nohz.cpu_mask) == num_active_cpus()) {
if (atomic_read(&nohz.load_balancer) == cpu) if (atomic_read(&nohz.load_balancer) == cpu)
atomic_set(&nohz.load_balancer, -1); atomic_set(&nohz.load_balancer, -1);
return 0; return 0;
@ -7093,7 +7093,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
int ret = 0; int ret = 0;
rq = task_rq_lock(p, &flags); rq = task_rq_lock(p, &flags);
if (!cpumask_intersects(new_mask, cpu_online_mask)) { if (!cpumask_intersects(new_mask, cpu_active_mask)) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
@ -7115,7 +7115,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
if (cpumask_test_cpu(task_cpu(p), new_mask)) if (cpumask_test_cpu(task_cpu(p), new_mask))
goto out; goto out;
if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) { if (migrate_task(p, cpumask_any_and(cpu_active_mask, new_mask), &req)) {
/* Need help from migration thread: drop lock and wait. */ /* Need help from migration thread: drop lock and wait. */
struct task_struct *mt = rq->migration_thread; struct task_struct *mt = rq->migration_thread;
@ -7269,19 +7269,19 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
again: again:
/* Look for allowed, online CPU in same node. */ /* Look for allowed, online CPU in same node. */
for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask) for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed)) if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
goto move; goto move;
/* Any allowed, online CPU? */ /* Any allowed, online CPU? */
dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask); dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
if (dest_cpu < nr_cpu_ids) if (dest_cpu < nr_cpu_ids)
goto move; goto move;
/* No more Mr. Nice Guy. */ /* No more Mr. Nice Guy. */
if (dest_cpu >= nr_cpu_ids) { if (dest_cpu >= nr_cpu_ids) {
cpuset_cpus_allowed_locked(p, &p->cpus_allowed); cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed); dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
/* /*
* Don't tell them about moving exiting tasks or * Don't tell them about moving exiting tasks or
@ -7310,7 +7310,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
*/ */
static void migrate_nr_uninterruptible(struct rq *rq_src) static void migrate_nr_uninterruptible(struct rq *rq_src)
{ {
struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask)); struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
unsigned long flags; unsigned long flags;
local_irq_save(flags); local_irq_save(flags);
@ -7564,7 +7564,7 @@ static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
static struct ctl_table_header *sd_sysctl_header; static struct ctl_table_header *sd_sysctl_header;
static void register_sched_domain_sysctl(void) static void register_sched_domain_sysctl(void)
{ {
int i, cpu_num = num_online_cpus(); int i, cpu_num = num_possible_cpus();
struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
char buf[32]; char buf[32];
@ -7574,7 +7574,7 @@ static void register_sched_domain_sysctl(void)
if (entry == NULL) if (entry == NULL)
return; return;
for_each_online_cpu(i) { for_each_possible_cpu(i) {
snprintf(buf, 32, "cpu%d", i); snprintf(buf, 32, "cpu%d", i);
entry->procname = kstrdup(buf, GFP_KERNEL); entry->procname = kstrdup(buf, GFP_KERNEL);
entry->mode = 0555; entry->mode = 0555;
@ -9100,7 +9100,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
if (doms_new == NULL) { if (doms_new == NULL) {
ndoms_cur = 0; ndoms_cur = 0;
doms_new = &fallback_doms; doms_new = &fallback_doms;
cpumask_andnot(doms_new[0], cpu_online_mask, cpu_isolated_map); cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
WARN_ON_ONCE(dattr_new); WARN_ON_ONCE(dattr_new);
} }
@ -9231,8 +9231,10 @@ static int update_sched_domains(struct notifier_block *nfb,
switch (action) { switch (action) {
case CPU_ONLINE: case CPU_ONLINE:
case CPU_ONLINE_FROZEN: case CPU_ONLINE_FROZEN:
case CPU_DEAD: case CPU_DOWN_PREPARE:
case CPU_DEAD_FROZEN: case CPU_DOWN_PREPARE_FROZEN:
case CPU_DOWN_FAILED:
case CPU_DOWN_FAILED_FROZEN:
partition_sched_domains(1, NULL, NULL); partition_sched_domains(1, NULL, NULL);
return NOTIFY_OK; return NOTIFY_OK;
@ -9279,7 +9281,7 @@ void __init sched_init_smp(void)
#endif #endif
get_online_cpus(); get_online_cpus();
mutex_lock(&sched_domains_mutex); mutex_lock(&sched_domains_mutex);
arch_init_sched_domains(cpu_online_mask); arch_init_sched_domains(cpu_active_mask);
cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
if (cpumask_empty(non_isolated_cpus)) if (cpumask_empty(non_isolated_cpus))
cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);