mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

sched/fair: Add NOHZ_STATS_KICK

Browse source 
Split the NOHZ idle balancer into doing two separate actions:

 - update blocked load statistic

 - actually load-balance

Since the latter requires the former, ensure this happens. For now
always tag both bits at the same time.

Prepares for a future where we can toggle only the STATS bit.

Suggested-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Peter Zijlstra 2017-12-21 10:11:09 +01:00 committed by Ingo Molnar
parent a22e47a4e3
commit b7031a02ec
3 changed files with 41 additions and 19 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
kernel/sched/core.c
View file

@ -583,7 +583,7 @@ static inline bool got_nohz_idle_kick(void)
{ {
int cpu = smp_processor_id(); int cpu = smp_processor_id();
if (!(atomic_read(nohz_flags(cpu)) & NOHZ_BALANCE_KICK)) if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))
return false; return false;
if (idle_cpu(cpu) && !need_resched()) if (idle_cpu(cpu) && !need_resched())
@ -593,7 +593,7 @@ static inline bool got_nohz_idle_kick(void)
* We can't run Idle Load Balance on this CPU for this time so we * We can't run Idle Load Balance on this CPU for this time so we
* cancel it and clear NOHZ_BALANCE_KICK * cancel it and clear NOHZ_BALANCE_KICK
*/ */
atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(cpu)); atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));
return false; return false;
} }

52
kernel/sched/fair.c
View file

@ -9082,8 +9082,8 @@ static void nohz_balancer_kick(void)
if (ilb_cpu >= nr_cpu_ids) if (ilb_cpu >= nr_cpu_ids)
return; return;
flags = atomic_fetch_or(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu)); flags = atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(ilb_cpu));
if (flags & NOHZ_BALANCE_KICK) if (flags & NOHZ_KICK_MASK)
return; return;
/* /*
* Use smp_send_reschedule() instead of resched_cpu(). * Use smp_send_reschedule() instead of resched_cpu().
@ -9202,8 +9202,6 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
int need_serialize, need_decay = 0; int need_serialize, need_decay = 0;
u64 max_cost = 0; u64 max_cost = 0;
update_blocked_averages(cpu);
rcu_read_lock(); rcu_read_lock();
for_each_domain(cpu, sd) { for_each_domain(cpu, sd) {
/* /*
@ -9298,20 +9296,27 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
* In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
* rebalancing for all the CPUs for whom scheduler ticks are stopped. * rebalancing for all the CPUs for whom scheduler ticks are stopped.
*/ */
static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
{ {
int this_cpu = this_rq->cpu;
struct rq *rq;
int balance_cpu;
/* Earliest time when we have to do rebalance again */ /* Earliest time when we have to do rebalance again */
unsigned long next_balance = jiffies + 60*HZ; unsigned long next_balance = jiffies + 60*HZ;
int update_next_balance = 0; int update_next_balance = 0;
int this_cpu = this_rq->cpu;
unsigned int flags;
int balance_cpu;
struct rq *rq;
if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_BALANCE_KICK)) if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK))
return; return false;
if (idle != CPU_IDLE) if (idle != CPU_IDLE) {
goto end; atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
return false;
}
flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));
SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {
if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
@ -9339,7 +9344,9 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
cpu_load_update_idle(rq); cpu_load_update_idle(rq);
rq_unlock_irq(rq, &rf); rq_unlock_irq(rq, &rf);
rebalance_domains(rq, CPU_IDLE); update_blocked_averages(rq->cpu);
if (flags & NOHZ_BALANCE_KICK)
rebalance_domains(rq, CPU_IDLE);
} }
if (time_after(next_balance, rq->next_balance)) { if (time_after(next_balance, rq->next_balance)) {
@ -9348,6 +9355,10 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
} }
} }
update_blocked_averages(this_cpu);
if (flags & NOHZ_BALANCE_KICK)
rebalance_domains(this_rq, CPU_IDLE);
/* /*
* next_balance will be updated only when there is a need. * next_balance will be updated only when there is a need.
* When the CPU is attached to null domain for ex, it will not be * When the CPU is attached to null domain for ex, it will not be
@ -9355,8 +9366,8 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
*/ */
if (likely(update_next_balance)) if (likely(update_next_balance))
nohz.next_balance = next_balance; nohz.next_balance = next_balance;
end:
atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); return true;
} }
/* /*
@ -9443,7 +9454,10 @@ static inline bool nohz_kick_needed(struct rq *rq)
return kick; return kick;
} }
#else #else
static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { } static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
{
return false;
}
#endif #endif
/* /*
@ -9464,7 +9478,11 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
* load balance only within the local sched_domain hierarchy * load balance only within the local sched_domain hierarchy
* and abort nohz_idle_balance altogether if we pull some load. * and abort nohz_idle_balance altogether if we pull some load.
*/ */
nohz_idle_balance(this_rq, idle); if (nohz_idle_balance(this_rq, idle))
return;
/* normal load balance */
update_blocked_averages(this_rq->cpu);
rebalance_domains(this_rq, idle); rebalance_domains(this_rq, idle);
} }

4
kernel/sched/sched.h
View file

@ -2036,9 +2036,13 @@ extern void cfs_bandwidth_usage_dec(void);
#ifdef CONFIG_NO_HZ_COMMON #ifdef CONFIG_NO_HZ_COMMON
#define NOHZ_TICK_STOPPED_BIT 0 #define NOHZ_TICK_STOPPED_BIT 0
#define NOHZ_BALANCE_KICK_BIT 1 #define NOHZ_BALANCE_KICK_BIT 1
#define NOHZ_STATS_KICK_BIT 2
#define NOHZ_TICK_STOPPED BIT(NOHZ_TICK_STOPPED_BIT) #define NOHZ_TICK_STOPPED BIT(NOHZ_TICK_STOPPED_BIT)
#define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT) #define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
#define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT)
#define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)