mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

sched/fair: Add NOHZ balancer flag for nohz.next_balance updates

Browse source 
A following patch will trigger NOHZ idle balances as a means to update
nohz.next_balance. Vincent noted that blocked load updates can have
non-negligible overhead, which should be avoided if the intent is to only
update nohz.next_balance.

Add a new NOHZ balance kick flag, NOHZ_NEXT_KICK. Gate NOHZ blocked load
update by the presence of NOHZ_STATS_KICK - currently all NOHZ balance
kicks will have the NOHZ_STATS_KICK flag set, so no change in behaviour is
expected.

Suggested-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Valentin Schneider <valentin.schneider@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Vincent Guittot <vincent.guittot@linaro.org>
Link: https://lkml.kernel.org/r/20210823111700.2842997-2-valentin.schneider@arm.com
This commit is contained in:
Valentin Schneider 2021-08-23 12:16:59 +01:00 committed by Peter Zijlstra
parent 9e1ff307c7
commit efd984c481
2 changed files with 21 additions and 11 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

24
kernel/sched/fair.c
View file

@ -10375,7 +10375,7 @@ static void nohz_balancer_kick(struct rq *rq)
goto out; goto out;
if (rq->nr_running >= 2) { if (rq->nr_running >= 2) {
flags = NOHZ_KICK_MASK; flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto out; goto out;
} }
@ -10389,7 +10389,7 @@ static void nohz_balancer_kick(struct rq *rq)
* on. * on.
*/ */
if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) { if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) {
flags = NOHZ_KICK_MASK; flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock; goto unlock;
} }
} }
@ -10403,7 +10403,7 @@ static void nohz_balancer_kick(struct rq *rq)
*/ */
for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
if (sched_asym_prefer(i, cpu)) { if (sched_asym_prefer(i, cpu)) {
flags = NOHZ_KICK_MASK; flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock; goto unlock;
} }
} }
@ -10416,7 +10416,7 @@ static void nohz_balancer_kick(struct rq *rq)
* to run the misfit task on. * to run the misfit task on.
*/ */
if (check_misfit_status(rq, sd)) { if (check_misfit_status(rq, sd)) {
flags = NOHZ_KICK_MASK; flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock; goto unlock;
} }
@ -10443,7 +10443,7 @@ static void nohz_balancer_kick(struct rq *rq)
*/ */
nr_busy = atomic_read(&sds->nr_busy_cpus); nr_busy = atomic_read(&sds->nr_busy_cpus);
if (nr_busy > 1) { if (nr_busy > 1) {
flags = NOHZ_KICK_MASK; flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
goto unlock; goto unlock;
} }
} }
@ -10605,7 +10605,8 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
* setting the flag, we are sure to not clear the state and not * setting the flag, we are sure to not clear the state and not
* check the load of an idle cpu. * check the load of an idle cpu.
*/ */
WRITE_ONCE(nohz.has_blocked, 0); if (flags & NOHZ_STATS_KICK)
WRITE_ONCE(nohz.has_blocked, 0);
/* /*
* Ensures that if we miss the CPU, we must see the has_blocked * Ensures that if we miss the CPU, we must see the has_blocked
@ -10627,13 +10628,15 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
* balancing owner will pick it up. * balancing owner will pick it up.
*/ */
if (need_resched()) { if (need_resched()) {
has_blocked_load = true; if (flags & NOHZ_STATS_KICK)
has_blocked_load = true;
goto abort; goto abort;
} }
rq = cpu_rq(balance_cpu); rq = cpu_rq(balance_cpu);
has_blocked_load |= update_nohz_stats(rq); if (flags & NOHZ_STATS_KICK)
has_blocked_load |= update_nohz_stats(rq);
/* /*
* If time for next balance is due, * If time for next balance is due,
@ -10664,8 +10667,9 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags,
if (likely(update_next_balance)) if (likely(update_next_balance))
nohz.next_balance = next_balance; nohz.next_balance = next_balance;
WRITE_ONCE(nohz.next_blocked, if (flags & NOHZ_STATS_KICK)
now + msecs_to_jiffies(LOAD_AVG_PERIOD)); WRITE_ONCE(nohz.next_blocked,
now + msecs_to_jiffies(LOAD_AVG_PERIOD));
abort: abort:
/* There is still blocked load, enable periodic update */ /* There is still blocked load, enable periodic update */

8
kernel/sched/sched.h
View file

@ -2709,12 +2709,18 @@ extern void cfs_bandwidth_usage_dec(void);
#define NOHZ_BALANCE_KICK_BIT 0 #define NOHZ_BALANCE_KICK_BIT 0
#define NOHZ_STATS_KICK_BIT 1 #define NOHZ_STATS_KICK_BIT 1
#define NOHZ_NEWILB_KICK_BIT 2 #define NOHZ_NEWILB_KICK_BIT 2
#define NOHZ_NEXT_KICK_BIT 3
/* Run rebalance_domains() */
#define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT) #define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
/* Update blocked load */
#define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT) #define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT)
/* Update blocked load when entering idle */
#define NOHZ_NEWILB_KICK BIT(NOHZ_NEWILB_KICK_BIT) #define NOHZ_NEWILB_KICK BIT(NOHZ_NEWILB_KICK_BIT)
/* Update nohz.next_balance */
#define NOHZ_NEXT_KICK BIT(NOHZ_NEXT_KICK_BIT)
#define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK) #define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK | NOHZ_NEXT_KICK)
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)