forked from mirrors/linux
73ab05aa46
With KASAN and PREEMPT_RT enabled, calling task_work_add() in
task_tick_mm_cid() may cause the following splat.
[ 63.696416] BUG: sleeping function called from invalid context at kernel/locking/spinlock_rt.c:48
[ 63.696416] in_atomic(): 1, irqs_disabled(): 1, non_block: 0, pid: 610, name: modprobe
[ 63.696416] preempt_count: 10001, expected: 0
[ 63.696416] RCU nest depth: 1, expected: 1
This problem is caused by the following call trace.
sched_tick() [ acquire rq->__lock ]
-> task_tick_mm_cid()
-> task_work_add()
-> __kasan_record_aux_stack()
-> kasan_save_stack()
-> stack_depot_save_flags()
-> alloc_pages_mpol_noprof()
-> __alloc_pages_noprof()
-> get_page_from_freelist()
-> rmqueue()
-> rmqueue_pcplist()
-> __rmqueue_pcplist()
-> rmqueue_bulk()
-> rt_spin_lock()
The rq lock is a raw_spinlock_t. We can't sleep while holding
it. IOW, we can't call alloc_pages() in stack_depot_save_flags().
The task_tick_mm_cid() function with its task_work_add() call was
introduced by commit 223baf9d17 ("sched: Fix performance regression
introduced by mm_cid") in v6.4 kernel.
Fortunately, there is a kasan_record_aux_stack_noalloc() variant that
calls stack_depot_save_flags() while not allowing it to allocate
new pages. To allow task_tick_mm_cid() to use task_work without
page allocation, a new TWAF_NO_ALLOC flag is added to enable calling
kasan_record_aux_stack_noalloc() instead of kasan_record_aux_stack()
if set. The task_tick_mm_cid() function is modified to add this new flag.
The possible downside is the missing stack trace in a KASAN report due
to new page allocation required when task_work_add_noallloc() is called
which should be rare.
Fixes: 223baf9d17 ("sched: Fix performance regression introduced by mm_cid")
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20241010014432.194742-1-longman@redhat.com
46 lines
1.1 KiB
C
46 lines
1.1 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_TASK_WORK_H
|
|
#define _LINUX_TASK_WORK_H
|
|
|
|
#include <linux/list.h>
|
|
#include <linux/sched.h>
|
|
|
|
typedef void (*task_work_func_t)(struct callback_head *);
|
|
|
|
static inline void
|
|
init_task_work(struct callback_head *twork, task_work_func_t func)
|
|
{
|
|
twork->func = func;
|
|
}
|
|
|
|
enum task_work_notify_mode {
|
|
TWA_NONE = 0,
|
|
TWA_RESUME,
|
|
TWA_SIGNAL,
|
|
TWA_SIGNAL_NO_IPI,
|
|
TWA_NMI_CURRENT,
|
|
|
|
TWA_FLAGS = 0xff00,
|
|
TWAF_NO_ALLOC = 0x0100,
|
|
};
|
|
|
|
static inline bool task_work_pending(struct task_struct *task)
|
|
{
|
|
return READ_ONCE(task->task_works);
|
|
}
|
|
|
|
int task_work_add(struct task_struct *task, struct callback_head *twork,
|
|
enum task_work_notify_mode mode);
|
|
|
|
struct callback_head *task_work_cancel_match(struct task_struct *task,
|
|
bool (*match)(struct callback_head *, void *data), void *data);
|
|
struct callback_head *task_work_cancel_func(struct task_struct *, task_work_func_t);
|
|
bool task_work_cancel(struct task_struct *task, struct callback_head *cb);
|
|
void task_work_run(void);
|
|
|
|
static inline void exit_task_work(struct task_struct *task)
|
|
{
|
|
task_work_run();
|
|
}
|
|
|
|
#endif /* _LINUX_TASK_WORK_H */
|