mirror of
https://github.com/torvalds/linux.git
synced 2025-11-04 10:40:15 +02:00
rcu: Introduce rcutorture testing for rcu_barrier()
Although rcutorture does invoke rcu_barrier() and friends, it cannot really be called a torture test given that it invokes them only once at the end of the test. This commit therefore introduces heavy-duty rcutorture testing for rcu_barrier(), which may be carried out concurrently with normal rcutorture testing. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Paul E. McKenney
Paul E. McKenney
parent
37e377d282
commit
fae4b54f28
2 changed files with 200 additions and 9 deletions
|
|
@ -47,6 +47,16 @@ irqreader Says to invoke RCU readers from irq level. This is currently
|
|||
permit this. (Or, more accurately, variants of RCU that do
|
||||
-not- permit this know to ignore this variable.)
|
||||
|
||||
n_barrier_cbs If this is nonzero, RCU barrier testing will be conducted,
|
||||
in which case n_barrier_cbs specifies the number of
|
||||
RCU callbacks (and corresponding kthreads) to use for
|
||||
this testing. The value cannot be negative. If you
|
||||
specify this to be non-zero when torture_type indicates a
|
||||
synchronous RCU implementation (one for which a member of
|
||||
the synchronize_rcu() rather than the call_rcu() family is
|
||||
used -- see the documentation for torture_type below), an
|
||||
error will be reported and no testing will be carried out.
|
||||
|
||||
nfakewriters This is the number of RCU fake writer threads to run. Fake
|
||||
writer threads repeatedly use the synchronous "wait for
|
||||
current readers" function of the interface selected by
|
||||
|
|
@ -188,7 +198,7 @@ OUTPUT
|
|||
The statistics output is as follows:
|
||||
|
||||
rcu-torture:--- Start of test: nreaders=16 nfakewriters=4 stat_interval=30 verbose=0 test_no_idle_hz=1 shuffle_interval=3 stutter=5 irqreader=1 fqs_duration=0 fqs_holdoff=0 fqs_stutter=3 test_boost=1/0 test_boost_interval=7 test_boost_duration=4
|
||||
rcu-torture: rtc: (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767
|
||||
rcu-torture: rtc: (null) ver: 155441 tfle: 0 rta: 155441 rtaf: 8884 rtf: 155440 rtmbe: 0 rtbe: 0 rtbke: 0 rtbre: 0 rtbf: 0 rtb: 0 nt: 3055767
|
||||
rcu-torture: Reader Pipe: 727860534 34213 0 0 0 0 0 0 0 0 0
|
||||
rcu-torture: Reader Batch: 727877838 17003 0 0 0 0 0 0 0 0 0
|
||||
rcu-torture: Free-Block Circulation: 155440 155440 155440 155440 155440 155440 155440 155440 155440 155440 0
|
||||
|
|
@ -230,6 +240,9 @@ o "rtmbe": A non-zero value indicates that rcutorture believes that
|
|||
rcu_assign_pointer() and rcu_dereference() are not working
|
||||
correctly. This value should be zero.
|
||||
|
||||
o "rtbe": A non-zero value indicates that one of the rcu_barrier()
|
||||
family of functions is not working correctly.
|
||||
|
||||
o "rtbke": rcutorture was unable to create the real-time kthreads
|
||||
used to force RCU priority inversion. This value should be zero.
|
||||
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ static int irqreader = 1; /* RCU readers from irq (timers). */
|
|||
static int fqs_duration; /* Duration of bursts (us), 0 to disable. */
|
||||
static int fqs_holdoff; /* Hold time within burst (us). */
|
||||
static int fqs_stutter = 3; /* Wait time between bursts (s). */
|
||||
static int n_barrier_cbs; /* Number of callbacks to test RCU barriers. */
|
||||
static int onoff_interval; /* Wait time between CPU hotplugs, 0=disable. */
|
||||
static int onoff_holdoff; /* Seconds after boot before CPU hotplugs. */
|
||||
static int shutdown_secs; /* Shutdown time (s). <=0 for no shutdown. */
|
||||
|
|
@ -96,6 +97,8 @@ module_param(fqs_holdoff, int, 0444);
|
|||
MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
|
||||
module_param(fqs_stutter, int, 0444);
|
||||
MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
|
||||
module_param(n_barrier_cbs, int, 0444);
|
||||
MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
|
||||
module_param(onoff_interval, int, 0444);
|
||||
MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
|
||||
module_param(onoff_holdoff, int, 0444);
|
||||
|
|
@ -139,6 +142,8 @@ static struct task_struct *shutdown_task;
|
|||
static struct task_struct *onoff_task;
|
||||
#endif /* #ifdef CONFIG_HOTPLUG_CPU */
|
||||
static struct task_struct *stall_task;
|
||||
static struct task_struct **barrier_cbs_tasks;
|
||||
static struct task_struct *barrier_task;
|
||||
|
||||
#define RCU_TORTURE_PIPE_LEN 10
|
||||
|
||||
|
|
@ -164,6 +169,7 @@ static atomic_t n_rcu_torture_alloc_fail;
|
|||
static atomic_t n_rcu_torture_free;
|
||||
static atomic_t n_rcu_torture_mberror;
|
||||
static atomic_t n_rcu_torture_error;
|
||||
static long n_rcu_torture_barrier_error;
|
||||
static long n_rcu_torture_boost_ktrerror;
|
||||
static long n_rcu_torture_boost_rterror;
|
||||
static long n_rcu_torture_boost_failure;
|
||||
|
|
@ -173,6 +179,8 @@ static long n_offline_attempts;
|
|||
static long n_offline_successes;
|
||||
static long n_online_attempts;
|
||||
static long n_online_successes;
|
||||
static long n_barrier_attempts;
|
||||
static long n_barrier_successes;
|
||||
static struct list_head rcu_torture_removed;
|
||||
static cpumask_var_t shuffle_tmp_mask;
|
||||
|
||||
|
|
@ -197,6 +205,10 @@ static unsigned long shutdown_time; /* jiffies to system shutdown. */
|
|||
static unsigned long boost_starttime; /* jiffies of next boost test start. */
|
||||
DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */
|
||||
/* and boost task create/destroy. */
|
||||
static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */
|
||||
static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */
|
||||
static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
|
||||
static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
|
||||
|
||||
/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */
|
||||
|
||||
|
|
@ -327,6 +339,7 @@ struct rcu_torture_ops {
|
|||
int (*completed)(void);
|
||||
void (*deferred_free)(struct rcu_torture *p);
|
||||
void (*sync)(void);
|
||||
void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
|
||||
void (*cb_barrier)(void);
|
||||
void (*fqs)(void);
|
||||
int (*stats)(char *page);
|
||||
|
|
@ -417,6 +430,7 @@ static struct rcu_torture_ops rcu_ops = {
|
|||
.completed = rcu_torture_completed,
|
||||
.deferred_free = rcu_torture_deferred_free,
|
||||
.sync = synchronize_rcu,
|
||||
.call = call_rcu,
|
||||
.cb_barrier = rcu_barrier,
|
||||
.fqs = rcu_force_quiescent_state,
|
||||
.stats = NULL,
|
||||
|
|
@ -460,6 +474,7 @@ static struct rcu_torture_ops rcu_sync_ops = {
|
|||
.completed = rcu_torture_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = synchronize_rcu,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.fqs = rcu_force_quiescent_state,
|
||||
.stats = NULL,
|
||||
|
|
@ -477,6 +492,7 @@ static struct rcu_torture_ops rcu_expedited_ops = {
|
|||
.completed = rcu_no_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = synchronize_rcu_expedited,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.fqs = rcu_force_quiescent_state,
|
||||
.stats = NULL,
|
||||
|
|
@ -519,6 +535,7 @@ static struct rcu_torture_ops rcu_bh_ops = {
|
|||
.completed = rcu_bh_torture_completed,
|
||||
.deferred_free = rcu_bh_torture_deferred_free,
|
||||
.sync = synchronize_rcu_bh,
|
||||
.call = call_rcu_bh,
|
||||
.cb_barrier = rcu_barrier_bh,
|
||||
.fqs = rcu_bh_force_quiescent_state,
|
||||
.stats = NULL,
|
||||
|
|
@ -535,6 +552,7 @@ static struct rcu_torture_ops rcu_bh_sync_ops = {
|
|||
.completed = rcu_bh_torture_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = synchronize_rcu_bh,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.fqs = rcu_bh_force_quiescent_state,
|
||||
.stats = NULL,
|
||||
|
|
@ -551,6 +569,7 @@ static struct rcu_torture_ops rcu_bh_expedited_ops = {
|
|||
.completed = rcu_bh_torture_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = synchronize_rcu_bh_expedited,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.fqs = rcu_bh_force_quiescent_state,
|
||||
.stats = NULL,
|
||||
|
|
@ -637,6 +656,7 @@ static struct rcu_torture_ops srcu_ops = {
|
|||
.completed = srcu_torture_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = srcu_torture_synchronize,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.stats = srcu_torture_stats,
|
||||
.name = "srcu"
|
||||
|
|
@ -661,6 +681,7 @@ static struct rcu_torture_ops srcu_raw_ops = {
|
|||
.completed = srcu_torture_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = srcu_torture_synchronize,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.stats = srcu_torture_stats,
|
||||
.name = "srcu_raw"
|
||||
|
|
@ -680,6 +701,7 @@ static struct rcu_torture_ops srcu_expedited_ops = {
|
|||
.completed = srcu_torture_completed,
|
||||
.deferred_free = rcu_sync_torture_deferred_free,
|
||||
.sync = srcu_torture_synchronize_expedited,
|
||||
.call = NULL,
|
||||
.cb_barrier = NULL,
|
||||
.stats = srcu_torture_stats,
|
||||
.name = "srcu_expedited"
|
||||
|
|
@ -1129,7 +1151,8 @@ rcu_torture_printk(char *page)
|
|||
"rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d "
|
||||
"rtmbe: %d rtbke: %ld rtbre: %ld "
|
||||
"rtbf: %ld rtb: %ld nt: %ld "
|
||||
"onoff: %ld/%ld:%ld/%ld",
|
||||
"onoff: %ld/%ld:%ld/%ld "
|
||||
"barrier: %ld/%ld:%ld",
|
||||
rcu_torture_current,
|
||||
rcu_torture_current_version,
|
||||
list_empty(&rcu_torture_freelist),
|
||||
|
|
@ -1145,14 +1168,17 @@ rcu_torture_printk(char *page)
|
|||
n_online_successes,
|
||||
n_online_attempts,
|
||||
n_offline_successes,
|
||||
n_offline_attempts);
|
||||
n_offline_attempts,
|
||||
n_barrier_successes,
|
||||
n_barrier_attempts,
|
||||
n_rcu_torture_barrier_error);
|
||||
cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
|
||||
if (atomic_read(&n_rcu_torture_mberror) != 0 ||
|
||||
n_rcu_torture_barrier_error != 0 ||
|
||||
n_rcu_torture_boost_ktrerror != 0 ||
|
||||
n_rcu_torture_boost_rterror != 0 ||
|
||||
n_rcu_torture_boost_failure != 0)
|
||||
cnt += sprintf(&page[cnt], " !!!");
|
||||
cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
|
||||
if (i > 1) {
|
||||
n_rcu_torture_boost_failure != 0 ||
|
||||
i > 1) {
|
||||
cnt += sprintf(&page[cnt], "!!! ");
|
||||
atomic_inc(&n_rcu_torture_error);
|
||||
WARN_ON_ONCE(1);
|
||||
|
|
@ -1560,6 +1586,151 @@ static void rcu_torture_stall_cleanup(void)
|
|||
stall_task = NULL;
|
||||
}
|
||||
|
||||
/* Callback function for RCU barrier testing. */
|
||||
void rcu_torture_barrier_cbf(struct rcu_head *rcu)
|
||||
{
|
||||
atomic_inc(&barrier_cbs_invoked);
|
||||
}
|
||||
|
||||
/* kthread function to register callbacks used to test RCU barriers. */
|
||||
static int rcu_torture_barrier_cbs(void *arg)
|
||||
{
|
||||
long myid = (long)arg;
|
||||
struct rcu_head rcu;
|
||||
|
||||
init_rcu_head_on_stack(&rcu);
|
||||
VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started");
|
||||
set_user_nice(current, 19);
|
||||
do {
|
||||
wait_event(barrier_cbs_wq[myid],
|
||||
atomic_read(&barrier_cbs_count) == n_barrier_cbs ||
|
||||
kthread_should_stop() ||
|
||||
fullstop != FULLSTOP_DONTSTOP);
|
||||
if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
|
||||
break;
|
||||
cur_ops->call(&rcu, rcu_torture_barrier_cbf);
|
||||
if (atomic_dec_and_test(&barrier_cbs_count))
|
||||
wake_up(&barrier_wq);
|
||||
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
||||
VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping");
|
||||
rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
|
||||
while (!kthread_should_stop())
|
||||
schedule_timeout_interruptible(1);
|
||||
cur_ops->cb_barrier();
|
||||
destroy_rcu_head_on_stack(&rcu);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* kthread function to drive and coordinate RCU barrier testing. */
|
||||
static int rcu_torture_barrier(void *arg)
|
||||
{
|
||||
int i;
|
||||
|
||||
VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting");
|
||||
do {
|
||||
atomic_set(&barrier_cbs_invoked, 0);
|
||||
atomic_set(&barrier_cbs_count, n_barrier_cbs);
|
||||
/* wake_up() path contains the required barriers. */
|
||||
for (i = 0; i < n_barrier_cbs; i++)
|
||||
wake_up(&barrier_cbs_wq[i]);
|
||||
wait_event(barrier_wq,
|
||||
atomic_read(&barrier_cbs_count) == 0 ||
|
||||
kthread_should_stop() ||
|
||||
fullstop != FULLSTOP_DONTSTOP);
|
||||
if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
|
||||
break;
|
||||
n_barrier_attempts++;
|
||||
cur_ops->cb_barrier();
|
||||
if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
|
||||
n_rcu_torture_barrier_error++;
|
||||
WARN_ON_ONCE(1);
|
||||
}
|
||||
n_barrier_successes++;
|
||||
schedule_timeout_interruptible(HZ / 10);
|
||||
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
||||
VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping");
|
||||
rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
|
||||
while (!kthread_should_stop())
|
||||
schedule_timeout_interruptible(1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Initialize RCU barrier testing. */
|
||||
static int rcu_torture_barrier_init(void)
|
||||
{
|
||||
int i;
|
||||
int ret;
|
||||
|
||||
if (n_barrier_cbs == 0)
|
||||
return 0;
|
||||
if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
|
||||
printk(KERN_ALERT "%s" TORTURE_FLAG
|
||||
" Call or barrier ops missing for %s,\n",
|
||||
torture_type, cur_ops->name);
|
||||
printk(KERN_ALERT "%s" TORTURE_FLAG
|
||||
" RCU barrier testing omitted from run.\n",
|
||||
torture_type);
|
||||
return 0;
|
||||
}
|
||||
atomic_set(&barrier_cbs_count, 0);
|
||||
atomic_set(&barrier_cbs_invoked, 0);
|
||||
barrier_cbs_tasks =
|
||||
kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
|
||||
GFP_KERNEL);
|
||||
barrier_cbs_wq =
|
||||
kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
|
||||
GFP_KERNEL);
|
||||
if (barrier_cbs_tasks == NULL || barrier_cbs_wq == 0)
|
||||
return -ENOMEM;
|
||||
for (i = 0; i < n_barrier_cbs; i++) {
|
||||
init_waitqueue_head(&barrier_cbs_wq[i]);
|
||||
barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs,
|
||||
(void *)i,
|
||||
"rcu_torture_barrier_cbs");
|
||||
if (IS_ERR(barrier_cbs_tasks[i])) {
|
||||
ret = PTR_ERR(barrier_cbs_tasks[i]);
|
||||
VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs");
|
||||
barrier_cbs_tasks[i] = NULL;
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
barrier_task = kthread_run(rcu_torture_barrier, NULL,
|
||||
"rcu_torture_barrier");
|
||||
if (IS_ERR(barrier_task)) {
|
||||
ret = PTR_ERR(barrier_task);
|
||||
VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier");
|
||||
barrier_task = NULL;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Clean up after RCU barrier testing. */
|
||||
static void rcu_torture_barrier_cleanup(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (barrier_task != NULL) {
|
||||
VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task");
|
||||
kthread_stop(barrier_task);
|
||||
barrier_task = NULL;
|
||||
}
|
||||
if (barrier_cbs_tasks != NULL) {
|
||||
for (i = 0; i < n_barrier_cbs; i++) {
|
||||
if (barrier_cbs_tasks[i] != NULL) {
|
||||
VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task");
|
||||
kthread_stop(barrier_cbs_tasks[i]);
|
||||
barrier_cbs_tasks[i] = NULL;
|
||||
}
|
||||
}
|
||||
kfree(barrier_cbs_tasks);
|
||||
barrier_cbs_tasks = NULL;
|
||||
}
|
||||
if (barrier_cbs_wq != NULL) {
|
||||
kfree(barrier_cbs_wq);
|
||||
barrier_cbs_wq = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static int rcutorture_cpu_notify(struct notifier_block *self,
|
||||
unsigned long action, void *hcpu)
|
||||
{
|
||||
|
|
@ -1602,6 +1773,7 @@ rcu_torture_cleanup(void)
|
|||
fullstop = FULLSTOP_RMMOD;
|
||||
mutex_unlock(&fullstop_mutex);
|
||||
unregister_reboot_notifier(&rcutorture_shutdown_nb);
|
||||
rcu_torture_barrier_cleanup();
|
||||
rcu_torture_stall_cleanup();
|
||||
if (stutter_task) {
|
||||
VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
|
||||
|
|
@ -1681,7 +1853,7 @@ rcu_torture_cleanup(void)
|
|||
|
||||
if (cur_ops->cleanup)
|
||||
cur_ops->cleanup();
|
||||
if (atomic_read(&n_rcu_torture_error))
|
||||
if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
|
||||
rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
|
||||
else if (n_online_successes != n_online_attempts ||
|
||||
n_offline_successes != n_offline_attempts)
|
||||
|
|
@ -1697,6 +1869,7 @@ rcu_torture_init(void)
|
|||
int i;
|
||||
int cpu;
|
||||
int firsterr = 0;
|
||||
int retval;
|
||||
static struct rcu_torture_ops *torture_ops[] =
|
||||
{ &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
|
||||
&rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
|
||||
|
|
@ -1754,6 +1927,7 @@ rcu_torture_init(void)
|
|||
atomic_set(&n_rcu_torture_free, 0);
|
||||
atomic_set(&n_rcu_torture_mberror, 0);
|
||||
atomic_set(&n_rcu_torture_error, 0);
|
||||
n_rcu_torture_barrier_error = 0;
|
||||
n_rcu_torture_boost_ktrerror = 0;
|
||||
n_rcu_torture_boost_rterror = 0;
|
||||
n_rcu_torture_boost_failure = 0;
|
||||
|
|
@ -1877,7 +2051,6 @@ rcu_torture_init(void)
|
|||
test_boost_duration = 2;
|
||||
if ((test_boost == 1 && cur_ops->can_boost) ||
|
||||
test_boost == 2) {
|
||||
int retval;
|
||||
|
||||
boost_starttime = jiffies + test_boost_interval * HZ;
|
||||
register_cpu_notifier(&rcutorture_cpu_nb);
|
||||
|
|
@ -1913,6 +2086,11 @@ rcu_torture_init(void)
|
|||
firsterr = i;
|
||||
goto unwind;
|
||||
}
|
||||
retval = rcu_torture_barrier_init();
|
||||
if (retval != 0) {
|
||||
firsterr = retval;
|
||||
goto unwind;
|
||||
}
|
||||
rcutorture_record_test_transition();
|
||||
mutex_unlock(&fullstop_mutex);
|
||||
return 0;
|
||||
|
|
|
|||
Loading…
Reference in a new issue