1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Read-Copy Update module-based torture test facility
4	*
5	* Copyright (C) IBM Corporation, 2005, 2006
6	*
7	* Authors: Paul E. McKenney <paulmck@linux.ibm.com>
8	* Josh Triplett <josh@joshtriplett.org>
9	*
10	* See also: Documentation/RCU/torture.rst
11	*/
12
13	#define pr_fmt(fmt) fmt
14
15	#include <linux/types.h>
16	#include <linux/kernel.h>
17	#include <linux/init.h>
18	#include <linux/module.h>
19	#include <linux/kthread.h>
20	#include <linux/err.h>
21	#include <linux/spinlock.h>
22	#include <linux/smp.h>
23	#include <linux/rcupdate_wait.h>
24	#include <linux/rcu_notifier.h>
25	#include <linux/interrupt.h>
26	#include <linux/sched/signal.h>
27	#include <uapi/linux/sched/types.h>
28	#include <linux/atomic.h>
29	#include <linux/bitops.h>
30	#include <linux/completion.h>
31	#include <linux/moduleparam.h>
32	#include <linux/percpu.h>
33	#include <linux/notifier.h>
34	#include <linux/reboot.h>
35	#include <linux/freezer.h>
36	#include <linux/cpu.h>
37	#include <linux/delay.h>
38	#include <linux/stat.h>
39	#include <linux/srcu.h>
40	#include <linux/slab.h>
41	#include <linux/trace_clock.h>
42	#include <asm/byteorder.h>
43	#include <linux/torture.h>
44	#include <linux/vmalloc.h>
45	#include <linux/sched/debug.h>
46	#include <linux/sched/sysctl.h>
47	#include <linux/oom.h>
48	#include <linux/tick.h>
49	#include <linux/rcupdate_trace.h>
50	#include <linux/nmi.h>
51
52	#include "rcu.h"
53
54	MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility");
55	MODULE_LICENSE("GPL");
56	MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
57
58	/* Bits for ->extendables field, extendables param, and related definitions. */
59	#define RCUTORTURE_RDR_SHIFT_1 8 /* Put SRCU index in upper bits. */
60	#define RCUTORTURE_RDR_MASK_1 (0xff << RCUTORTURE_RDR_SHIFT_1)
61	#define RCUTORTURE_RDR_SHIFT_2 16 /* Put SRCU index in upper bits. */
62	#define RCUTORTURE_RDR_MASK_2 (0xff << RCUTORTURE_RDR_SHIFT_2)
63	#define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */
64	#define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */
65	#define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */
66	#define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */
67	#define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */
68	#define RCUTORTURE_RDR_RCU_1 0x20 /* ... entering another RCU reader. */
69	#define RCUTORTURE_RDR_RCU_2 0x40 /* ... entering another RCU reader. */
70	#define RCUTORTURE_RDR_NBITS 7 /* Number of bits defined above. */
71	#define RCUTORTURE_MAX_EXTEND \
72	(RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \
73	RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED)
74	#define RCUTORTURE_RDR_ALLBITS \
75	(RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \
76	RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2)
77	#define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */
78	/* Must be power of two minus one. */
79	#define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3)
80
81	torture_param(int, extendables, RCUTORTURE_MAX_EXTEND,
82	"Extend readers by disabling bh (1), irqs (2), or preempt (4)");
83	torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable");
84	torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
85	torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
86	torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)");
87	torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait");
88	torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)");
89	torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()");
90	torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
91	torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives");
92	torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives");
93	torture_param(bool, gp_cond_exp_full, false,
94	"Use conditional/async full-stateexpedited GP wait primitives");
95	torture_param(int, gp_cond_wi, 16 * USEC_PER_SEC / HZ,
96	"Wait interval for normal conditional grace periods, us (default 16 jiffies)");
97	torture_param(int, gp_cond_wi_exp, 128,
98	"Wait interval for expedited conditional grace periods, us (default 128 us)");
99	torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
100	torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");
101	torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
102	torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");
103	torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives");
104	torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives");
105	torture_param(int, gp_poll_wi, 16 * USEC_PER_SEC / HZ,
106	"Wait interval for normal polled grace periods, us (default 16 jiffies)");
107	torture_param(int, gp_poll_wi_exp, 128,
108	"Wait interval for expedited polled grace periods, us (default 128 us)");
109	torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
110	torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
111	torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
112	torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing");
113	torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
114	torture_param(int, nreaders, -1, "Number of RCU reader threads");
115	torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing");
116	torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
117	torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable");
118	torture_param(bool, gpwrap_lag, true, "Enable grace-period wrap lag testing");
119	torture_param(int, gpwrap_lag_gps, 8, "Value to set for set_gpwrap_lag during an active testing period.");
120	torture_param(int, gpwrap_lag_cycle_mins, 30, "Total cycle duration for gpwrap lag testing (in minutes)");
121	torture_param(int, gpwrap_lag_active_mins, 5, "Duration for which gpwrap lag is active within each cycle (in minutes)");
122	torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable");
123	torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)");
124	torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable");
125	torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)");
126	torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)");
127	torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable");
128	torture_param(int, reader_flavor, SRCU_READ_FLAVOR_NORMAL, "Reader flavors to use, one per bit.");
129	torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
130	torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
131	torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
132	torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s).");
133	torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall.");
134	torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
135	torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");
136	torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one.");
137	torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s).");
138	torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
139	torture_param(int, stutter, 5, "Number of seconds to run/halt test");
140	torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
141	torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds.");
142	torture_param(int, test_boost_holdoff, 0, "Holdoff time from rcutorture start, seconds.");
143	torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds.");
144	torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable.");
145	torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs");
146	torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario.");
147	torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
148
149	static char *torture_type = "rcu";
150	module_param(torture_type, charp, 0444);
151	MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)");
152
153	static int nrealnocbers;
154	static int nrealreaders;
155	static int nrealfakewriters;
156	static struct task_struct *writer_task;
157	static struct task_struct **fakewriter_tasks;
158	static struct task_struct **reader_tasks;
159	static struct task_struct **nocb_tasks;
160	static struct task_struct *stats_task;
161	static struct task_struct *fqs_task;
162	static struct task_struct *boost_tasks[NR_CPUS];
163	static struct task_struct *stall_task;
164	static struct task_struct **fwd_prog_tasks;
165	static struct task_struct **barrier_cbs_tasks;
166	static struct task_struct *barrier_task;
167	static struct task_struct *read_exit_task;
168	static struct task_struct *preempt_task;
169
170	#define RCU_TORTURE_PIPE_LEN 10
171
172	// Mailbox-like structure to check RCU global memory ordering.
173	struct rcu_torture_reader_check {
174	unsigned long rtc_myloops;
175	int rtc_chkrdr;
176	unsigned long rtc_chkloops;
177	int rtc_ready;
178	struct rcu_torture_reader_check *rtc_assigner;
179	} ____cacheline_internodealigned_in_smp;
180
181	// Update-side data structure used to check RCU readers.
182	struct rcu_torture {
183	struct rcu_head rtort_rcu;
184	int rtort_pipe_count;
185	struct list_head rtort_free;
186	int rtort_mbtest;
187	struct rcu_torture_reader_check *rtort_chkp;
188	};
189
190	static LIST_HEAD(rcu_torture_freelist);
191	static struct rcu_torture __rcu *rcu_torture_current;
192	static unsigned long rcu_torture_current_version;
193	static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
194	static DEFINE_SPINLOCK(rcu_torture_lock);
195	static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count);
196	static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch);
197	static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
198	static struct rcu_torture_reader_check *rcu_torture_reader_mbchk;
199	static atomic_t n_rcu_torture_alloc;
200	static atomic_t n_rcu_torture_alloc_fail;
201	static atomic_t n_rcu_torture_free;
202	static atomic_t n_rcu_torture_mberror;
203	static atomic_t n_rcu_torture_mbchk_fail;
204	static atomic_t n_rcu_torture_mbchk_tries;
205	static atomic_t n_rcu_torture_error;
206	static long n_rcu_torture_barrier_error;
207	static long n_rcu_torture_boost_ktrerror;
208	static long n_rcu_torture_boost_failure;
209	static long n_rcu_torture_boosts;
210	static atomic_long_t n_rcu_torture_timers;
211	static long n_barrier_attempts;
212	static long n_barrier_successes; /* did rcu_barrier test succeed? */
213	static unsigned long n_read_exits;
214	static struct list_head rcu_torture_removed;
215	static unsigned long shutdown_jiffies;
216	static unsigned long start_gp_seq;
217	static atomic_long_t n_nocb_offload;
218	static atomic_long_t n_nocb_deoffload;
219
220	static int rcu_torture_writer_state;
221	#define RTWS_FIXED_DELAY 0
222	#define RTWS_DELAY 1
223	#define RTWS_REPLACE 2
224	#define RTWS_DEF_FREE 3
225	#define RTWS_EXP_SYNC 4
226	#define RTWS_COND_GET 5
227	#define RTWS_COND_GET_FULL 6
228	#define RTWS_COND_GET_EXP 7
229	#define RTWS_COND_GET_EXP_FULL 8
230	#define RTWS_COND_SYNC 9
231	#define RTWS_COND_SYNC_FULL 10
232	#define RTWS_COND_SYNC_EXP 11
233	#define RTWS_COND_SYNC_EXP_FULL 12
234	#define RTWS_POLL_GET 13
235	#define RTWS_POLL_GET_FULL 14
236	#define RTWS_POLL_GET_EXP 15
237	#define RTWS_POLL_GET_EXP_FULL 16
238	#define RTWS_POLL_WAIT 17
239	#define RTWS_POLL_WAIT_FULL 18
240	#define RTWS_POLL_WAIT_EXP 19
241	#define RTWS_POLL_WAIT_EXP_FULL 20
242	#define RTWS_SYNC 21
243	#define RTWS_STUTTER 22
244	#define RTWS_STOPPING 23
245	static const char * const rcu_torture_writer_state_names[] = {
246	"RTWS_FIXED_DELAY",
247	"RTWS_DELAY",
248	"RTWS_REPLACE",
249	"RTWS_DEF_FREE",
250	"RTWS_EXP_SYNC",
251	"RTWS_COND_GET",
252	"RTWS_COND_GET_FULL",
253	"RTWS_COND_GET_EXP",
254	"RTWS_COND_GET_EXP_FULL",
255	"RTWS_COND_SYNC",
256	"RTWS_COND_SYNC_FULL",
257	"RTWS_COND_SYNC_EXP",
258	"RTWS_COND_SYNC_EXP_FULL",
259	"RTWS_POLL_GET",
260	"RTWS_POLL_GET_FULL",
261	"RTWS_POLL_GET_EXP",
262	"RTWS_POLL_GET_EXP_FULL",
263	"RTWS_POLL_WAIT",
264	"RTWS_POLL_WAIT_FULL",
265	"RTWS_POLL_WAIT_EXP",
266	"RTWS_POLL_WAIT_EXP_FULL",
267	"RTWS_SYNC",
268	"RTWS_STUTTER",
269	"RTWS_STOPPING",
270	};
271
272	/* Record reader segment types and duration for first failing read. */
273	struct rt_read_seg {
274	int rt_readstate;
275	unsigned long rt_delay_jiffies;
276	unsigned long rt_delay_ms;
277	unsigned long rt_delay_us;
278	bool rt_preempted;
279	int rt_cpu;
280	int rt_end_cpu;
281	unsigned long long rt_gp_seq;
282	unsigned long long rt_gp_seq_end;
283	u64 rt_ts;
284	};
285	static int err_segs_recorded;
286	static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS];
287	static int rt_read_nsegs;
288	static int rt_read_preempted;
289
290	static const char *rcu_torture_writer_state_getname(void)
291	{
292	unsigned int i = READ_ONCE(rcu_torture_writer_state);
293
294	if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
295	return "???";
296	return rcu_torture_writer_state_names[i];
297	}
298
299	#ifdef CONFIG_RCU_TRACE
300	static u64 notrace rcu_trace_clock_local(void)
301	{
302	u64 ts = trace_clock_local();
303
304	(void)do_div(ts, NSEC_PER_USEC);
305	return ts;
306	}
307	#else /* #ifdef CONFIG_RCU_TRACE */
308	static u64 notrace rcu_trace_clock_local(void)
309	{
310	return 0ULL;
311	}
312	#endif /* #else #ifdef CONFIG_RCU_TRACE */
313
314	/*
315	* Stop aggressive CPU-hog tests a bit before the end of the test in order
316	* to avoid interfering with test shutdown.
317	*/
318	static bool shutdown_time_arrived(void)
319	{
320	return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ);
321	}
322
323	static unsigned long boost_starttime; /* jiffies of next boost test start. */
324	static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */
325	/* and boost task create/destroy. */
326	static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */
327	static bool barrier_phase; /* Test phase. */
328	static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */
329	static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
330	static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
331
332	static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */
333
334	/*
335	* Allocate an element from the rcu_tortures pool.
336	*/
337	static struct rcu_torture *
338	rcu_torture_alloc(void)
339	{
340	struct list_head *p;
341
342	spin_lock_bh(lock: &rcu_torture_lock);
343	if (list_empty(head: &rcu_torture_freelist)) {
344	atomic_inc(v: &n_rcu_torture_alloc_fail);
345	spin_unlock_bh(lock: &rcu_torture_lock);
346	return NULL;
347	}
348	atomic_inc(v: &n_rcu_torture_alloc);
349	p = rcu_torture_freelist.next;
350	list_del_init(entry: p);
351	spin_unlock_bh(lock: &rcu_torture_lock);
352	return container_of(p, struct rcu_torture, rtort_free);
353	}
354
355	/*
356	* Free an element to the rcu_tortures pool.
357	*/
358	static void
359	rcu_torture_free(struct rcu_torture *p)
360	{
361	atomic_inc(v: &n_rcu_torture_free);
362	spin_lock_bh(lock: &rcu_torture_lock);
363	list_add_tail(new: &p->rtort_free, head: &rcu_torture_freelist);
364	spin_unlock_bh(lock: &rcu_torture_lock);
365	}
366
367	/*
368	* Operations vector for selecting different types of tests.
369	*/
370
371	struct rcu_torture_ops {
372	int ttype;
373	void (*init)(void);
374	void (*cleanup)(void);
375	int (*readlock)(void);
376	void (*read_delay)(struct torture_random_state *rrsp,
377	struct rt_read_seg *rtrsp);
378	void (*readunlock)(int idx);
379	int (*readlock_held)(void); // lockdep.
380	int (*readlock_nesting)(void); // actual nesting, if available, -1 if not.
381	unsigned long (*get_gp_seq)(void);
382	unsigned long (*gp_diff)(unsigned long new, unsigned long old);
383	void (*deferred_free)(struct rcu_torture *p);
384	void (*sync)(void);
385	void (*exp_sync)(void);
386	unsigned long (*get_gp_state_exp)(void);
387	unsigned long (*start_gp_poll_exp)(void);
388	void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp);
389	bool (*poll_gp_state_exp)(unsigned long oldstate);
390	void (*cond_sync_exp)(unsigned long oldstate);
391	void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp);
392	unsigned long (*get_comp_state)(void);
393	void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp);
394	bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2);
395	bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2);
396	unsigned long (*get_gp_state)(void);
397	void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);
398	unsigned long (*start_gp_poll)(void);
399	void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp);
400	bool (*poll_gp_state)(unsigned long oldstate);
401	bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);
402	bool (*poll_need_2gp)(bool poll, bool poll_full);
403	void (*cond_sync)(unsigned long oldstate);
404	void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp);
405	int poll_active;
406	int poll_active_full;
407	call_rcu_func_t call;
408	void (*cb_barrier)(void);
409	void (*fqs)(void);
410	void (*stats)(void);
411	void (*gp_kthread_dbg)(void);
412	bool (*check_boost_failed)(unsigned long gp_state, int *cpup);
413	int (*stall_dur)(void);
414	void (*get_gp_data)(int *flags, unsigned long *gp_seq);
415	void (*gp_slow_register)(atomic_t *rgssp);
416	void (*gp_slow_unregister)(atomic_t *rgssp);
417	bool (*reader_blocked)(void);
418	unsigned long long (*gather_gp_seqs)(void);
419	void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len);
420	void (*set_gpwrap_lag)(unsigned long lag);
421	int (*get_gpwrap_count)(int cpu);
422	long cbflood_max;
423	int irq_capable;
424	int can_boost;
425	int extendables;
426	int slow_gps;
427	int no_pi_lock;
428	int debug_objects;
429	int start_poll_irqsoff;
430	const char *name;
431	};
432
433	static struct rcu_torture_ops *cur_ops;
434
435	/*
436	* Definitions for rcu torture testing.
437	*/
438
439	static int torture_readlock_not_held(void)
440	{
441	return rcu_read_lock_bh_held() || rcu_read_lock_sched_held();
442	}
443
444	static int rcu_torture_read_lock(void)
445	{
446	rcu_read_lock();
447	return 0;
448	}
449
450	static void
451	rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
452	{
453	unsigned long started;
454	unsigned long completed;
455	const unsigned long shortdelay_us = 200;
456	unsigned long longdelay_ms = 300;
457	unsigned long long ts;
458
459	/* We want a short delay sometimes to make a reader delay the grace
460	* period, and we want a long delay occasionally to trigger
461	* force_quiescent_state. */
462
463	if (!atomic_read(v: &rcu_fwd_cb_nodelay) &&
464	!(torture_random(trsp: rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
465	started = cur_ops->get_gp_seq();
466	ts = rcu_trace_clock_local();
467	if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK))
468	longdelay_ms = 5; /* Avoid triggering BH limits. */
469	mdelay(longdelay_ms);
470	rtrsp->rt_delay_ms = longdelay_ms;
471	completed = cur_ops->get_gp_seq();
472	do_trace_rcu_torture_read(rcutorturename: cur_ops->name, NULL, secs: ts,
473	c_old: started, c: completed);
474	}
475	if (!(torture_random(trsp: rrsp) % (nrealreaders * 2 * shortdelay_us))) {
476	udelay(usec: shortdelay_us);
477	rtrsp->rt_delay_us = shortdelay_us;
478	}
479	if (!preempt_count() &&
480	!(torture_random(trsp: rrsp) % (nrealreaders * 500)))
481	torture_preempt_schedule(); /* QS only if preemptible. */
482	}
483
484	static void rcu_torture_read_unlock(int idx)
485	{
486	rcu_read_unlock();
487	}
488
489	static int rcu_torture_readlock_nesting(void)
490	{
491	if (IS_ENABLED(CONFIG_PREEMPT_RCU))
492	return rcu_preempt_depth();
493	if (IS_ENABLED(CONFIG_PREEMPT_COUNT))
494	return (preempt_count() & PREEMPT_MASK);
495	return -1;
496	}
497
498	/*
499	* Update callback in the pipe. This should be invoked after a grace period.
500	*/
501	static bool
502	rcu_torture_pipe_update_one(struct rcu_torture *rp)
503	{
504	int i;
505	struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp);
506
507	if (rtrcp) {
508	WRITE_ONCE(rp->rtort_chkp, NULL);
509	smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire().
510	}
511	i = rp->rtort_pipe_count;
512	if (i > RCU_TORTURE_PIPE_LEN)
513	i = RCU_TORTURE_PIPE_LEN;
514	atomic_inc(v: &rcu_torture_wcount[i]);
515	WRITE_ONCE(rp->rtort_pipe_count, i + 1);
516	ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count);
517	if (i + 1 >= RCU_TORTURE_PIPE_LEN) {
518	rp->rtort_mbtest = 0;
519	return true;
520	}
521	return false;
522	}
523
524	/*
525	* Update all callbacks in the pipe. Suitable for synchronous grace-period
526	* primitives.
527	*/
528	static void
529	rcu_torture_pipe_update(struct rcu_torture *old_rp)
530	{
531	struct rcu_torture *rp;
532	struct rcu_torture *rp1;
533
534	if (old_rp)
535	list_add(new: &old_rp->rtort_free, head: &rcu_torture_removed);
536	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
537	if (rcu_torture_pipe_update_one(rp)) {
538	list_del(entry: &rp->rtort_free);
539	rcu_torture_free(p: rp);
540	}
541	}
542	}
543
544	static void
545	rcu_torture_cb(struct rcu_head *p)
546	{
547	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
548
549	if (torture_must_stop_irq()) {
550	/* Test is ending, just drop callbacks on the floor. */
551	/* The next initialization will pick up the pieces. */
552	return;
553	}
554	if (rcu_torture_pipe_update_one(rp))
555	rcu_torture_free(p: rp);
556	else
557	cur_ops->deferred_free(rp);
558	}
559
560	static unsigned long rcu_no_completed(void)
561	{
562	return 0;
563	}
564
565	static void rcu_torture_deferred_free(struct rcu_torture *p)
566	{
567	call_rcu_hurry(head: &p->rtort_rcu, func: rcu_torture_cb);
568	}
569
570	static void rcu_sync_torture_init(void)
571	{
572	INIT_LIST_HEAD(list: &rcu_torture_removed);
573	}
574
575	static bool rcu_poll_need_2gp(bool poll, bool poll_full)
576	{
577	return poll;
578	}
579
580	static struct rcu_torture_ops rcu_ops = {
581	.ttype = RCU_FLAVOR,
582	.init = rcu_sync_torture_init,
583	.readlock = rcu_torture_read_lock,
584	.read_delay = rcu_read_delay,
585	.readunlock = rcu_torture_read_unlock,
586	.readlock_held = torture_readlock_not_held,
587	.readlock_nesting = rcu_torture_readlock_nesting,
588	.get_gp_seq = rcu_get_gp_seq,
589	.gp_diff = rcu_seq_diff,
590	.deferred_free = rcu_torture_deferred_free,
591	.sync = synchronize_rcu,
592	.exp_sync = synchronize_rcu_expedited,
593	.same_gp_state = same_state_synchronize_rcu,
594	.same_gp_state_full = same_state_synchronize_rcu_full,
595	.get_comp_state = get_completed_synchronize_rcu,
596	.get_comp_state_full = get_completed_synchronize_rcu_full,
597	.get_gp_state = get_state_synchronize_rcu,
598	.get_gp_state_full = get_state_synchronize_rcu_full,
599	.start_gp_poll = start_poll_synchronize_rcu,
600	.start_gp_poll_full = start_poll_synchronize_rcu_full,
601	.poll_gp_state = poll_state_synchronize_rcu,
602	.poll_gp_state_full = poll_state_synchronize_rcu_full,
603	.poll_need_2gp = rcu_poll_need_2gp,
604	.cond_sync = cond_synchronize_rcu,
605	.cond_sync_full = cond_synchronize_rcu_full,
606	.poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE,
607	.poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE,
608	.get_gp_state_exp = get_state_synchronize_rcu,
609	.start_gp_poll_exp = start_poll_synchronize_rcu_expedited,
610	.start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full,
611	.poll_gp_state_exp = poll_state_synchronize_rcu,
612	.cond_sync_exp = cond_synchronize_rcu_expedited,
613	.cond_sync_exp_full = cond_synchronize_rcu_expedited_full,
614	.call = call_rcu_hurry,
615	.cb_barrier = rcu_barrier,
616	.fqs = rcu_force_quiescent_state,
617	.gp_kthread_dbg = show_rcu_gp_kthreads,
618	.check_boost_failed = rcu_check_boost_fail,
619	.stall_dur = rcu_jiffies_till_stall_check,
620	.get_gp_data = rcutorture_get_gp_data,
621	.gp_slow_register = rcu_gp_slow_register,
622	.gp_slow_unregister = rcu_gp_slow_unregister,
623	.reader_blocked = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)
624	? has_rcu_reader_blocked
625	: NULL,
626	.gather_gp_seqs = rcutorture_gather_gp_seqs,
627	.format_gp_seqs = rcutorture_format_gp_seqs,
628	.set_gpwrap_lag = rcu_set_gpwrap_lag,
629	.get_gpwrap_count = rcu_get_gpwrap_count,
630	.irq_capable = 1,
631	.can_boost = IS_ENABLED(CONFIG_RCU_BOOST),
632	.extendables = RCUTORTURE_MAX_EXTEND,
633	.debug_objects = 1,
634	.start_poll_irqsoff = 1,
635	.name = "rcu"
636	};
637
638	/*
639	* Don't even think about trying any of these in real life!!!
640	* The names includes "busted", and they really means it!
641	* The only purpose of these functions is to provide a buggy RCU
642	* implementation to make sure that rcutorture correctly emits
643	* buggy-RCU error messages.
644	*/
645	static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
646	{
647	/* This is a deliberate bug for testing purposes only! */
648	rcu_torture_cb(p: &p->rtort_rcu);
649	}
650
651	static void synchronize_rcu_busted(void)
652	{
653	/* This is a deliberate bug for testing purposes only! */
654	}
655
656	static void
657	call_rcu_busted(struct rcu_head *head, rcu_callback_t func)
658	{
659	/* This is a deliberate bug for testing purposes only! */
660	func(head);
661	}
662
663	static struct rcu_torture_ops rcu_busted_ops = {
664	.ttype = INVALID_RCU_FLAVOR,
665	.init = rcu_sync_torture_init,
666	.readlock = rcu_torture_read_lock,
667	.read_delay = rcu_read_delay, /* just reuse rcu's version. */
668	.readunlock = rcu_torture_read_unlock,
669	.readlock_held = torture_readlock_not_held,
670	.get_gp_seq = rcu_no_completed,
671	.deferred_free = rcu_busted_torture_deferred_free,
672	.sync = synchronize_rcu_busted,
673	.exp_sync = synchronize_rcu_busted,
674	.call = call_rcu_busted,
675	.gather_gp_seqs = rcutorture_gather_gp_seqs,
676	.format_gp_seqs = rcutorture_format_gp_seqs,
677	.irq_capable = 1,
678	.extendables = RCUTORTURE_MAX_EXTEND,
679	.name = "busted"
680	};
681
682	/*
683	* Definitions for srcu torture testing.
684	*/
685
686	DEFINE_STATIC_SRCU(srcu_ctl);
687	static struct srcu_struct srcu_ctld;
688	static struct srcu_struct *srcu_ctlp = &srcu_ctl;
689	static struct rcu_torture_ops srcud_ops;
690
691	static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)
692	{
693	srcutorture_get_gp_data(sp: srcu_ctlp, flags, gp_seq);
694	}
695
696	static int srcu_torture_read_lock(void)
697	{
698	int idx;
699	struct srcu_ctr __percpu *scp;
700	int ret = 0;
701
702	WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL);
703
704	if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) {
705	idx = srcu_read_lock(ssp: srcu_ctlp);
706	WARN_ON_ONCE(idx & ~0x1);
707	ret += idx;
708	}
709	if (reader_flavor & SRCU_READ_FLAVOR_NMI) {
710	idx = srcu_read_lock_nmisafe(ssp: srcu_ctlp);
711	WARN_ON_ONCE(idx & ~0x1);
712	ret += idx << 1;
713	}
714	if (reader_flavor & SRCU_READ_FLAVOR_LITE) {
715	idx = srcu_read_lock_lite(ssp: srcu_ctlp);
716	WARN_ON_ONCE(idx & ~0x1);
717	ret += idx << 2;
718	}
719	if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
720	scp = srcu_read_lock_fast(ssp: srcu_ctlp);
721	idx = __srcu_ptr_to_ctr(ssp: srcu_ctlp, scpp: scp);
722	WARN_ON_ONCE(idx & ~0x1);
723	ret += idx << 3;
724	}
725	return ret;
726	}
727
728	static void
729	srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
730	{
731	long delay;
732	const long uspertick = 1000000 / HZ;
733	const long longdelay = 10;
734
735	/* We want there to be long-running readers, but not all the time. */
736
737	delay = torture_random(trsp: rrsp) %
738	(nrealreaders * 2 * longdelay * uspertick);
739	if (!delay && in_task()) {
740	schedule_timeout_interruptible(timeout: longdelay);
741	rtrsp->rt_delay_jiffies = longdelay;
742	} else {
743	rcu_read_delay(rrsp, rtrsp);
744	}
745	}
746
747	static void srcu_torture_read_unlock(int idx)
748	{
749	WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
750	if (reader_flavor & SRCU_READ_FLAVOR_FAST)
751	srcu_read_unlock_fast(ssp: srcu_ctlp, scp: __srcu_ctr_to_ptr(ssp: srcu_ctlp, idx: (idx & 0x8) >> 3));
752	if (reader_flavor & SRCU_READ_FLAVOR_LITE)
753	srcu_read_unlock_lite(ssp: srcu_ctlp, idx: (idx & 0x4) >> 2);
754	if (reader_flavor & SRCU_READ_FLAVOR_NMI)
755	srcu_read_unlock_nmisafe(ssp: srcu_ctlp, idx: (idx & 0x2) >> 1);
756	if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL))
757	srcu_read_unlock(ssp: srcu_ctlp, idx: idx & 0x1);
758	}
759
760	static int torture_srcu_read_lock_held(void)
761	{
762	return srcu_read_lock_held(ssp: srcu_ctlp);
763	}
764
765	static unsigned long srcu_torture_completed(void)
766	{
767	return srcu_batches_completed(sp: srcu_ctlp);
768	}
769
770	static void srcu_torture_deferred_free(struct rcu_torture *rp)
771	{
772	call_srcu(ssp: srcu_ctlp, head: &rp->rtort_rcu, func: rcu_torture_cb);
773	}
774
775	static void srcu_torture_synchronize(void)
776	{
777	synchronize_srcu(ssp: srcu_ctlp);
778	}
779
780	static unsigned long srcu_torture_get_gp_state(void)
781	{
782	return get_state_synchronize_srcu(ssp: srcu_ctlp);
783	}
784
785	static unsigned long srcu_torture_start_gp_poll(void)
786	{
787	return start_poll_synchronize_srcu(ssp: srcu_ctlp);
788	}
789
790	static bool srcu_torture_poll_gp_state(unsigned long oldstate)
791	{
792	return poll_state_synchronize_srcu(ssp: srcu_ctlp, cookie: oldstate);
793	}
794
795	static void srcu_torture_call(struct rcu_head *head,
796	rcu_callback_t func)
797	{
798	call_srcu(ssp: srcu_ctlp, head, func);
799	}
800
801	static void srcu_torture_barrier(void)
802	{
803	srcu_barrier(ssp: srcu_ctlp);
804	}
805
806	static void srcu_torture_stats(void)
807	{
808	srcu_torture_stats_print(ssp: srcu_ctlp, tt: torture_type, TORTURE_FLAG);
809	}
810
811	static void srcu_torture_synchronize_expedited(void)
812	{
813	synchronize_srcu_expedited(ssp: srcu_ctlp);
814	}
815
816	static struct rcu_torture_ops srcu_ops = {
817	.ttype = SRCU_FLAVOR,
818	.init = rcu_sync_torture_init,
819	.readlock = srcu_torture_read_lock,
820	.read_delay = srcu_read_delay,
821	.readunlock = srcu_torture_read_unlock,
822	.readlock_held = torture_srcu_read_lock_held,
823	.get_gp_seq = srcu_torture_completed,
824	.gp_diff = rcu_seq_diff,
825	.deferred_free = srcu_torture_deferred_free,
826	.sync = srcu_torture_synchronize,
827	.exp_sync = srcu_torture_synchronize_expedited,
828	.same_gp_state = same_state_synchronize_srcu,
829	.get_comp_state = get_completed_synchronize_srcu,
830	.get_gp_state = srcu_torture_get_gp_state,
831	.start_gp_poll = srcu_torture_start_gp_poll,
832	.poll_gp_state = srcu_torture_poll_gp_state,
833	.poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE,
834	.call = srcu_torture_call,
835	.cb_barrier = srcu_torture_barrier,
836	.stats = srcu_torture_stats,
837	.get_gp_data = srcu_get_gp_data,
838	.cbflood_max = 50000,
839	.irq_capable = 1,
840	.no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
841	.debug_objects = 1,
842	.name = "srcu"
843	};
844
845	static void srcu_torture_init(void)
846	{
847	rcu_sync_torture_init();
848	WARN_ON(init_srcu_struct(&srcu_ctld));
849	srcu_ctlp = &srcu_ctld;
850	}
851
852	static void srcu_torture_cleanup(void)
853	{
854	cleanup_srcu_struct(ssp: &srcu_ctld);
855	srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */
856	}
857
858	/* As above, but dynamically allocated. */
859	static struct rcu_torture_ops srcud_ops = {
860	.ttype = SRCU_FLAVOR,
861	.init = srcu_torture_init,
862	.cleanup = srcu_torture_cleanup,
863	.readlock = srcu_torture_read_lock,
864	.read_delay = srcu_read_delay,
865	.readunlock = srcu_torture_read_unlock,
866	.readlock_held = torture_srcu_read_lock_held,
867	.get_gp_seq = srcu_torture_completed,
868	.gp_diff = rcu_seq_diff,
869	.deferred_free = srcu_torture_deferred_free,
870	.sync = srcu_torture_synchronize,
871	.exp_sync = srcu_torture_synchronize_expedited,
872	.same_gp_state = same_state_synchronize_srcu,
873	.get_comp_state = get_completed_synchronize_srcu,
874	.get_gp_state = srcu_torture_get_gp_state,
875	.start_gp_poll = srcu_torture_start_gp_poll,
876	.poll_gp_state = srcu_torture_poll_gp_state,
877	.poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE,
878	.call = srcu_torture_call,
879	.cb_barrier = srcu_torture_barrier,
880	.stats = srcu_torture_stats,
881	.get_gp_data = srcu_get_gp_data,
882	.cbflood_max = 50000,
883	.irq_capable = 1,
884	.no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
885	.debug_objects = 1,
886	.name = "srcud"
887	};
888
889	/* As above, but broken due to inappropriate reader extension. */
890	static struct rcu_torture_ops busted_srcud_ops = {
891	.ttype = SRCU_FLAVOR,
892	.init = srcu_torture_init,
893	.cleanup = srcu_torture_cleanup,
894	.readlock = srcu_torture_read_lock,
895	.read_delay = rcu_read_delay,
896	.readunlock = srcu_torture_read_unlock,
897	.readlock_held = torture_srcu_read_lock_held,
898	.get_gp_seq = srcu_torture_completed,
899	.deferred_free = srcu_torture_deferred_free,
900	.sync = srcu_torture_synchronize,
901	.exp_sync = srcu_torture_synchronize_expedited,
902	.call = srcu_torture_call,
903	.cb_barrier = srcu_torture_barrier,
904	.stats = srcu_torture_stats,
905	.irq_capable = 1,
906	.no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU),
907	.extendables = RCUTORTURE_MAX_EXTEND,
908	.name = "busted_srcud"
909	};
910
911	/*
912	* Definitions for trivial CONFIG_PREEMPT=n-only torture testing.
913	* This implementation does not necessarily work well with CPU hotplug.
914	*/
915
916	static void synchronize_rcu_trivial(void)
917	{
918	int cpu;
919
920	for_each_online_cpu(cpu) {
921	torture_sched_setaffinity(current->pid, cpumask_of(cpu), dowarn: true);
922	WARN_ON_ONCE(raw_smp_processor_id() != cpu);
923	}
924	}
925
926	static int rcu_torture_read_lock_trivial(void)
927	{
928	preempt_disable();
929	return 0;
930	}
931
932	static void rcu_torture_read_unlock_trivial(int idx)
933	{
934	preempt_enable();
935	}
936
937	static struct rcu_torture_ops trivial_ops = {
938	.ttype = RCU_TRIVIAL_FLAVOR,
939	.init = rcu_sync_torture_init,
940	.readlock = rcu_torture_read_lock_trivial,
941	.read_delay = rcu_read_delay, /* just reuse rcu's version. */
942	.readunlock = rcu_torture_read_unlock_trivial,
943	.readlock_held = torture_readlock_not_held,
944	.get_gp_seq = rcu_no_completed,
945	.sync = synchronize_rcu_trivial,
946	.exp_sync = synchronize_rcu_trivial,
947	.irq_capable = 1,
948	.name = "trivial"
949	};
950
951	#ifdef CONFIG_TASKS_RCU
952
953	/*
954	* Definitions for RCU-tasks torture testing.
955	*/
956
957	static int tasks_torture_read_lock(void)
958	{
959	return 0;
960	}
961
962	static void tasks_torture_read_unlock(int idx)
963	{
964	}
965
966	static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
967	{
968	call_rcu_tasks(head: &p->rtort_rcu, func: rcu_torture_cb);
969	}
970
971	static void synchronize_rcu_mult_test(void)
972	{
973	synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry);
974	}
975
976	static struct rcu_torture_ops tasks_ops = {
977	.ttype = RCU_TASKS_FLAVOR,
978	.init = rcu_sync_torture_init,
979	.readlock = tasks_torture_read_lock,
980	.read_delay = rcu_read_delay, /* just reuse rcu's version. */
981	.readunlock = tasks_torture_read_unlock,
982	.get_gp_seq = rcu_no_completed,
983	.deferred_free = rcu_tasks_torture_deferred_free,
984	.sync = synchronize_rcu_tasks,
985	.exp_sync = synchronize_rcu_mult_test,
986	.call = call_rcu_tasks,
987	.cb_barrier = rcu_barrier_tasks,
988	.gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread,
989	.get_gp_data = rcu_tasks_get_gp_data,
990	.irq_capable = 1,
991	.slow_gps = 1,
992	.name = "tasks"
993	};
994
995	#define TASKS_OPS &tasks_ops,
996
997	#else // #ifdef CONFIG_TASKS_RCU
998
999	#define TASKS_OPS
1000
1001	#endif // #else #ifdef CONFIG_TASKS_RCU
1002
1003
1004	#ifdef CONFIG_TASKS_RUDE_RCU
1005
1006	/*
1007	* Definitions for rude RCU-tasks torture testing.
1008	*/
1009
1010	static struct rcu_torture_ops tasks_rude_ops = {
1011	.ttype = RCU_TASKS_RUDE_FLAVOR,
1012	.init = rcu_sync_torture_init,
1013	.readlock = rcu_torture_read_lock_trivial,
1014	.read_delay = rcu_read_delay, /* just reuse rcu's version. */
1015	.readunlock = rcu_torture_read_unlock_trivial,
1016	.get_gp_seq = rcu_no_completed,
1017	.sync = synchronize_rcu_tasks_rude,
1018	.exp_sync = synchronize_rcu_tasks_rude,
1019	.gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread,
1020	.get_gp_data = rcu_tasks_rude_get_gp_data,
1021	.cbflood_max = 50000,
1022	.irq_capable = 1,
1023	.name = "tasks-rude"
1024	};
1025
1026	#define TASKS_RUDE_OPS &tasks_rude_ops,
1027
1028	#else // #ifdef CONFIG_TASKS_RUDE_RCU
1029
1030	#define TASKS_RUDE_OPS
1031
1032	#endif // #else #ifdef CONFIG_TASKS_RUDE_RCU
1033
1034
1035	#ifdef CONFIG_TASKS_TRACE_RCU
1036
1037	/*
1038	* Definitions for tracing RCU-tasks torture testing.
1039	*/
1040
1041	static int tasks_tracing_torture_read_lock(void)
1042	{
1043	rcu_read_lock_trace();
1044	return 0;
1045	}
1046
1047	static void tasks_tracing_torture_read_unlock(int idx)
1048	{
1049	rcu_read_unlock_trace();
1050	}
1051
1052	static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p)
1053	{
1054	call_rcu_tasks_trace(rhp: &p->rtort_rcu, func: rcu_torture_cb);
1055	}
1056
1057	static struct rcu_torture_ops tasks_tracing_ops = {
1058	.ttype = RCU_TASKS_TRACING_FLAVOR,
1059	.init = rcu_sync_torture_init,
1060	.readlock = tasks_tracing_torture_read_lock,
1061	.read_delay = srcu_read_delay, /* just reuse srcu's version. */
1062	.readunlock = tasks_tracing_torture_read_unlock,
1063	.readlock_held = rcu_read_lock_trace_held,
1064	.get_gp_seq = rcu_no_completed,
1065	.deferred_free = rcu_tasks_tracing_torture_deferred_free,
1066	.sync = synchronize_rcu_tasks_trace,
1067	.exp_sync = synchronize_rcu_tasks_trace,
1068	.call = call_rcu_tasks_trace,
1069	.cb_barrier = rcu_barrier_tasks_trace,
1070	.gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread,
1071	.get_gp_data = rcu_tasks_trace_get_gp_data,
1072	.cbflood_max = 50000,
1073	.irq_capable = 1,
1074	.slow_gps = 1,
1075	.name = "tasks-tracing"
1076	};
1077
1078	#define TASKS_TRACING_OPS &tasks_tracing_ops,
1079
1080	#else // #ifdef CONFIG_TASKS_TRACE_RCU
1081
1082	#define TASKS_TRACING_OPS
1083
1084	#endif // #else #ifdef CONFIG_TASKS_TRACE_RCU
1085
1086
1087	static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old)
1088	{
1089	if (!cur_ops->gp_diff)
1090	return new - old;
1091	return cur_ops->gp_diff(new, old);
1092	}
1093
1094	/*
1095	* RCU torture priority-boost testing. Runs one real-time thread per
1096	* CPU for moderate bursts, repeatedly starting grace periods and waiting
1097	* for them to complete. If a given grace period takes too long, we assume
1098	* that priority inversion has occurred.
1099	*/
1100
1101	static int old_rt_runtime = -1;
1102
1103	static void rcu_torture_disable_rt_throttle(void)
1104	{
1105	/*
1106	* Disable RT throttling so that rcutorture's boost threads don't get
1107	* throttled. Only possible if rcutorture is built-in otherwise the
1108	* user should manually do this by setting the sched_rt_period_us and
1109	* sched_rt_runtime sysctls.
1110	*/
1111	if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1)
1112	return;
1113
1114	old_rt_runtime = sysctl_sched_rt_runtime;
1115	sysctl_sched_rt_runtime = -1;
1116	}
1117
1118	static void rcu_torture_enable_rt_throttle(void)
1119	{
1120	if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1)
1121	return;
1122
1123	sysctl_sched_rt_runtime = old_rt_runtime;
1124	old_rt_runtime = -1;
1125	}
1126
1127	static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start)
1128	{
1129	int cpu;
1130	static int dbg_done;
1131	unsigned long end = jiffies;
1132	bool gp_done;
1133	unsigned long j;
1134	static unsigned long last_persist;
1135	unsigned long lp;
1136	unsigned long mininterval = test_boost_duration * HZ - HZ / 2;
1137
1138	if (end - *start > mininterval) {
1139	// Recheck after checking time to avoid false positives.
1140	smp_mb(); // Time check before grace-period check.
1141	if (cur_ops->poll_gp_state(gp_state))
1142	return false; // passed, though perhaps just barely
1143	if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {
1144	// At most one persisted message per boost test.
1145	j = jiffies;
1146	lp = READ_ONCE(last_persist);
1147	if (time_after(j, lp + mininterval) &&
1148	cmpxchg(&last_persist, lp, j) == lp) {
1149	if (cpu < 0)
1150	pr_info("Boost inversion persisted: QS from all CPUs\n");
1151	else
1152	pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu);
1153	}
1154	return false; // passed on a technicality
1155	}
1156	VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
1157	n_rcu_torture_boost_failure++;
1158	if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {
1159	pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n",
1160	current->rt_priority, gp_state, end - *start);
1161	cur_ops->gp_kthread_dbg();
1162	// Recheck after print to flag grace period ending during splat.
1163	gp_done = cur_ops->poll_gp_state(gp_state);
1164	pr_info("Boost inversion: GP %lu %s.\n", gp_state,
1165	gp_done ? "ended already" : "still pending");
1166
1167	}
1168
1169	return true; // failed
1170	} else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) {
1171	*start = jiffies;
1172	}
1173
1174	return false; // passed
1175	}
1176
1177	static int rcu_torture_boost(void *arg)
1178	{
1179	unsigned long endtime;
1180	unsigned long gp_state;
1181	unsigned long gp_state_time;
1182	unsigned long oldstarttime;
1183	unsigned long booststarttime = get_torture_init_jiffies() + test_boost_holdoff * HZ;
1184
1185	if (test_boost_holdoff <= 0 || time_after(jiffies, booststarttime)) {
1186	VERBOSE_TOROUT_STRING("rcu_torture_boost started");
1187	} else {
1188	VERBOSE_TOROUT_STRING("rcu_torture_boost started holdoff period");
1189	while (time_before(jiffies, booststarttime)) {
1190	schedule_timeout_idle(HZ);
1191	if (kthread_should_stop())
1192	goto cleanup;
1193	}
1194	VERBOSE_TOROUT_STRING("rcu_torture_boost finished holdoff period");
1195	}
1196
1197	/* Set real-time priority. */
1198	sched_set_fifo_low(current);
1199
1200	/* Each pass through the following loop does one boost-test cycle. */
1201	do {
1202	bool failed = false; // Test failed already in this test interval
1203	bool gp_initiated = false;
1204
1205	if (kthread_should_stop())
1206	goto checkwait;
1207
1208	/* Wait for the next test interval. */
1209	oldstarttime = READ_ONCE(boost_starttime);
1210	while (time_before(jiffies, oldstarttime)) {
1211	schedule_timeout_interruptible(timeout: oldstarttime - jiffies);
1212	if (stutter_wait(title: "rcu_torture_boost"))
1213	sched_set_fifo_low(current);
1214	if (torture_must_stop())
1215	goto checkwait;
1216	}
1217
1218	// Do one boost-test interval.
1219	endtime = oldstarttime + test_boost_duration * HZ;
1220	while (time_before(jiffies, endtime)) {
1221	// Has current GP gone too long?
1222	if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1223	failed = rcu_torture_boost_failed(gp_state, start: &gp_state_time);
1224	// If we don't have a grace period in flight, start one.
1225	if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) {
1226	gp_state = cur_ops->start_gp_poll();
1227	gp_initiated = true;
1228	gp_state_time = jiffies;
1229	}
1230	if (stutter_wait(title: "rcu_torture_boost")) {
1231	sched_set_fifo_low(current);
1232	// If the grace period already ended,
1233	// we don't know when that happened, so
1234	// start over.
1235	if (cur_ops->poll_gp_state(gp_state))
1236	gp_initiated = false;
1237	}
1238	if (torture_must_stop())
1239	goto checkwait;
1240	}
1241
1242	// In case the grace period extended beyond the end of the loop.
1243	if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1244	rcu_torture_boost_failed(gp_state, start: &gp_state_time);
1245
1246	/*
1247	* Set the start time of the next test interval.
1248	* Yes, this is vulnerable to long delays, but such
1249	* delays simply cause a false negative for the next
1250	* interval. Besides, we are running at RT priority,
1251	* so delays should be relatively rare.
1252	*/
1253	while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) {
1254	if (mutex_trylock(&boost_mutex)) {
1255	if (oldstarttime == boost_starttime) {
1256	WRITE_ONCE(boost_starttime,
1257	jiffies + test_boost_interval * HZ);
1258	n_rcu_torture_boosts++;
1259	}
1260	mutex_unlock(lock: &boost_mutex);
1261	break;
1262	}
1263	schedule_timeout_uninterruptible(HZ / 20);
1264	}
1265
1266	/* Go do the stutter. */
1267	checkwait: if (stutter_wait(title: "rcu_torture_boost"))
1268	sched_set_fifo_low(current);
1269	} while (!torture_must_stop());
1270
1271	cleanup:
1272	/* Clean up and exit. */
1273	while (!kthread_should_stop()) {
1274	torture_shutdown_absorb(title: "rcu_torture_boost");
1275	schedule_timeout_uninterruptible(HZ / 20);
1276	}
1277	torture_kthread_stopping(title: "rcu_torture_boost");
1278	return 0;
1279	}
1280
1281	/*
1282	* RCU torture force-quiescent-state kthread. Repeatedly induces
1283	* bursts of calls to force_quiescent_state(), increasing the probability
1284	* of occurrence of some important types of race conditions.
1285	*/
1286	static int
1287	rcu_torture_fqs(void *arg)
1288	{
1289	unsigned long fqs_resume_time;
1290	int fqs_burst_remaining;
1291	int oldnice = task_nice(current);
1292
1293	VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
1294	do {
1295	fqs_resume_time = jiffies + fqs_stutter * HZ;
1296	while (time_before(jiffies, fqs_resume_time) &&
1297	!kthread_should_stop()) {
1298	schedule_timeout_interruptible(HZ / 20);
1299	}
1300	fqs_burst_remaining = fqs_duration;
1301	while (fqs_burst_remaining > 0 &&
1302	!kthread_should_stop()) {
1303	cur_ops->fqs();
1304	udelay(usec: fqs_holdoff);
1305	fqs_burst_remaining -= fqs_holdoff;
1306	}
1307	if (stutter_wait(title: "rcu_torture_fqs"))
1308	sched_set_normal(current, nice: oldnice);
1309	} while (!torture_must_stop());
1310	torture_kthread_stopping(title: "rcu_torture_fqs");
1311	return 0;
1312	}
1313
1314	// Used by writers to randomly choose from the available grace-period primitives.
1315	static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { };
1316	static int nsynctypes;
1317
1318	/*
1319	* Determine which grace-period primitives are available.
1320	*/
1321	static void rcu_torture_write_types(void)
1322	{
1323	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full;
1324	bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp;
1325	bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
1326	bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;
1327
1328	/* Initialize synctype[] array. If none set, take default. */
1329	if (!gp_cond1 &&
1330	!gp_cond_exp1 &&
1331	!gp_cond_full1 &&
1332	!gp_cond_exp_full1 &&
1333	!gp_exp1 &&
1334	!gp_poll_exp1 &&
1335	!gp_poll_exp_full1 &&
1336	!gp_normal1 &&
1337	!gp_poll1 &&
1338	!gp_poll_full1 &&
1339	!gp_sync1) {
1340	gp_cond1 = true;
1341	gp_cond_exp1 = true;
1342	gp_cond_full1 = true;
1343	gp_cond_exp_full1 = true;
1344	gp_exp1 = true;
1345	gp_poll_exp1 = true;
1346	gp_poll_exp_full1 = true;
1347	gp_normal1 = true;
1348	gp_poll1 = true;
1349	gp_poll_full1 = true;
1350	gp_sync1 = true;
1351	}
1352	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
1353	synctype[nsynctypes++] = RTWS_COND_GET;
1354	pr_info("%s: Testing conditional GPs.\n", __func__);
1355	} else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {
1356	pr_alert("%s: gp_cond without primitives.\n", __func__);
1357	}
1358	if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {
1359	synctype[nsynctypes++] = RTWS_COND_GET_EXP;
1360	pr_info("%s: Testing conditional expedited GPs.\n", __func__);
1361	} else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {
1362	pr_alert("%s: gp_cond_exp without primitives.\n", __func__);
1363	}
1364	if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {
1365	synctype[nsynctypes++] = RTWS_COND_GET_FULL;
1366	pr_info("%s: Testing conditional full-state GPs.\n", __func__);
1367	} else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {
1368	pr_alert("%s: gp_cond_full without primitives.\n", __func__);
1369	}
1370	if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {
1371	synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL;
1372	pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__);
1373	} else if (gp_cond_exp_full &&
1374	(!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {
1375	pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__);
1376	}
1377	if (gp_exp1 && cur_ops->exp_sync) {
1378	synctype[nsynctypes++] = RTWS_EXP_SYNC;
1379	pr_info("%s: Testing expedited GPs.\n", __func__);
1380	} else if (gp_exp && !cur_ops->exp_sync) {
1381	pr_alert("%s: gp_exp without primitives.\n", __func__);
1382	}
1383	if (gp_normal1 && cur_ops->deferred_free) {
1384	synctype[nsynctypes++] = RTWS_DEF_FREE;
1385	pr_info("%s: Testing asynchronous GPs.\n", __func__);
1386	} else if (gp_normal && !cur_ops->deferred_free) {
1387	pr_alert("%s: gp_normal without primitives.\n", __func__);
1388	}
1389	if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state &&
1390	cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
1391	synctype[nsynctypes++] = RTWS_POLL_GET;
1392	pr_info("%s: Testing polling GPs.\n", __func__);
1393	} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
1394	pr_alert("%s: gp_poll without primitives.\n", __func__);
1395	}
1396	if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full
1397	&& cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
1398	synctype[nsynctypes++] = RTWS_POLL_GET_FULL;
1399	pr_info("%s: Testing polling full-state GPs.\n", __func__);
1400	} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
1401	pr_alert("%s: gp_poll_full without primitives.\n", __func__);
1402	}
1403	if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
1404	synctype[nsynctypes++] = RTWS_POLL_GET_EXP;
1405	pr_info("%s: Testing polling expedited GPs.\n", __func__);
1406	} else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {
1407	pr_alert("%s: gp_poll_exp without primitives.\n", __func__);
1408	}
1409	if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {
1410	synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL;
1411	pr_info("%s: Testing polling full-state expedited GPs.\n", __func__);
1412	} else if (gp_poll_exp_full &&
1413	(!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {
1414	pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__);
1415	}
1416	if (gp_sync1 && cur_ops->sync) {
1417	synctype[nsynctypes++] = RTWS_SYNC;
1418	pr_info("%s: Testing normal GPs.\n", __func__);
1419	} else if (gp_sync && !cur_ops->sync) {
1420	pr_alert("%s: gp_sync without primitives.\n", __func__);
1421	}
1422	pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes);
1423	pr_info("%s: gp_cond_wi %d gp_cond_wi_exp %d gp_poll_wi %d gp_poll_wi_exp %d\n", __func__, gp_cond_wi, gp_cond_wi_exp, gp_poll_wi, gp_poll_wi_exp);
1424	}
1425
1426	/*
1427	* Do the specified rcu_torture_writer() synchronous grace period,
1428	* while also testing out the polled APIs. Note well that the single-CPU
1429	* grace-period optimizations must be accounted for.
1430	*/
1431	static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void))
1432	{
1433	unsigned long cookie;
1434	struct rcu_gp_oldstate cookie_full;
1435	bool dopoll;
1436	bool dopoll_full;
1437	unsigned long r = torture_random(trsp);
1438
1439	dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);
1440	dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);
1441	if (dopoll || dopoll_full)
1442	cpus_read_lock();
1443	if (dopoll)
1444	cookie = cur_ops->get_gp_state();
1445	if (dopoll_full)
1446	cur_ops->get_gp_state_full(&cookie_full);
1447	if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))
1448	sync();
1449	sync();
1450	WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),
1451	"%s: Cookie check 3 failed %pS() online %*pbl.",
1452	__func__, sync, cpumask_pr_args(cpu_online_mask));
1453	WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),
1454	"%s: Cookie check 4 failed %pS() online %*pbl",
1455	__func__, sync, cpumask_pr_args(cpu_online_mask));
1456	if (dopoll || dopoll_full)
1457	cpus_read_unlock();
1458	}
1459
1460	/*
1461	* RCU torture writer kthread. Repeatedly substitutes a new structure
1462	* for that pointed to by rcu_torture_current, freeing the old structure
1463	* after a series of grace periods (the "pipeline").
1464	*/
1465	static int
1466	rcu_torture_writer(void *arg)
1467	{
1468	bool boot_ended;
1469	bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();
1470	unsigned long cookie;
1471	struct rcu_gp_oldstate cookie_full;
1472	int expediting = 0;
1473	unsigned long gp_snap;
1474	unsigned long gp_snap1;
1475	struct rcu_gp_oldstate gp_snap_full;
1476	struct rcu_gp_oldstate gp_snap1_full;
1477	int i;
1478	int idx;
1479	int oldnice = task_nice(current);
1480	struct rcu_gp_oldstate *rgo = NULL;
1481	int rgo_size = 0;
1482	struct rcu_torture *rp;
1483	struct rcu_torture *old_rp;
1484	static DEFINE_TORTURE_RANDOM(rand);
1485	unsigned long stallsdone = jiffies;
1486	bool stutter_waited;
1487	unsigned long *ulo = NULL;
1488	int ulo_size = 0;
1489
1490	// If a new stall test is added, this must be adjusted.
1491	if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu)
1492	stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) *
1493	HZ * (stall_cpu_repeat + 1);
1494	VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
1495	if (!can_expedite)
1496	pr_alert("%s" TORTURE_FLAG
1497	" GP expediting controlled from boot/sysfs for %s.\n",
1498	torture_type, cur_ops->name);
1499	if (WARN_ONCE(nsynctypes == 0,
1500	"%s: No update-side primitives.\n", __func__)) {
1501	/*
1502	* No updates primitives, so don't try updating.
1503	* The resulting test won't be testing much, hence the
1504	* above WARN_ONCE().
1505	*/
1506	rcu_torture_writer_state = RTWS_STOPPING;
1507	torture_kthread_stopping(title: "rcu_torture_writer");
1508	return 0;
1509	}
1510	if (cur_ops->poll_active > 0) {
1511	ulo = kzalloc(cur_ops->poll_active * sizeof(ulo[0]), GFP_KERNEL);
1512	if (!WARN_ON(!ulo))
1513	ulo_size = cur_ops->poll_active;
1514	}
1515	if (cur_ops->poll_active_full > 0) {
1516	rgo = kzalloc(cur_ops->poll_active_full * sizeof(rgo[0]), GFP_KERNEL);
1517	if (!WARN_ON(!rgo))
1518	rgo_size = cur_ops->poll_active_full;
1519	}
1520
1521	do {
1522	rcu_torture_writer_state = RTWS_FIXED_DELAY;
1523	torture_hrtimeout_us(baset_us: 500, fuzzt_ns: 1000, trsp: &rand);
1524	rp = rcu_torture_alloc();
1525	if (rp == NULL)
1526	continue;
1527	rp->rtort_pipe_count = 0;
1528	ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count);
1529	rcu_torture_writer_state = RTWS_DELAY;
1530	udelay(usec: torture_random(trsp: &rand) & 0x3ff);
1531	rcu_torture_writer_state = RTWS_REPLACE;
1532	old_rp = rcu_dereference_check(rcu_torture_current,
1533	current == writer_task);
1534	rp->rtort_mbtest = 1;
1535	rcu_assign_pointer(rcu_torture_current, rp);
1536	smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
1537	if (old_rp) {
1538	i = old_rp->rtort_pipe_count;
1539	if (i > RCU_TORTURE_PIPE_LEN)
1540	i = RCU_TORTURE_PIPE_LEN;
1541	atomic_inc(v: &rcu_torture_wcount[i]);
1542	WRITE_ONCE(old_rp->rtort_pipe_count,
1543	old_rp->rtort_pipe_count + 1);
1544	ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count);
1545
1546	// Make sure readers block polled grace periods.
1547	if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {
1548	idx = cur_ops->readlock();
1549	cookie = cur_ops->get_gp_state();
1550	WARN_ONCE(cur_ops->poll_gp_state(cookie),
1551	"%s: Cookie check 1 failed %s(%d) %lu->%lu\n",
1552	__func__,
1553	rcu_torture_writer_state_getname(),
1554	rcu_torture_writer_state,
1555	cookie, cur_ops->get_gp_state());
1556	if (cur_ops->get_comp_state) {
1557	cookie = cur_ops->get_comp_state();
1558	WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
1559	}
1560	cur_ops->readunlock(idx);
1561	}
1562	if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {
1563	idx = cur_ops->readlock();
1564	cur_ops->get_gp_state_full(&cookie_full);
1565	WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
1566	"%s: Cookie check 5 failed %s(%d) online %*pbl\n",
1567	__func__,
1568	rcu_torture_writer_state_getname(),
1569	rcu_torture_writer_state,
1570	cpumask_pr_args(cpu_online_mask));
1571	if (cur_ops->get_comp_state_full) {
1572	cur_ops->get_comp_state_full(&cookie_full);
1573	WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));
1574	}
1575	cur_ops->readunlock(idx);
1576	}
1577	switch (synctype[torture_random(trsp: &rand) % nsynctypes]) {
1578	case RTWS_DEF_FREE:
1579	rcu_torture_writer_state = RTWS_DEF_FREE;
1580	cur_ops->deferred_free(old_rp);
1581	break;
1582	case RTWS_EXP_SYNC:
1583	rcu_torture_writer_state = RTWS_EXP_SYNC;
1584	do_rtws_sync(trsp: &rand, sync: cur_ops->exp_sync);
1585	rcu_torture_pipe_update(old_rp);
1586	break;
1587	case RTWS_COND_GET:
1588	rcu_torture_writer_state = RTWS_COND_GET;
1589	gp_snap = cur_ops->get_gp_state();
1590	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_cond_wi,
1591	fuzzt_ns: 1000, trsp: &rand);
1592	rcu_torture_writer_state = RTWS_COND_SYNC;
1593	cur_ops->cond_sync(gp_snap);
1594	rcu_torture_pipe_update(old_rp);
1595	break;
1596	case RTWS_COND_GET_EXP:
1597	rcu_torture_writer_state = RTWS_COND_GET_EXP;
1598	gp_snap = cur_ops->get_gp_state_exp();
1599	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_cond_wi_exp,
1600	fuzzt_ns: 1000, trsp: &rand);
1601	rcu_torture_writer_state = RTWS_COND_SYNC_EXP;
1602	cur_ops->cond_sync_exp(gp_snap);
1603	rcu_torture_pipe_update(old_rp);
1604	break;
1605	case RTWS_COND_GET_FULL:
1606	rcu_torture_writer_state = RTWS_COND_GET_FULL;
1607	cur_ops->get_gp_state_full(&gp_snap_full);
1608	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_cond_wi,
1609	fuzzt_ns: 1000, trsp: &rand);
1610	rcu_torture_writer_state = RTWS_COND_SYNC_FULL;
1611	cur_ops->cond_sync_full(&gp_snap_full);
1612	rcu_torture_pipe_update(old_rp);
1613	break;
1614	case RTWS_COND_GET_EXP_FULL:
1615	rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL;
1616	cur_ops->get_gp_state_full(&gp_snap_full);
1617	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_cond_wi_exp,
1618	fuzzt_ns: 1000, trsp: &rand);
1619	rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL;
1620	cur_ops->cond_sync_exp_full(&gp_snap_full);
1621	rcu_torture_pipe_update(old_rp);
1622	break;
1623	case RTWS_POLL_GET:
1624	rcu_torture_writer_state = RTWS_POLL_GET;
1625	for (i = 0; i < ulo_size; i++)
1626	ulo[i] = cur_ops->get_comp_state();
1627	gp_snap = cur_ops->start_gp_poll();
1628	rcu_torture_writer_state = RTWS_POLL_WAIT;
1629	while (!cur_ops->poll_gp_state(gp_snap)) {
1630	gp_snap1 = cur_ops->get_gp_state();
1631	for (i = 0; i < ulo_size; i++)
1632	if (cur_ops->poll_gp_state(ulo[i]) ||
1633	cur_ops->same_gp_state(ulo[i], gp_snap1)) {
1634	ulo[i] = gp_snap1;
1635	break;
1636	}
1637	WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size);
1638	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_poll_wi,
1639	fuzzt_ns: 1000, trsp: &rand);
1640	}
1641	rcu_torture_pipe_update(old_rp);
1642	break;
1643	case RTWS_POLL_GET_FULL:
1644	rcu_torture_writer_state = RTWS_POLL_GET_FULL;
1645	for (i = 0; i < rgo_size; i++)
1646	cur_ops->get_comp_state_full(&rgo[i]);
1647	cur_ops->start_gp_poll_full(&gp_snap_full);
1648	rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
1649	while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1650	cur_ops->get_gp_state_full(&gp_snap1_full);
1651	for (i = 0; i < rgo_size; i++)
1652	if (cur_ops->poll_gp_state_full(&rgo[i]) ||
1653	cur_ops->same_gp_state_full(&rgo[i],
1654	&gp_snap1_full)) {
1655	rgo[i] = gp_snap1_full;
1656	break;
1657	}
1658	WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size);
1659	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_poll_wi,
1660	fuzzt_ns: 1000, trsp: &rand);
1661	}
1662	rcu_torture_pipe_update(old_rp);
1663	break;
1664	case RTWS_POLL_GET_EXP:
1665	rcu_torture_writer_state = RTWS_POLL_GET_EXP;
1666	gp_snap = cur_ops->start_gp_poll_exp();
1667	rcu_torture_writer_state = RTWS_POLL_WAIT_EXP;
1668	while (!cur_ops->poll_gp_state_exp(gp_snap))
1669	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_poll_wi_exp,
1670	fuzzt_ns: 1000, trsp: &rand);
1671	rcu_torture_pipe_update(old_rp);
1672	break;
1673	case RTWS_POLL_GET_EXP_FULL:
1674	rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL;
1675	cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1676	rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL;
1677	while (!cur_ops->poll_gp_state_full(&gp_snap_full))
1678	torture_hrtimeout_us(baset_us: torture_random(trsp: &rand) % gp_poll_wi_exp,
1679	fuzzt_ns: 1000, trsp: &rand);
1680	rcu_torture_pipe_update(old_rp);
1681	break;
1682	case RTWS_SYNC:
1683	rcu_torture_writer_state = RTWS_SYNC;
1684	do_rtws_sync(trsp: &rand, sync: cur_ops->sync);
1685	rcu_torture_pipe_update(old_rp);
1686	break;
1687	default:
1688	WARN_ON_ONCE(1);
1689	break;
1690	}
1691	}
1692	WRITE_ONCE(rcu_torture_current_version,
1693	rcu_torture_current_version + 1);
1694	/* Cycle through nesting levels of rcu_expedite_gp() calls. */
1695	if (can_expedite &&
1696	!(torture_random(trsp: &rand) & 0xff & (!!expediting - 1))) {
1697	WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
1698	if (expediting >= 0)
1699	rcu_expedite_gp();
1700	else
1701	rcu_unexpedite_gp();
1702	if (++expediting > 3)
1703	expediting = -expediting;
1704	} else if (!can_expedite) { /* Disabled during boot, recheck. */
1705	can_expedite = !rcu_gp_is_expedited() &&
1706	!rcu_gp_is_normal();
1707	}
1708	rcu_torture_writer_state = RTWS_STUTTER;
1709	boot_ended = rcu_inkernel_boot_has_ended();
1710	stutter_waited = stutter_wait(title: "rcu_torture_writer");
1711	if (stutter_waited &&
1712	!atomic_read(v: &rcu_fwd_cb_nodelay) &&
1713	!cur_ops->slow_gps &&
1714	!torture_must_stop() &&
1715	boot_ended &&
1716	time_after(jiffies, stallsdone))
1717	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)
1718	if (list_empty(head: &rcu_tortures[i].rtort_free) &&
1719	rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) {
1720	tracing_off();
1721	if (cur_ops->gp_kthread_dbg)
1722	cur_ops->gp_kthread_dbg();
1723	WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count);
1724	rcu_ftrace_dump(DUMP_ALL);
1725	}
1726	if (stutter_waited)
1727	sched_set_normal(current, nice: oldnice);
1728	} while (!torture_must_stop());
1729	rcu_torture_current = NULL; // Let stats task know that we are done.
1730	/* Reset expediting back to unexpedited. */
1731	if (expediting > 0)
1732	expediting = -expediting;
1733	while (can_expedite && expediting++ < 0)
1734	rcu_unexpedite_gp();
1735	WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
1736	if (!can_expedite)
1737	pr_alert("%s" TORTURE_FLAG
1738	" Dynamic grace-period expediting was disabled.\n",
1739	torture_type);
1740	kfree(objp: ulo);
1741	kfree(objp: rgo);
1742	rcu_torture_writer_state = RTWS_STOPPING;
1743	torture_kthread_stopping(title: "rcu_torture_writer");
1744	return 0;
1745	}
1746
1747	/*
1748	* RCU torture fake writer kthread. Repeatedly calls sync, with a random
1749	* delay between calls.
1750	*/
1751	static int
1752	rcu_torture_fakewriter(void *arg)
1753	{
1754	unsigned long gp_snap;
1755	struct rcu_gp_oldstate gp_snap_full;
1756	DEFINE_TORTURE_RANDOM(rand);
1757
1758	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1759	set_user_nice(current, MAX_NICE);
1760
1761	if (WARN_ONCE(nsynctypes == 0,
1762	"%s: No update-side primitives.\n", __func__)) {
1763	/*
1764	* No updates primitives, so don't try updating.
1765	* The resulting test won't be testing much, hence the
1766	* above WARN_ONCE().
1767	*/
1768	torture_kthread_stopping(title: "rcu_torture_fakewriter");
1769	return 0;
1770	}
1771
1772	do {
1773	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 10, trsp: &rand);
1774	if (cur_ops->cb_barrier != NULL &&
1775	torture_random(trsp: &rand) % (nrealfakewriters * 8) == 0) {
1776	cur_ops->cb_barrier();
1777	} else {
1778	switch (synctype[torture_random(trsp: &rand) % nsynctypes]) {
1779	case RTWS_DEF_FREE:
1780	break;
1781	case RTWS_EXP_SYNC:
1782	cur_ops->exp_sync();
1783	break;
1784	case RTWS_COND_GET:
1785	gp_snap = cur_ops->get_gp_state();
1786	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16, trsp: &rand);
1787	cur_ops->cond_sync(gp_snap);
1788	break;
1789	case RTWS_COND_GET_EXP:
1790	gp_snap = cur_ops->get_gp_state_exp();
1791	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16, trsp: &rand);
1792	cur_ops->cond_sync_exp(gp_snap);
1793	break;
1794	case RTWS_COND_GET_FULL:
1795	cur_ops->get_gp_state_full(&gp_snap_full);
1796	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16, trsp: &rand);
1797	cur_ops->cond_sync_full(&gp_snap_full);
1798	break;
1799	case RTWS_COND_GET_EXP_FULL:
1800	cur_ops->get_gp_state_full(&gp_snap_full);
1801	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16, trsp: &rand);
1802	cur_ops->cond_sync_exp_full(&gp_snap_full);
1803	break;
1804	case RTWS_POLL_GET:
1805	if (cur_ops->start_poll_irqsoff)
1806	local_irq_disable();
1807	gp_snap = cur_ops->start_gp_poll();
1808	if (cur_ops->start_poll_irqsoff)
1809	local_irq_enable();
1810	while (!cur_ops->poll_gp_state(gp_snap)) {
1811	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16,
1812	trsp: &rand);
1813	}
1814	break;
1815	case RTWS_POLL_GET_FULL:
1816	if (cur_ops->start_poll_irqsoff)
1817	local_irq_disable();
1818	cur_ops->start_gp_poll_full(&gp_snap_full);
1819	if (cur_ops->start_poll_irqsoff)
1820	local_irq_enable();
1821	while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1822	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16,
1823	trsp: &rand);
1824	}
1825	break;
1826	case RTWS_POLL_GET_EXP:
1827	gp_snap = cur_ops->start_gp_poll_exp();
1828	while (!cur_ops->poll_gp_state_exp(gp_snap)) {
1829	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16,
1830	trsp: &rand);
1831	}
1832	break;
1833	case RTWS_POLL_GET_EXP_FULL:
1834	cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1835	while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1836	torture_hrtimeout_jiffies(baset_j: torture_random(trsp: &rand) % 16,
1837	trsp: &rand);
1838	}
1839	break;
1840	case RTWS_SYNC:
1841	cur_ops->sync();
1842	break;
1843	default:
1844	WARN_ON_ONCE(1);
1845	break;
1846	}
1847	}
1848	stutter_wait(title: "rcu_torture_fakewriter");
1849	} while (!torture_must_stop());
1850
1851	torture_kthread_stopping(title: "rcu_torture_fakewriter");
1852	return 0;
1853	}
1854
1855	static void rcu_torture_timer_cb(struct rcu_head *rhp)
1856	{
1857	kfree(objp: rhp);
1858	}
1859
1860	// Set up and carry out testing of RCU's global memory ordering
1861	static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp,
1862	struct torture_random_state *trsp)
1863	{
1864	unsigned long loops;
1865	int noc = torture_num_online_cpus();
1866	int rdrchked;
1867	int rdrchker;
1868	struct rcu_torture_reader_check *rtrcp; // Me.
1869	struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking.
1870	struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked.
1871	struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me.
1872
1873	if (myid < 0)
1874	return; // Don't try this from timer handlers.
1875
1876	// Increment my counter.
1877	rtrcp = &rcu_torture_reader_mbchk[myid];
1878	WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1);
1879
1880	// Attempt to assign someone else some checking work.
1881	rdrchked = torture_random(trsp) % nrealreaders;
1882	rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked];
1883	rdrchker = torture_random(trsp) % nrealreaders;
1884	rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker];
1885	if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker &&
1886	smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below.
1887	!READ_ONCE(rtp->rtort_chkp) &&
1888	!smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below.
1889	rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops);
1890	WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0);
1891	rtrcp->rtc_chkrdr = rdrchked;
1892	WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends.
1893	if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) ||
1894	cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp))
1895	(void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out.
1896	}
1897
1898	// If assigned some completed work, do it!
1899	rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner);
1900	if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready))
1901	return; // No work or work not yet ready.
1902	rdrchked = rtrcp_assigner->rtc_chkrdr;
1903	if (WARN_ON_ONCE(rdrchked < 0))
1904	return;
1905	rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked];
1906	loops = READ_ONCE(rtrcp_chked->rtc_myloops);
1907	atomic_inc(v: &n_rcu_torture_mbchk_tries);
1908	if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops))
1909	atomic_inc(v: &n_rcu_torture_mbchk_fail);
1910	rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2;
1911	rtrcp_assigner->rtc_ready = 0;
1912	smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work.
1913	smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign.
1914	}
1915
1916	// Verify the specified RCUTORTURE_RDR* state.
1917	#define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count()
1918	static void rcutorture_one_extend_check(char *s, int curstate, int new, int old, bool insoftirq)
1919	{
1920	int mask;
1921
1922	if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE))
1923	return;
1924
1925	WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled(), ROEC_ARGS);
1926	WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS);
1927
1928	// If CONFIG_PREEMPT_COUNT=n, further checks are unreliable.
1929	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
1930	return;
1931
1932	WARN_ONCE((curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) &&
1933	!(preempt_count() & SOFTIRQ_MASK), ROEC_ARGS);
1934	WARN_ONCE((curstate & (RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED)) &&
1935	!(preempt_count() & PREEMPT_MASK), ROEC_ARGS);
1936	WARN_ONCE(cur_ops->readlock_nesting &&
1937	(curstate & (RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2)) &&
1938	cur_ops->readlock_nesting() == 0, ROEC_ARGS);
1939
1940	// Timer handlers have all sorts of stuff disabled, so ignore
1941	// unintended disabling.
1942	if (insoftirq)
1943	return;
1944
1945	WARN_ONCE(cur_ops->extendables &&
1946	!(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) &&
1947	(preempt_count() & SOFTIRQ_MASK), ROEC_ARGS);
1948
1949	/*
1950	* non-preemptible RCU in a preemptible kernel uses preempt_disable()
1951	* as rcu_read_lock().
1952	*/
1953	mask = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
1954	if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
1955	mask |= RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
1956
1957	WARN_ONCE(cur_ops->extendables && !(curstate & mask) &&
1958	(preempt_count() & PREEMPT_MASK), ROEC_ARGS);
1959
1960	/*
1961	* non-preemptible RCU in a preemptible kernel uses "preempt_count() &
1962	* PREEMPT_MASK" as ->readlock_nesting().
1963	*/
1964	mask = RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
1965	if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
1966	mask |= RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
1967
1968	WARN_ONCE(cur_ops->readlock_nesting && !(curstate & mask) &&
1969	cur_ops->readlock_nesting() > 0, ROEC_ARGS);
1970	}
1971
1972	/*
1973	* Do one extension of an RCU read-side critical section using the
1974	* current reader state in readstate (set to zero for initial entry
1975	* to extended critical section), set the new state as specified by
1976	* newstate (set to zero for final exit from extended critical section),
1977	* and random-number-generator state in trsp. If this is neither the
1978	* beginning or end of the critical section and if there was actually a
1979	* change, do a ->read_delay().
1980	*/
1981	static void rcutorture_one_extend(int *readstate, int newstate, bool insoftirq,
1982	struct torture_random_state *trsp,
1983	struct rt_read_seg *rtrsp)
1984	{
1985	bool first;
1986	unsigned long flags;
1987	int idxnew1 = -1;
1988	int idxnew2 = -1;
1989	int idxold1 = *readstate;
1990	int idxold2 = idxold1;
1991	int statesnew = ~*readstate & newstate;
1992	int statesold = *readstate & ~newstate;
1993
1994	first = idxold1 == 0;
1995	WARN_ON_ONCE(idxold2 < 0);
1996	WARN_ON_ONCE(idxold2 & ~RCUTORTURE_RDR_ALLBITS);
1997	rcutorture_one_extend_check(s: "before change", curstate: idxold1, new: statesnew, old: statesold, insoftirq);
1998	rtrsp->rt_readstate = newstate;
1999
2000	/* First, put new protection in place to avoid critical-section gap. */
2001	if (statesnew & RCUTORTURE_RDR_BH)
2002	local_bh_disable();
2003	if (statesnew & RCUTORTURE_RDR_RBH)
2004	rcu_read_lock_bh();
2005	if (statesnew & RCUTORTURE_RDR_IRQ)
2006	local_irq_disable();
2007	if (statesnew & RCUTORTURE_RDR_PREEMPT)
2008	preempt_disable();
2009	if (statesnew & RCUTORTURE_RDR_SCHED)
2010	rcu_read_lock_sched();
2011	if (statesnew & RCUTORTURE_RDR_RCU_1)
2012	idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1;
2013	if (statesnew & RCUTORTURE_RDR_RCU_2)
2014	idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2;
2015
2016	// Complain unless both the old and the new protection is in place.
2017	rcutorture_one_extend_check(s: "during change",
2018	curstate: idxold1 | statesnew, new: statesnew, old: statesold, insoftirq);
2019
2020	// Sample CPU under both sets of protections to reduce confusion.
2021	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) {
2022	int cpu = raw_smp_processor_id();
2023	rtrsp->rt_cpu = cpu;
2024	if (!first) {
2025	rtrsp[-1].rt_end_cpu = cpu;
2026	if (cur_ops->reader_blocked)
2027	rtrsp[-1].rt_preempted = cur_ops->reader_blocked();
2028	}
2029	}
2030	// Sample grace-period sequence number, as good a place as any.
2031	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && cur_ops->gather_gp_seqs) {
2032	rtrsp->rt_gp_seq = cur_ops->gather_gp_seqs();
2033	rtrsp->rt_ts = ktime_get_mono_fast_ns();
2034	if (!first)
2035	rtrsp[-1].rt_gp_seq_end = rtrsp->rt_gp_seq;
2036	}
2037
2038	/*
2039	* Next, remove old protection, in decreasing order of strength
2040	* to avoid unlock paths that aren't safe in the stronger
2041	* context. Namely: BH can not be enabled with disabled interrupts.
2042	* Additionally PREEMPT_RT requires that BH is enabled in preemptible
2043	* context.
2044	*/
2045	if (statesold & RCUTORTURE_RDR_IRQ)
2046	local_irq_enable();
2047	if (statesold & RCUTORTURE_RDR_PREEMPT)
2048	preempt_enable();
2049	if (statesold & RCUTORTURE_RDR_SCHED)
2050	rcu_read_unlock_sched();
2051	if (statesold & RCUTORTURE_RDR_BH)
2052	local_bh_enable();
2053	if (statesold & RCUTORTURE_RDR_RBH)
2054	rcu_read_unlock_bh();
2055	if (statesold & RCUTORTURE_RDR_RCU_2) {
2056	cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2);
2057	WARN_ON_ONCE(idxnew2 != -1);
2058	idxold2 = 0;
2059	}
2060	if (statesold & RCUTORTURE_RDR_RCU_1) {
2061	bool lockit;
2062
2063	lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);
2064	if (lockit)
2065	raw_spin_lock_irqsave(&current->pi_lock, flags);
2066	cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1);
2067	WARN_ON_ONCE(idxnew1 != -1);
2068	idxold1 = 0;
2069	if (lockit)
2070	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
2071	}
2072
2073	/* Delay if neither beginning nor end and there was a change. */
2074	if ((statesnew || statesold) && *readstate && newstate)
2075	cur_ops->read_delay(trsp, rtrsp);
2076
2077	/* Update the reader state. */
2078	if (idxnew1 == -1)
2079	idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1;
2080	WARN_ON_ONCE(idxnew1 < 0);
2081	if (idxnew2 == -1)
2082	idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2;
2083	WARN_ON_ONCE(idxnew2 < 0);
2084	*readstate = idxnew1 | idxnew2 | newstate;
2085	WARN_ON_ONCE(*readstate < 0);
2086	if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS))
2087	pr_info("Unexpected readstate value of %#x\n", *readstate);
2088	rcutorture_one_extend_check(s: "after change", curstate: *readstate, new: statesnew, old: statesold, insoftirq);
2089	}
2090
2091	/* Return the biggest extendables mask given current RCU and boot parameters. */
2092	static int rcutorture_extend_mask_max(void)
2093	{
2094	int mask;
2095
2096	WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND);
2097	mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;
2098	mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
2099	return mask;
2100	}
2101
2102	/* Return a random protection state mask, but with at least one bit set. */
2103	static int
2104	rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp)
2105	{
2106	int mask = rcutorture_extend_mask_max();
2107	unsigned long randmask1 = torture_random(trsp);
2108	unsigned long randmask2 = randmask1 >> 3;
2109	unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
2110	unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ;
2111	unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
2112
2113	WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); // Can't have reader idx bits.
2114	/* Mostly only one bit (need preemption!), sometimes lots of bits. */
2115	if (!(randmask1 & 0x7))
2116	mask = mask & randmask2;
2117	else
2118	mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));
2119
2120	// Can't have nested RCU reader without outer RCU reader.
2121	if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) {
2122	if (oldmask & RCUTORTURE_RDR_RCU_1)
2123	mask &= ~RCUTORTURE_RDR_RCU_2;
2124	else
2125	mask |= RCUTORTURE_RDR_RCU_1;
2126	}
2127
2128	/*
2129	* Can't enable bh w/irq disabled.
2130	*/
2131	if (mask & RCUTORTURE_RDR_IRQ)
2132	mask |= oldmask & bhs;
2133
2134	/*
2135	* Ideally these sequences would be detected in debug builds
2136	* (regardless of RT), but until then don't stop testing
2137	* them on non-RT.
2138	*/
2139	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
2140	/* Can't modify BH in atomic context */
2141	if (oldmask & preempts_irq)
2142	mask &= ~bhs;
2143	if ((oldmask | mask) & preempts_irq)
2144	mask |= oldmask & bhs;
2145	}
2146
2147	return mask ?: RCUTORTURE_RDR_RCU_1;
2148	}
2149
2150	/*
2151	* Do a randomly selected number of extensions of an existing RCU read-side
2152	* critical section.
2153	*/
2154	static struct rt_read_seg *
2155	rcutorture_loop_extend(int *readstate, bool insoftirq, struct torture_random_state *trsp,
2156	struct rt_read_seg *rtrsp)
2157	{
2158	int i;
2159	int j;
2160	int mask = rcutorture_extend_mask_max();
2161
2162	WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */
2163	if (!((mask - 1) & mask))
2164	return rtrsp; /* Current RCU reader not extendable. */
2165	/* Bias towards larger numbers of loops. */
2166	i = torture_random(trsp);
2167	i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1;
2168	for (j = 0; j < i; j++) {
2169	mask = rcutorture_extend_mask(oldmask: *readstate, trsp);
2170	rcutorture_one_extend(readstate, newstate: mask, insoftirq, trsp, rtrsp: &rtrsp[j]);
2171	}
2172	return &rtrsp[j];
2173	}
2174
2175	struct rcu_torture_one_read_state {
2176	bool checkpolling;
2177	unsigned long cookie;
2178	struct rcu_gp_oldstate cookie_full;
2179	unsigned long started;
2180	struct rcu_torture *p;
2181	int readstate;
2182	struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS];
2183	struct rt_read_seg *rtrsp;
2184	unsigned long long ts;
2185	};
2186
2187	static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp,
2188	struct torture_random_state *trsp)
2189	{
2190	memset(rtorsp, 0, sizeof(*rtorsp));
2191	rtorsp->checkpolling = !(torture_random(trsp) & 0xfff);
2192	rtorsp->rtrsp = &rtorsp->rtseg[0];
2193	}
2194
2195	/*
2196	* Set up the first segment of a series of overlapping read-side
2197	* critical sections. The caller must have actually initiated the
2198	* outermost read-side critical section.
2199	*/
2200	static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp,
2201	struct torture_random_state *trsp, long myid)
2202	{
2203	if (rtorsp->checkpolling) {
2204	if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2205	rtorsp->cookie = cur_ops->get_gp_state();
2206	if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2207	cur_ops->get_gp_state_full(&rtorsp->cookie_full);
2208	}
2209	rtorsp->started = cur_ops->get_gp_seq();
2210	rtorsp->ts = rcu_trace_clock_local();
2211	rtorsp->p = rcu_dereference_check(rcu_torture_current,
2212	!cur_ops->readlock_held || cur_ops->readlock_held());
2213	if (rtorsp->p == NULL) {
2214	/* Wait for rcu_torture_writer to get underway */
2215	rcutorture_one_extend(readstate: &rtorsp->readstate, newstate: 0, insoftirq: myid < 0, trsp, rtrsp: rtorsp->rtrsp);
2216	return false;
2217	}
2218	if (rtorsp->p->rtort_mbtest == 0)
2219	atomic_inc(v: &n_rcu_torture_mberror);
2220	rcu_torture_reader_do_mbchk(myid, rtp: rtorsp->p, trsp);
2221	return true;
2222	}
2223
2224	/*
2225	* Complete the last segment of a series of overlapping read-side
2226	* critical sections and check for errors.
2227	*/
2228	static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp,
2229	struct torture_random_state *trsp, long myid)
2230	{
2231	int i;
2232	unsigned long completed;
2233	int pipe_count;
2234	bool preempted = false;
2235	struct rt_read_seg *rtrsp1;
2236
2237	preempt_disable();
2238	pipe_count = READ_ONCE(rtorsp->p->rtort_pipe_count);
2239	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
2240	// Should not happen in a correct RCU implementation,
2241	// happens quite often for torture_type=busted.
2242	pipe_count = RCU_TORTURE_PIPE_LEN;
2243	}
2244	completed = cur_ops->get_gp_seq();
2245	if (pipe_count > 1) {
2246	do_trace_rcu_torture_read(rcutorturename: cur_ops->name, rhp: &rtorsp->p->rtort_rcu,
2247	secs: rtorsp->ts, c_old: rtorsp->started, c: completed);
2248	rcu_ftrace_dump(DUMP_ALL);
2249	}
2250	__this_cpu_inc(rcu_torture_count[pipe_count]);
2251	completed = rcutorture_seq_diff(new: completed, old: rtorsp->started);
2252	if (completed > RCU_TORTURE_PIPE_LEN) {
2253	/* Should not happen, but... */
2254	completed = RCU_TORTURE_PIPE_LEN;
2255	}
2256	__this_cpu_inc(rcu_torture_batch[completed]);
2257	preempt_enable();
2258	if (rtorsp->checkpolling) {
2259	if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2260	WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie),
2261	"%s: Cookie check 2 failed %s(%d) %lu->%lu\n",
2262	__func__,
2263	rcu_torture_writer_state_getname(),
2264	rcu_torture_writer_state,
2265	rtorsp->cookie, cur_ops->get_gp_state());
2266	if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2267	WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full),
2268	"%s: Cookie check 6 failed %s(%d) online %*pbl\n",
2269	__func__,
2270	rcu_torture_writer_state_getname(),
2271	rcu_torture_writer_state,
2272	cpumask_pr_args(cpu_online_mask));
2273	}
2274	if (cur_ops->reader_blocked)
2275	preempted = cur_ops->reader_blocked();
2276	rcutorture_one_extend(readstate: &rtorsp->readstate, newstate: 0, insoftirq: myid < 0, trsp, rtrsp: rtorsp->rtrsp);
2277	WARN_ON_ONCE(rtorsp->readstate);
2278	// This next splat is expected behavior if leakpointer, especially
2279	// for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels.
2280	WARN_ON_ONCE(leakpointer && READ_ONCE(rtorsp->p->rtort_pipe_count) > 1);
2281
2282	/* If error or close call, record the sequence of reader protections. */
2283	if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) {
2284	i = 0;
2285	for (rtrsp1 = &rtorsp->rtseg[0]; rtrsp1 < rtorsp->rtrsp; rtrsp1++)
2286	err_segs[i++] = *rtrsp1;
2287	rt_read_nsegs = i;
2288	rt_read_preempted = preempted;
2289	}
2290	}
2291
2292	/*
2293	* Do one read-side critical section, returning false if there was
2294	* no data to read. Can be invoked both from process context and
2295	* from a timer handler.
2296	*/
2297	static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
2298	{
2299	int newstate;
2300	struct rcu_torture_one_read_state rtors;
2301
2302	WARN_ON_ONCE(!rcu_is_watching());
2303	init_rcu_torture_one_read_state(rtorsp: &rtors, trsp);
2304	newstate = rcutorture_extend_mask(oldmask: rtors.readstate, trsp);
2305	rcutorture_one_extend(readstate: &rtors.readstate, newstate, insoftirq: myid < 0, trsp, rtrsp: rtors.rtrsp++);
2306	if (!rcu_torture_one_read_start(rtorsp: &rtors, trsp, myid)) {
2307	rcutorture_one_extend(readstate: &rtors.readstate, newstate: 0, insoftirq: myid < 0, trsp, rtrsp: rtors.rtrsp);
2308	return false;
2309	}
2310	rtors.rtrsp = rcutorture_loop_extend(readstate: &rtors.readstate, insoftirq: myid < 0, trsp, rtrsp: rtors.rtrsp);
2311	rcu_torture_one_read_end(rtorsp: &rtors, trsp, myid);
2312	return true;
2313	}
2314
2315	static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand);
2316
2317	/*
2318	* RCU torture reader from timer handler. Dereferences rcu_torture_current,
2319	* incrementing the corresponding element of the pipeline array. The
2320	* counter in the element should never be greater than 1, otherwise, the
2321	* RCU implementation is broken.
2322	*/
2323	static void rcu_torture_timer(struct timer_list *unused)
2324	{
2325	atomic_long_inc(v: &n_rcu_torture_timers);
2326	(void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), myid: -1);
2327
2328	/* Test call_rcu() invocation from interrupt handler. */
2329	if (cur_ops->call) {
2330	struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT);
2331
2332	if (rhp)
2333	cur_ops->call(rhp, rcu_torture_timer_cb);
2334	}
2335	}
2336
2337	/*
2338	* RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
2339	* incrementing the corresponding element of the pipeline array. The
2340	* counter in the element should never be greater than 1, otherwise, the
2341	* RCU implementation is broken.
2342	*/
2343	static int
2344	rcu_torture_reader(void *arg)
2345	{
2346	unsigned long lastsleep = jiffies;
2347	long myid = (long)arg;
2348	int mynumonline = myid;
2349	DEFINE_TORTURE_RANDOM(rand);
2350	struct timer_list t;
2351
2352	VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
2353	set_user_nice(current, MAX_NICE);
2354	if (irqreader && cur_ops->irq_capable)
2355	timer_setup_on_stack(&t, rcu_torture_timer, 0);
2356	tick_dep_set_task(current, bit: TICK_DEP_BIT_RCU); // CPU bound, so need tick.
2357	do {
2358	if (irqreader && cur_ops->irq_capable) {
2359	if (!timer_pending(timer: &t))
2360	mod_timer(timer: &t, expires: jiffies + 1);
2361	}
2362	if (!rcu_torture_one_read(trsp: &rand, myid) && !torture_must_stop())
2363	schedule_timeout_interruptible(HZ);
2364	if (time_after(jiffies, lastsleep) && !torture_must_stop()) {
2365	torture_hrtimeout_us(baset_us: 500, fuzzt_ns: 1000, trsp: &rand);
2366	lastsleep = jiffies + 10;
2367	}
2368	while (torture_num_online_cpus() < mynumonline && !torture_must_stop())
2369	schedule_timeout_interruptible(HZ / 5);
2370	stutter_wait(title: "rcu_torture_reader");
2371	} while (!torture_must_stop());
2372	if (irqreader && cur_ops->irq_capable) {
2373	timer_delete_sync(timer: &t);
2374	timer_destroy_on_stack(timer: &t);
2375	}
2376	tick_dep_clear_task(current, bit: TICK_DEP_BIT_RCU);
2377	torture_kthread_stopping(title: "rcu_torture_reader");
2378	return 0;
2379	}
2380
2381	/*
2382	* Randomly Toggle CPUs' callback-offload state. This uses hrtimers to
2383	* increase race probabilities and fuzzes the interval between toggling.
2384	*/
2385	static int rcu_nocb_toggle(void *arg)
2386	{
2387	int cpu;
2388	int maxcpu = -1;
2389	int oldnice = task_nice(current);
2390	long r;
2391	DEFINE_TORTURE_RANDOM(rand);
2392	ktime_t toggle_delay;
2393	unsigned long toggle_fuzz;
2394	ktime_t toggle_interval = ms_to_ktime(ms: nocbs_toggle);
2395
2396	VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started");
2397	while (!rcu_inkernel_boot_has_ended())
2398	schedule_timeout_interruptible(HZ / 10);
2399	for_each_possible_cpu(cpu)
2400	maxcpu = cpu;
2401	WARN_ON(maxcpu < 0);
2402	if (toggle_interval > ULONG_MAX)
2403	toggle_fuzz = ULONG_MAX >> 3;
2404	else
2405	toggle_fuzz = toggle_interval >> 3;
2406	if (toggle_fuzz <= 0)
2407	toggle_fuzz = NSEC_PER_USEC;
2408	do {
2409	r = torture_random(trsp: &rand);
2410	cpu = (r >> 1) % (maxcpu + 1);
2411	if (r & 0x1) {
2412	rcu_nocb_cpu_offload(cpu);
2413	atomic_long_inc(v: &n_nocb_offload);
2414	} else {
2415	rcu_nocb_cpu_deoffload(cpu);
2416	atomic_long_inc(v: &n_nocb_deoffload);
2417	}
2418	toggle_delay = torture_random(trsp: &rand) % toggle_fuzz + toggle_interval;
2419	set_current_state(TASK_INTERRUPTIBLE);
2420	schedule_hrtimeout(expires: &toggle_delay, mode: HRTIMER_MODE_REL);
2421	if (stutter_wait(title: "rcu_nocb_toggle"))
2422	sched_set_normal(current, nice: oldnice);
2423	} while (!torture_must_stop());
2424	torture_kthread_stopping(title: "rcu_nocb_toggle");
2425	return 0;
2426	}
2427
2428	/*
2429	* Print torture statistics. Caller must ensure that there is only
2430	* one call to this function at a given time!!! This is normally
2431	* accomplished by relying on the module system to only have one copy
2432	* of the module loaded, and then by giving the rcu_torture_stats
2433	* kthread full control (or the init/cleanup functions when rcu_torture_stats
2434	* thread is not running).
2435	*/
2436	static void
2437	rcu_torture_stats_print(void)
2438	{
2439	int cpu;
2440	int i;
2441	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
2442	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
2443	long n_gpwraps = 0;
2444	struct rcu_torture *rtcp;
2445	static unsigned long rtcv_snap = ULONG_MAX;
2446	static bool splatted;
2447	struct task_struct *wtp;
2448
2449	for_each_possible_cpu(cpu) {
2450	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2451	pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
2452	batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
2453	}
2454	if (cur_ops->get_gpwrap_count)
2455	n_gpwraps += cur_ops->get_gpwrap_count(cpu);
2456	}
2457	for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) {
2458	if (pipesummary[i] != 0)
2459	break;
2460	}
2461
2462	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2463	rtcp = rcu_access_pointer(rcu_torture_current);
2464	pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
2465	rtcp,
2466	rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
2467	rcu_torture_current_version,
2468	list_empty(&rcu_torture_freelist),
2469	atomic_read(&n_rcu_torture_alloc),
2470	atomic_read(&n_rcu_torture_alloc_fail),
2471	atomic_read(&n_rcu_torture_free));
2472	pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ",
2473	atomic_read(&n_rcu_torture_mberror),
2474	atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries),
2475	n_rcu_torture_barrier_error,
2476	n_rcu_torture_boost_ktrerror);
2477	pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
2478	n_rcu_torture_boost_failure,
2479	n_rcu_torture_boosts,
2480	atomic_long_read(&n_rcu_torture_timers));
2481	torture_onoff_stats();
2482	pr_cont("barrier: %ld/%ld:%ld ",
2483	data_race(n_barrier_successes),
2484	data_race(n_barrier_attempts),
2485	data_race(n_rcu_torture_barrier_error));
2486	pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic.
2487	pr_cont("nocb-toggles: %ld:%ld ",
2488	atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload));
2489	pr_cont("gpwraps: %ld\n", n_gpwraps);
2490
2491	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2492	if (atomic_read(v: &n_rcu_torture_mberror) ||
2493	atomic_read(v: &n_rcu_torture_mbchk_fail) ||
2494	n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror ||
2495	n_rcu_torture_boost_failure || i > 1) {
2496	pr_cont("%s", "!!! ");
2497	atomic_inc(v: &n_rcu_torture_error);
2498	WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror));
2499	WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail));
2500	WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier()
2501	WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread
2502	WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?)
2503	WARN_ON_ONCE(i > 1); // Too-short grace period
2504	}
2505	pr_cont("Reader Pipe: ");
2506	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2507	pr_cont(" %ld", pipesummary[i]);
2508	pr_cont("\n");
2509
2510	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2511	pr_cont("Reader Batch: ");
2512	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2513	pr_cont(" %ld", batchsummary[i]);
2514	pr_cont("\n");
2515
2516	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2517	pr_cont("Free-Block Circulation: ");
2518	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2519	pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
2520	}
2521	pr_cont("\n");
2522
2523	if (cur_ops->stats)
2524	cur_ops->stats();
2525	if (rtcv_snap == rcu_torture_current_version &&
2526	rcu_access_pointer(rcu_torture_current) &&
2527	!rcu_stall_is_suppressed()) {
2528	int __maybe_unused flags = 0;
2529	unsigned long __maybe_unused gp_seq = 0;
2530
2531	if (cur_ops->get_gp_data)
2532	cur_ops->get_gp_data(&flags, &gp_seq);
2533	wtp = READ_ONCE(writer_task);
2534	pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n",
2535	rcu_torture_writer_state_getname(),
2536	rcu_torture_writer_state, gp_seq, flags,
2537	wtp == NULL ? ~0U : wtp->__state,
2538	wtp == NULL ? -1 : (int)task_cpu(wtp));
2539	if (!splatted && wtp) {
2540	sched_show_task(p: wtp);
2541	splatted = true;
2542	}
2543	if (cur_ops->gp_kthread_dbg)
2544	cur_ops->gp_kthread_dbg();
2545	rcu_ftrace_dump(DUMP_ALL);
2546	}
2547	rtcv_snap = rcu_torture_current_version;
2548	}
2549
2550	/*
2551	* Periodically prints torture statistics, if periodic statistics printing
2552	* was specified via the stat_interval module parameter.
2553	*/
2554	static int
2555	rcu_torture_stats(void *arg)
2556	{
2557	VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
2558	do {
2559	schedule_timeout_interruptible(timeout: stat_interval * HZ);
2560	rcu_torture_stats_print();
2561	torture_shutdown_absorb(title: "rcu_torture_stats");
2562	} while (!torture_must_stop());
2563	torture_kthread_stopping(title: "rcu_torture_stats");
2564	return 0;
2565	}
2566
2567	/* Test mem_dump_obj() and friends. */
2568	static void rcu_torture_mem_dump_obj(void)
2569	{
2570	struct rcu_head *rhp;
2571	struct kmem_cache *kcp;
2572	static int z;
2573
2574	kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL);
2575	if (WARN_ON_ONCE(!kcp))
2576	return;
2577	rhp = kmem_cache_alloc(kcp, GFP_KERNEL);
2578	if (WARN_ON_ONCE(!rhp)) {
2579	kmem_cache_destroy(s: kcp);
2580	return;
2581	}
2582	pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z);
2583	pr_alert("mem_dump_obj(ZERO_SIZE_PTR):");
2584	mem_dump_obj(ZERO_SIZE_PTR);
2585	pr_alert("mem_dump_obj(NULL):");
2586	mem_dump_obj(NULL);
2587	pr_alert("mem_dump_obj(%px):", &rhp);
2588	mem_dump_obj(object: &rhp);
2589	pr_alert("mem_dump_obj(%px):", rhp);
2590	mem_dump_obj(object: rhp);
2591	pr_alert("mem_dump_obj(%px):", &rhp->func);
2592	mem_dump_obj(object: &rhp->func);
2593	pr_alert("mem_dump_obj(%px):", &z);
2594	mem_dump_obj(object: &z);
2595	kmem_cache_free(s: kcp, objp: rhp);
2596	kmem_cache_destroy(s: kcp);
2597	rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
2598	if (WARN_ON_ONCE(!rhp))
2599	return;
2600	pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp);
2601	pr_alert("mem_dump_obj(kmalloc %px):", rhp);
2602	mem_dump_obj(object: rhp);
2603	pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func);
2604	mem_dump_obj(object: &rhp->func);
2605	kfree(objp: rhp);
2606	rhp = vmalloc(4096);
2607	if (WARN_ON_ONCE(!rhp))
2608	return;
2609	pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp);
2610	pr_alert("mem_dump_obj(vmalloc %px):", rhp);
2611	mem_dump_obj(object: rhp);
2612	pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func);
2613	mem_dump_obj(object: &rhp->func);
2614	vfree(addr: rhp);
2615	}
2616
2617	static void
2618	rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
2619	{
2620	pr_alert("%s" TORTURE_FLAG
2621	"--- %s: nreaders=%d nfakewriters=%d "
2622	"stat_interval=%d verbose=%d test_no_idle_hz=%d "
2623	"shuffle_interval=%d stutter=%d irqreader=%d "
2624	"fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
2625	"test_boost=%d/%d test_boost_interval=%d "
2626	"test_boost_duration=%d test_boost_holdoff=%d shutdown_secs=%d "
2627	"stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "
2628	"stall_cpu_block=%d stall_cpu_repeat=%d "
2629	"n_barrier_cbs=%d "
2630	"onoff_interval=%d onoff_holdoff=%d "
2631	"read_exit_delay=%d read_exit_burst=%d "
2632	"reader_flavor=%x "
2633	"nocbs_nthreads=%d nocbs_toggle=%d "
2634	"test_nmis=%d "
2635	"preempt_duration=%d preempt_interval=%d\n",
2636	torture_type, tag, nrealreaders, nrealfakewriters,
2637	stat_interval, verbose, test_no_idle_hz, shuffle_interval,
2638	stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
2639	test_boost, cur_ops->can_boost,
2640	test_boost_interval, test_boost_duration, test_boost_holdoff, shutdown_secs,
2641	stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,
2642	stall_cpu_block, stall_cpu_repeat,
2643	n_barrier_cbs,
2644	onoff_interval, onoff_holdoff,
2645	read_exit_delay, read_exit_burst,
2646	reader_flavor,
2647	nocbs_nthreads, nocbs_toggle,
2648	test_nmis,
2649	preempt_duration, preempt_interval);
2650	}
2651
2652	static int rcutorture_booster_cleanup(unsigned int cpu)
2653	{
2654	struct task_struct *t;
2655
2656	if (boost_tasks[cpu] == NULL)
2657	return 0;
2658	mutex_lock(&boost_mutex);
2659	t = boost_tasks[cpu];
2660	boost_tasks[cpu] = NULL;
2661	rcu_torture_enable_rt_throttle();
2662	mutex_unlock(lock: &boost_mutex);
2663
2664	/* This must be outside of the mutex, otherwise deadlock! */
2665	torture_stop_kthread(rcu_torture_boost, t);
2666	return 0;
2667	}
2668
2669	static int rcutorture_booster_init(unsigned int cpu)
2670	{
2671	int retval;
2672
2673	if (boost_tasks[cpu] != NULL)
2674	return 0; /* Already created, nothing more to do. */
2675
2676	// Testing RCU priority boosting requires rcutorture do
2677	// some serious abuse. Counter this by running ksoftirqd
2678	// at higher priority.
2679	if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) {
2680	struct sched_param sp;
2681	struct task_struct *t;
2682
2683	t = per_cpu(ksoftirqd, cpu);
2684	WARN_ON_ONCE(!t);
2685	sp.sched_priority = 2;
2686	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
2687	#ifdef CONFIG_IRQ_FORCED_THREADING
2688	if (force_irqthreads()) {
2689	t = per_cpu(ktimerd, cpu);
2690	WARN_ON_ONCE(!t);
2691	sp.sched_priority = 2;
2692	sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
2693	}
2694	#endif
2695	}
2696
2697	/* Don't allow time recalculation while creating a new task. */
2698	mutex_lock(&boost_mutex);
2699	rcu_torture_disable_rt_throttle();
2700	VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
2701	boost_tasks[cpu] = kthread_run_on_cpu(threadfn: rcu_torture_boost, NULL,
2702	cpu, namefmt: "rcu_torture_boost_%u");
2703	if (IS_ERR(ptr: boost_tasks[cpu])) {
2704	retval = PTR_ERR(ptr: boost_tasks[cpu]);
2705	VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
2706	n_rcu_torture_boost_ktrerror++;
2707	boost_tasks[cpu] = NULL;
2708	mutex_unlock(lock: &boost_mutex);
2709	return retval;
2710	}
2711	mutex_unlock(lock: &boost_mutex);
2712	return 0;
2713	}
2714
2715	static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr)
2716	{
2717	pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr);
2718	return NOTIFY_OK;
2719	}
2720
2721	static struct notifier_block rcu_torture_stall_block = {
2722	.notifier_call = rcu_torture_stall_nf,
2723	};
2724
2725	/*
2726	* CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then
2727	* induces a CPU stall for the time specified by stall_cpu. If a new
2728	* stall test is added, stallsdone in rcu_torture_writer() must be adjusted.
2729	*/
2730	static void rcu_torture_stall_one(int rep, int irqsoff)
2731	{
2732	int idx;
2733	unsigned long stop_at;
2734
2735	if (stall_cpu_holdoff > 0) {
2736	VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
2737	schedule_timeout_interruptible(timeout: stall_cpu_holdoff * HZ);
2738	VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
2739	}
2740	if (!kthread_should_stop() && stall_gp_kthread > 0) {
2741	VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall");
2742	rcu_gp_set_torture_wait(duration: stall_gp_kthread * HZ);
2743	for (idx = 0; idx < stall_gp_kthread + 2; idx++) {
2744	if (kthread_should_stop())
2745	break;
2746	schedule_timeout_uninterruptible(HZ);
2747	}
2748	}
2749	if (!kthread_should_stop() && stall_cpu > 0) {
2750	VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall");
2751	stop_at = ktime_get_seconds() + stall_cpu;
2752	/* RCU CPU stall is expected behavior in following code. */
2753	idx = cur_ops->readlock();
2754	if (irqsoff)
2755	local_irq_disable();
2756	else if (!stall_cpu_block)
2757	preempt_disable();
2758	pr_alert("%s start stall episode %d on CPU %d.\n",
2759	__func__, rep + 1, raw_smp_processor_id());
2760	while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) &&
2761	!kthread_should_stop())
2762	if (stall_cpu_block) {
2763	#ifdef CONFIG_PREEMPTION
2764	preempt_schedule();
2765	#else
2766	schedule_timeout_uninterruptible(HZ);
2767	#endif
2768	} else if (stall_no_softlockup) {
2769	touch_softlockup_watchdog();
2770	}
2771	if (irqsoff)
2772	local_irq_enable();
2773	else if (!stall_cpu_block)
2774	preempt_enable();
2775	cur_ops->readunlock(idx);
2776	}
2777	}
2778
2779	/*
2780	* CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many
2781	* additional times as specified by the stall_cpu_repeat module parameter.
2782	* Note that stall_cpu_irqsoff is ignored on the second and subsequent
2783	* stall.
2784	*/
2785	static int rcu_torture_stall(void *args)
2786	{
2787	int i;
2788	int repeat = stall_cpu_repeat;
2789	int ret;
2790
2791	VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
2792	if (repeat < 0) {
2793	repeat = 0;
2794	WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST));
2795	}
2796	if (rcu_cpu_stall_notifiers) {
2797	ret = rcu_stall_chain_notifier_register(n: &rcu_torture_stall_block);
2798	if (ret)
2799	pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n",
2800	__func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : "");
2801	}
2802	for (i = 0; i <= repeat; i++) {
2803	if (kthread_should_stop())
2804	break;
2805	rcu_torture_stall_one(rep: i, irqsoff: i == 0 ? stall_cpu_irqsoff : 0);
2806	}
2807	pr_alert("%s end.\n", __func__);
2808	if (rcu_cpu_stall_notifiers && !ret) {
2809	ret = rcu_stall_chain_notifier_unregister(n: &rcu_torture_stall_block);
2810	if (ret)
2811	pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret);
2812	}
2813	torture_shutdown_absorb(title: "rcu_torture_stall");
2814	while (!kthread_should_stop())
2815	schedule_timeout_interruptible(timeout: 10 * HZ);
2816	return 0;
2817	}
2818
2819	/* Spawn CPU-stall kthread, if stall_cpu specified. */
2820	static int __init rcu_torture_stall_init(void)
2821	{
2822	if (stall_cpu <= 0 && stall_gp_kthread <= 0)
2823	return 0;
2824	return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
2825	}
2826
2827	/* State structure for forward-progress self-propagating RCU callback. */
2828	struct fwd_cb_state {
2829	struct rcu_head rh;
2830	int stop;
2831	};
2832
2833	/*
2834	* Forward-progress self-propagating RCU callback function. Because
2835	* callbacks run from softirq, this function is an implicit RCU read-side
2836	* critical section.
2837	*/
2838	static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp)
2839	{
2840	struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh);
2841
2842	if (READ_ONCE(fcsp->stop)) {
2843	WRITE_ONCE(fcsp->stop, 2);
2844	return;
2845	}
2846	cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);
2847	}
2848
2849	/* State for continuous-flood RCU callbacks. */
2850	struct rcu_fwd_cb {
2851	struct rcu_head rh;
2852	struct rcu_fwd_cb *rfc_next;
2853	struct rcu_fwd *rfc_rfp;
2854	int rfc_gps;
2855	};
2856
2857	#define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */
2858	#define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */
2859	#define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */
2860	#define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */
2861	#define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV))
2862
2863	struct rcu_launder_hist {
2864	long n_launders;
2865	unsigned long launder_gp_seq;
2866	};
2867
2868	struct rcu_fwd {
2869	spinlock_t rcu_fwd_lock;
2870	struct rcu_fwd_cb *rcu_fwd_cb_head;
2871	struct rcu_fwd_cb **rcu_fwd_cb_tail;
2872	long n_launders_cb;
2873	unsigned long rcu_fwd_startat;
2874	struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST];
2875	unsigned long rcu_launder_gp_seq_start;
2876	int rcu_fwd_id;
2877	};
2878
2879	static DEFINE_MUTEX(rcu_fwd_mutex);
2880	static struct rcu_fwd *rcu_fwds;
2881	static unsigned long rcu_fwd_seq;
2882	static atomic_long_t rcu_fwd_max_cbs;
2883	static bool rcu_fwd_emergency_stop;
2884
2885	static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)
2886	{
2887	unsigned long gps;
2888	unsigned long gps_old;
2889	int i;
2890	int j;
2891
2892	for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)
2893	if (rfp->n_launders_hist[i].n_launders > 0)
2894	break;
2895	pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):",
2896	__func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat);
2897	gps_old = rfp->rcu_launder_gp_seq_start;
2898	for (j = 0; j <= i; j++) {
2899	gps = rfp->n_launders_hist[j].launder_gp_seq;
2900	pr_cont(" %ds/%d: %ld:%ld",
2901	j + 1, FWD_CBS_HIST_DIV,
2902	rfp->n_launders_hist[j].n_launders,
2903	rcutorture_seq_diff(gps, gps_old));
2904	gps_old = gps;
2905	}
2906	pr_cont("\n");
2907	}
2908
2909	/* Callback function for continuous-flood RCU callbacks. */
2910	static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
2911	{
2912	unsigned long flags;
2913	int i;
2914	struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh);
2915	struct rcu_fwd_cb **rfcpp;
2916	struct rcu_fwd *rfp = rfcp->rfc_rfp;
2917
2918	rfcp->rfc_next = NULL;
2919	rfcp->rfc_gps++;
2920	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
2921	rfcpp = rfp->rcu_fwd_cb_tail;
2922	rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;
2923	smp_store_release(rfcpp, rfcp);
2924	WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);
2925	i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));
2926	if (i >= ARRAY_SIZE(rfp->n_launders_hist))
2927	i = ARRAY_SIZE(rfp->n_launders_hist) - 1;
2928	rfp->n_launders_hist[i].n_launders++;
2929	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
2930	spin_unlock_irqrestore(lock: &rfp->rcu_fwd_lock, flags);
2931	}
2932
2933	// Give the scheduler a chance, even on nohz_full CPUs.
2934	static void rcu_torture_fwd_prog_cond_resched(unsigned long iter)
2935	{
2936	if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) {
2937	// Real call_rcu() floods hit userspace, so emulate that.
2938	if (need_resched() || (iter & 0xfff))
2939	schedule();
2940	return;
2941	}
2942	// No userspace emulation: CB invocation throttles call_rcu()
2943	cond_resched();
2944	}
2945
2946	/*
2947	* Free all callbacks on the rcu_fwd_cb_head list, either because the
2948	* test is over or because we hit an OOM event.
2949	*/
2950	static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)
2951	{
2952	unsigned long flags;
2953	unsigned long freed = 0;
2954	struct rcu_fwd_cb *rfcp;
2955
2956	for (;;) {
2957	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
2958	rfcp = rfp->rcu_fwd_cb_head;
2959	if (!rfcp) {
2960	spin_unlock_irqrestore(lock: &rfp->rcu_fwd_lock, flags);
2961	break;
2962	}
2963	rfp->rcu_fwd_cb_head = rfcp->rfc_next;
2964	if (!rfp->rcu_fwd_cb_head)
2965	rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;
2966	spin_unlock_irqrestore(lock: &rfp->rcu_fwd_lock, flags);
2967	kfree(objp: rfcp);
2968	freed++;
2969	rcu_torture_fwd_prog_cond_resched(iter: freed);
2970	if (tick_nohz_full_enabled()) {
2971	local_irq_save(flags);
2972	rcu_momentary_eqs();
2973	local_irq_restore(flags);
2974	}
2975	}
2976	return freed;
2977	}
2978
2979	/* Carry out need_resched()/cond_resched() forward-progress testing. */
2980	static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,
2981	int *tested, int *tested_tries)
2982	{
2983	unsigned long cver;
2984	unsigned long dur;
2985	struct fwd_cb_state fcs;
2986	unsigned long gps;
2987	int idx;
2988	int sd;
2989	int sd4;
2990	bool selfpropcb = false;
2991	unsigned long stopat;
2992	static DEFINE_TORTURE_RANDOM(trs);
2993
2994	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
2995	if (!cur_ops->sync)
2996	return; // Cannot do need_resched() forward progress testing without ->sync.
2997	if (cur_ops->call && cur_ops->cb_barrier) {
2998	init_rcu_head_on_stack(head: &fcs.rh);
2999	selfpropcb = true;
3000	}
3001
3002	/* Tight loop containing cond_resched(). */
3003	atomic_inc(v: &rcu_fwd_cb_nodelay);
3004	cur_ops->sync(); /* Later readers see above write. */
3005	if (selfpropcb) {
3006	WRITE_ONCE(fcs.stop, 0);
3007	cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
3008	}
3009	cver = READ_ONCE(rcu_torture_current_version);
3010	gps = cur_ops->get_gp_seq();
3011	sd = cur_ops->stall_dur() + 1;
3012	sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
3013	dur = sd4 + torture_random(trsp: &trs) % (sd - sd4);
3014	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
3015	stopat = rfp->rcu_fwd_startat + dur;
3016	while (time_before(jiffies, stopat) &&
3017	!shutdown_time_arrived() &&
3018	!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
3019	idx = cur_ops->readlock();
3020	udelay(usec: 10);
3021	cur_ops->readunlock(idx);
3022	if (!fwd_progress_need_resched || need_resched())
3023	cond_resched();
3024	}
3025	(*tested_tries)++;
3026	if (!time_before(jiffies, stopat) &&
3027	!shutdown_time_arrived() &&
3028	!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
3029	(*tested)++;
3030	cver = READ_ONCE(rcu_torture_current_version) - cver;
3031	gps = rcutorture_seq_diff(new: cur_ops->get_gp_seq(), old: gps);
3032	WARN_ON(!cver && gps < 2);
3033	pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__,
3034	rfp->rcu_fwd_id, dur, cver, gps);
3035	}
3036	if (selfpropcb) {
3037	WRITE_ONCE(fcs.stop, 1);
3038	cur_ops->sync(); /* Wait for running CB to complete. */
3039	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3040	cur_ops->cb_barrier(); /* Wait for queued callbacks. */
3041	}
3042
3043	if (selfpropcb) {
3044	WARN_ON(READ_ONCE(fcs.stop) != 2);
3045	destroy_rcu_head_on_stack(head: &fcs.rh);
3046	}
3047	schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */
3048	atomic_dec(v: &rcu_fwd_cb_nodelay);
3049	}
3050
3051	/* Carry out call_rcu() forward-progress testing. */
3052	static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)
3053	{
3054	unsigned long cver;
3055	unsigned long flags;
3056	unsigned long gps;
3057	int i;
3058	long n_launders;
3059	long n_launders_cb_snap;
3060	long n_launders_sa;
3061	long n_max_cbs;
3062	long n_max_gps;
3063	struct rcu_fwd_cb *rfcp;
3064	struct rcu_fwd_cb *rfcpn;
3065	unsigned long stopat;
3066	unsigned long stoppedat;
3067
3068	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3069	if (READ_ONCE(rcu_fwd_emergency_stop))
3070	return; /* Get out of the way quickly, no GP wait! */
3071	if (!cur_ops->call)
3072	return; /* Can't do call_rcu() fwd prog without ->call. */
3073
3074	/* Loop continuously posting RCU callbacks. */
3075	atomic_inc(v: &rcu_fwd_cb_nodelay);
3076	cur_ops->sync(); /* Later readers see above write. */
3077	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
3078	stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;
3079	n_launders = 0;
3080	rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread
3081	n_launders_sa = 0;
3082	n_max_cbs = 0;
3083	n_max_gps = 0;
3084	for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)
3085	rfp->n_launders_hist[i].n_launders = 0;
3086	cver = READ_ONCE(rcu_torture_current_version);
3087	gps = cur_ops->get_gp_seq();
3088	rfp->rcu_launder_gp_seq_start = gps;
3089	tick_dep_set_task(current, bit: TICK_DEP_BIT_RCU); // CPU bound, so need tick.
3090	while (time_before(jiffies, stopat) &&
3091	!shutdown_time_arrived() &&
3092	!READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
3093	rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);
3094	rfcpn = NULL;
3095	if (rfcp)
3096	rfcpn = READ_ONCE(rfcp->rfc_next);
3097	if (rfcpn) {
3098	if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
3099	++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
3100	break;
3101	rfp->rcu_fwd_cb_head = rfcpn;
3102	n_launders++;
3103	n_launders_sa++;
3104	} else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {
3105	rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);
3106	if (WARN_ON_ONCE(!rfcp)) {
3107	schedule_timeout_interruptible(timeout: 1);
3108	continue;
3109	}
3110	n_max_cbs++;
3111	n_launders_sa = 0;
3112	rfcp->rfc_gps = 0;
3113	rfcp->rfc_rfp = rfp;
3114	} else {
3115	rfcp = NULL;
3116	}
3117	if (rfcp)
3118	cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
3119	rcu_torture_fwd_prog_cond_resched(iter: n_launders + n_max_cbs);
3120	if (tick_nohz_full_enabled()) {
3121	local_irq_save(flags);
3122	rcu_momentary_eqs();
3123	local_irq_restore(flags);
3124	}
3125	}
3126	stoppedat = jiffies;
3127	n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);
3128	cver = READ_ONCE(rcu_torture_current_version) - cver;
3129	gps = rcutorture_seq_diff(new: cur_ops->get_gp_seq(), old: gps);
3130	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3131	cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
3132	(void)rcu_torture_fwd_prog_cbfree(rfp);
3133
3134	if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) &&
3135	!shutdown_time_arrived()) {
3136	if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg)
3137	cur_ops->gp_kthread_dbg();
3138	pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n",
3139	__func__,
3140	stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,
3141	n_launders + n_max_cbs - n_launders_cb_snap,
3142	n_launders, n_launders_sa,
3143	n_max_gps, n_max_cbs, cver, gps, num_online_cpus());
3144	atomic_long_add(i: n_max_cbs, v: &rcu_fwd_max_cbs);
3145	mutex_lock(&rcu_fwd_mutex); // Serialize histograms.
3146	rcu_torture_fwd_cb_hist(rfp);
3147	mutex_unlock(lock: &rcu_fwd_mutex);
3148	}
3149	schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */
3150	tick_dep_clear_task(current, bit: TICK_DEP_BIT_RCU);
3151	atomic_dec(v: &rcu_fwd_cb_nodelay);
3152	}
3153
3154
3155	/*
3156	* OOM notifier, but this only prints diagnostic information for the
3157	* current forward-progress test.
3158	*/
3159	static int rcutorture_oom_notify(struct notifier_block *self,
3160	unsigned long notused, void *nfreed)
3161	{
3162	int i;
3163	long ncbs;
3164	struct rcu_fwd *rfp;
3165
3166	mutex_lock(&rcu_fwd_mutex);
3167	rfp = rcu_fwds;
3168	if (!rfp) {
3169	mutex_unlock(lock: &rcu_fwd_mutex);
3170	return NOTIFY_OK;
3171	}
3172	WARN(1, "%s invoked upon OOM during forward-progress testing.\n",
3173	__func__);
3174	for (i = 0; i < fwd_progress; i++) {
3175	rcu_torture_fwd_cb_hist(rfp: &rfp[i]);
3176	rcu_fwd_progress_check(j: 1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2);
3177	}
3178	WRITE_ONCE(rcu_fwd_emergency_stop, true);
3179	smp_mb(); /* Emergency stop before free and wait to avoid hangs. */
3180	ncbs = 0;
3181	for (i = 0; i < fwd_progress; i++)
3182	ncbs += rcu_torture_fwd_prog_cbfree(rfp: &rfp[i]);
3183	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
3184	cur_ops->cb_barrier();
3185	ncbs = 0;
3186	for (i = 0; i < fwd_progress; i++)
3187	ncbs += rcu_torture_fwd_prog_cbfree(rfp: &rfp[i]);
3188	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
3189	cur_ops->cb_barrier();
3190	ncbs = 0;
3191	for (i = 0; i < fwd_progress; i++)
3192	ncbs += rcu_torture_fwd_prog_cbfree(rfp: &rfp[i]);
3193	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
3194	smp_mb(); /* Frees before return to avoid redoing OOM. */
3195	(*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */
3196	pr_info("%s returning after OOM processing.\n", __func__);
3197	mutex_unlock(lock: &rcu_fwd_mutex);
3198	return NOTIFY_OK;
3199	}
3200
3201	static struct notifier_block rcutorture_oom_nb = {
3202	.notifier_call = rcutorture_oom_notify
3203	};
3204
3205	/* Carry out grace-period forward-progress testing. */
3206	static int rcu_torture_fwd_prog(void *args)
3207	{
3208	bool firsttime = true;
3209	long max_cbs;
3210	int oldnice = task_nice(current);
3211	unsigned long oldseq = READ_ONCE(rcu_fwd_seq);
3212	struct rcu_fwd *rfp = args;
3213	int tested = 0;
3214	int tested_tries = 0;
3215
3216	VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started");
3217	rcu_bind_current_to_nocb();
3218	if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
3219	set_user_nice(current, MAX_NICE);
3220	do {
3221	if (!rfp->rcu_fwd_id) {
3222	schedule_timeout_interruptible(timeout: fwd_progress_holdoff * HZ);
3223	WRITE_ONCE(rcu_fwd_emergency_stop, false);
3224	if (!firsttime) {
3225	max_cbs = atomic_long_xchg(v: &rcu_fwd_max_cbs, new: 0);
3226	pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs);
3227	}
3228	firsttime = false;
3229	WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1);
3230	} else {
3231	while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop())
3232	schedule_timeout_interruptible(HZ / 20);
3233	oldseq = READ_ONCE(rcu_fwd_seq);
3234	}
3235	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3236	if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)
3237	rcu_torture_fwd_prog_cr(rfp);
3238	if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&
3239	(!IS_ENABLED(CONFIG_TINY_RCU) ||
3240	(rcu_inkernel_boot_has_ended() &&
3241	torture_num_online_cpus() > rfp->rcu_fwd_id)))
3242	rcu_torture_fwd_prog_nr(rfp, tested: &tested, tested_tries: &tested_tries);
3243
3244	/* Avoid slow periods, better to test when busy. */
3245	if (stutter_wait(title: "rcu_torture_fwd_prog"))
3246	sched_set_normal(current, nice: oldnice);
3247	} while (!torture_must_stop());
3248	/* Short runs might not contain a valid forward-progress attempt. */
3249	if (!rfp->rcu_fwd_id) {
3250	WARN_ON(!tested && tested_tries >= 5);
3251	pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
3252	}
3253	torture_kthread_stopping(title: "rcu_torture_fwd_prog");
3254	return 0;
3255	}
3256
3257	/* If forward-progress checking is requested and feasible, spawn the thread. */
3258	static int __init rcu_torture_fwd_prog_init(void)
3259	{
3260	int i;
3261	int ret = 0;
3262	struct rcu_fwd *rfp;
3263
3264	if (!fwd_progress)
3265	return 0; /* Not requested, so don't do it. */
3266	if (fwd_progress >= nr_cpu_ids) {
3267	VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n");
3268	fwd_progress = nr_cpu_ids;
3269	} else if (fwd_progress < 0) {
3270	fwd_progress = nr_cpu_ids;
3271	}
3272	if ((!cur_ops->sync && !cur_ops->call) ||
3273	(!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||
3274	cur_ops == &rcu_busted_ops) {
3275	VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test");
3276	fwd_progress = 0;
3277	return 0;
3278	}
3279	if (stall_cpu > 0 || (preempt_duration > 0 && IS_ENABLED(CONFIG_RCU_NOCB_CPU))) {
3280	VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall and/or preemption testing");
3281	fwd_progress = 0;
3282	if (IS_MODULE(CONFIG_RCU_TORTURE_TEST))
3283	return -EINVAL; /* In module, can fail back to user. */
3284	WARN_ON(1); /* Make sure rcutorture scripting notices conflict. */
3285	return 0;
3286	}
3287	if (fwd_progress_holdoff <= 0)
3288	fwd_progress_holdoff = 1;
3289	if (fwd_progress_div <= 0)
3290	fwd_progress_div = 4;
3291	rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL);
3292	fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL);
3293	if (!rfp || !fwd_prog_tasks) {
3294	kfree(objp: rfp);
3295	kfree(objp: fwd_prog_tasks);
3296	fwd_prog_tasks = NULL;
3297	fwd_progress = 0;
3298	return -ENOMEM;
3299	}
3300	for (i = 0; i < fwd_progress; i++) {
3301	spin_lock_init(&rfp[i].rcu_fwd_lock);
3302	rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head;
3303	rfp[i].rcu_fwd_id = i;
3304	}
3305	mutex_lock(&rcu_fwd_mutex);
3306	rcu_fwds = rfp;
3307	mutex_unlock(lock: &rcu_fwd_mutex);
3308	register_oom_notifier(nb: &rcutorture_oom_nb);
3309	for (i = 0; i < fwd_progress; i++) {
3310	ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]);
3311	if (ret) {
3312	fwd_progress = i;
3313	return ret;
3314	}
3315	}
3316	return 0;
3317	}
3318
3319	static void rcu_torture_fwd_prog_cleanup(void)
3320	{
3321	int i;
3322	struct rcu_fwd *rfp;
3323
3324	if (!rcu_fwds || !fwd_prog_tasks)
3325	return;
3326	for (i = 0; i < fwd_progress; i++)
3327	torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]);
3328	unregister_oom_notifier(nb: &rcutorture_oom_nb);
3329	mutex_lock(&rcu_fwd_mutex);
3330	rfp = rcu_fwds;
3331	rcu_fwds = NULL;
3332	mutex_unlock(lock: &rcu_fwd_mutex);
3333	kfree(objp: rfp);
3334	kfree(objp: fwd_prog_tasks);
3335	fwd_prog_tasks = NULL;
3336	}
3337
3338	/* Callback function for RCU barrier testing. */
3339	static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
3340	{
3341	atomic_inc(v: &barrier_cbs_invoked);
3342	}
3343
3344	/* IPI handler to get callback posted on desired CPU, if online. */
3345	static int rcu_torture_barrier1cb(void *rcu_void)
3346	{
3347	struct rcu_head *rhp = rcu_void;
3348
3349	cur_ops->call(rhp, rcu_torture_barrier_cbf);
3350	return 0;
3351	}
3352
3353	/* kthread function to register callbacks used to test RCU barriers. */
3354	static int rcu_torture_barrier_cbs(void *arg)
3355	{
3356	long myid = (long)arg;
3357	bool lastphase = false;
3358	bool newphase;
3359	struct rcu_head rcu;
3360
3361	init_rcu_head_on_stack(head: &rcu);
3362	VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
3363	set_user_nice(current, MAX_NICE);
3364	do {
3365	wait_event(barrier_cbs_wq[myid],
3366	(newphase =
3367	smp_load_acquire(&barrier_phase)) != lastphase ||
3368	torture_must_stop());
3369	lastphase = newphase;
3370	if (torture_must_stop())
3371	break;
3372	/*
3373	* The above smp_load_acquire() ensures barrier_phase load
3374	* is ordered before the following ->call().
3375	*/
3376	if (smp_call_on_cpu(cpu: myid, func: rcu_torture_barrier1cb, par: &rcu, phys: 1))
3377	cur_ops->call(&rcu, rcu_torture_barrier_cbf);
3378
3379	if (atomic_dec_and_test(v: &barrier_cbs_count))
3380	wake_up(&barrier_wq);
3381	} while (!torture_must_stop());
3382	if (cur_ops->cb_barrier != NULL)
3383	cur_ops->cb_barrier();
3384	destroy_rcu_head_on_stack(head: &rcu);
3385	torture_kthread_stopping(title: "rcu_torture_barrier_cbs");
3386	return 0;
3387	}
3388
3389	/* kthread function to drive and coordinate RCU barrier testing. */
3390	static int rcu_torture_barrier(void *arg)
3391	{
3392	int i;
3393
3394	VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
3395	do {
3396	atomic_set(v: &barrier_cbs_invoked, i: 0);
3397	atomic_set(v: &barrier_cbs_count, i: n_barrier_cbs);
3398	/* Ensure barrier_phase ordered after prior assignments. */
3399	smp_store_release(&barrier_phase, !barrier_phase);
3400	for (i = 0; i < n_barrier_cbs; i++)
3401	wake_up(&barrier_cbs_wq[i]);
3402	wait_event(barrier_wq,
3403	atomic_read(&barrier_cbs_count) == 0 ||
3404	torture_must_stop());
3405	if (torture_must_stop())
3406	break;
3407	n_barrier_attempts++;
3408	cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
3409	if (atomic_read(v: &barrier_cbs_invoked) != n_barrier_cbs) {
3410	n_rcu_torture_barrier_error++;
3411	pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
3412	atomic_read(&barrier_cbs_invoked),
3413	n_barrier_cbs);
3414	WARN_ON(1);
3415	// Wait manually for the remaining callbacks
3416	i = 0;
3417	do {
3418	if (WARN_ON(i++ > HZ))
3419	i = INT_MIN;
3420	schedule_timeout_interruptible(timeout: 1);
3421	cur_ops->cb_barrier();
3422	} while (atomic_read(v: &barrier_cbs_invoked) !=
3423	n_barrier_cbs &&
3424	!torture_must_stop());
3425	smp_mb(); // Can't trust ordering if broken.
3426	if (!torture_must_stop())
3427	pr_err("Recovered: barrier_cbs_invoked = %d\n",
3428	atomic_read(&barrier_cbs_invoked));
3429	} else {
3430	n_barrier_successes++;
3431	}
3432	schedule_timeout_interruptible(HZ / 10);
3433	} while (!torture_must_stop());
3434	torture_kthread_stopping(title: "rcu_torture_barrier");
3435	return 0;
3436	}
3437
3438	/* Initialize RCU barrier testing. */
3439	static int rcu_torture_barrier_init(void)
3440	{
3441	int i;
3442	int ret;
3443
3444	if (n_barrier_cbs <= 0)
3445	return 0;
3446	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
3447	pr_alert("%s" TORTURE_FLAG
3448	" Call or barrier ops missing for %s,\n",
3449	torture_type, cur_ops->name);
3450	pr_alert("%s" TORTURE_FLAG
3451	" RCU barrier testing omitted from run.\n",
3452	torture_type);
3453	return 0;
3454	}
3455	atomic_set(v: &barrier_cbs_count, i: 0);
3456	atomic_set(v: &barrier_cbs_invoked, i: 0);
3457	barrier_cbs_tasks =
3458	kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]),
3459	GFP_KERNEL);
3460	barrier_cbs_wq =
3461	kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL);
3462	if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
3463	return -ENOMEM;
3464	for (i = 0; i < n_barrier_cbs; i++) {
3465	init_waitqueue_head(&barrier_cbs_wq[i]);
3466	ret = torture_create_kthread(rcu_torture_barrier_cbs,
3467	(void *)(long)i,
3468	barrier_cbs_tasks[i]);
3469	if (ret)
3470	return ret;
3471	}
3472	return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
3473	}
3474
3475	/* Clean up after RCU barrier testing. */
3476	static void rcu_torture_barrier_cleanup(void)
3477	{
3478	int i;
3479
3480	torture_stop_kthread(rcu_torture_barrier, barrier_task);
3481	if (barrier_cbs_tasks != NULL) {
3482	for (i = 0; i < n_barrier_cbs; i++)
3483	torture_stop_kthread(rcu_torture_barrier_cbs,
3484	barrier_cbs_tasks[i]);
3485	kfree(objp: barrier_cbs_tasks);
3486	barrier_cbs_tasks = NULL;
3487	}
3488	if (barrier_cbs_wq != NULL) {
3489	kfree(objp: barrier_cbs_wq);
3490	barrier_cbs_wq = NULL;
3491	}
3492	}
3493
3494	static bool rcu_torture_can_boost(void)
3495	{
3496	static int boost_warn_once;
3497	int prio;
3498
3499	if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)
3500	return false;
3501	if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)
3502	return false;
3503
3504	prio = rcu_get_gp_kthreads_prio();
3505	if (!prio)
3506	return false;
3507
3508	if (prio < 2) {
3509	if (boost_warn_once == 1)
3510	return false;
3511
3512	pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME);
3513	boost_warn_once = 1;
3514	return false;
3515	}
3516
3517	return true;
3518	}
3519
3520	static bool read_exit_child_stop;
3521	static bool read_exit_child_stopped;
3522	static wait_queue_head_t read_exit_wq;
3523
3524	// Child kthread which just does an rcutorture reader and exits.
3525	static int rcu_torture_read_exit_child(void *trsp_in)
3526	{
3527	struct torture_random_state *trsp = trsp_in;
3528
3529	set_user_nice(current, MAX_NICE);
3530	// Minimize time between reading and exiting.
3531	while (!kthread_should_stop())
3532	schedule_timeout_uninterruptible(HZ / 20);
3533	(void)rcu_torture_one_read(trsp, myid: -1);
3534	return 0;
3535	}
3536
3537	// Parent kthread which creates and destroys read-exit child kthreads.
3538	static int rcu_torture_read_exit(void *unused)
3539	{
3540	bool errexit = false;
3541	int i;
3542	struct task_struct *tsp;
3543	DEFINE_TORTURE_RANDOM(trs);
3544
3545	// Allocate and initialize.
3546	set_user_nice(current, MAX_NICE);
3547	VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test");
3548
3549	// Each pass through this loop does one read-exit episode.
3550	do {
3551	VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode");
3552	for (i = 0; i < read_exit_burst; i++) {
3553	if (READ_ONCE(read_exit_child_stop))
3554	break;
3555	stutter_wait(title: "rcu_torture_read_exit");
3556	// Spawn child.
3557	tsp = kthread_run(rcu_torture_read_exit_child,
3558	&trs, "%s", "rcu_torture_read_exit_child");
3559	if (IS_ERR(ptr: tsp)) {
3560	TOROUT_ERRSTRING("out of memory");
3561	errexit = true;
3562	break;
3563	}
3564	cond_resched();
3565	kthread_stop(k: tsp);
3566	n_read_exits++;
3567	}
3568	VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode");
3569	rcu_barrier(); // Wait for task_struct free, avoid OOM.
3570	i = 0;
3571	for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++)
3572	schedule_timeout_uninterruptible(HZ);
3573	} while (!errexit && !READ_ONCE(read_exit_child_stop));
3574
3575	// Clean up and exit.
3576	smp_store_release(&read_exit_child_stopped, true); // After reaping.
3577	smp_mb(); // Store before wakeup.
3578	wake_up(&read_exit_wq);
3579	while (!torture_must_stop())
3580	schedule_timeout_uninterruptible(HZ / 20);
3581	torture_kthread_stopping(title: "rcu_torture_read_exit");
3582	return 0;
3583	}
3584
3585	static int rcu_torture_read_exit_init(void)
3586	{
3587	if (read_exit_burst <= 0)
3588	return 0;
3589	init_waitqueue_head(&read_exit_wq);
3590	read_exit_child_stop = false;
3591	read_exit_child_stopped = false;
3592	return torture_create_kthread(rcu_torture_read_exit, NULL,
3593	read_exit_task);
3594	}
3595
3596	static void rcu_torture_read_exit_cleanup(void)
3597	{
3598	if (!read_exit_task)
3599	return;
3600	WRITE_ONCE(read_exit_child_stop, true);
3601	smp_mb(); // Above write before wait.
3602	wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped));
3603	torture_stop_kthread(rcutorture_read_exit, read_exit_task);
3604	}
3605
3606	static void rcutorture_test_nmis(int n)
3607	{
3608	#if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)
3609	int cpu;
3610	int dumpcpu;
3611	int i;
3612
3613	for (i = 0; i < n; i++) {
3614	preempt_disable();
3615	cpu = smp_processor_id();
3616	dumpcpu = cpu + 1;
3617	if (dumpcpu >= nr_cpu_ids)
3618	dumpcpu = 0;
3619	pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu);
3620	dump_cpu_task(cpu: dumpcpu);
3621	preempt_enable();
3622	schedule_timeout_uninterruptible(timeout: 15 * HZ);
3623	}
3624	#else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)
3625	WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis);
3626	#endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)
3627	}
3628
3629	// Randomly preempt online CPUs.
3630	static int rcu_torture_preempt(void *unused)
3631	{
3632	int cpu = -1;
3633	DEFINE_TORTURE_RANDOM(rand);
3634
3635	schedule_timeout_idle(timeout: stall_cpu_holdoff);
3636	do {
3637	// Wait for preempt_interval ms with up to 100us fuzz.
3638	torture_hrtimeout_ms(baset_ms: preempt_interval, fuzzt_us: 100, trsp: &rand);
3639	// Select online CPU.
3640	cpu = cpumask_next(n: cpu, cpu_online_mask);
3641	if (cpu >= nr_cpu_ids)
3642	cpu = cpumask_next(n: -1, cpu_online_mask);
3643	WARN_ON_ONCE(cpu >= nr_cpu_ids);
3644	// Move to that CPU, if can't do so, retry later.
3645	if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), dowarn: false))
3646	continue;
3647	// Preempt at high-ish priority, then reset to normal.
3648	sched_set_fifo(current);
3649	torture_sched_setaffinity(current->pid, cpu_present_mask, dowarn: true);
3650	mdelay(preempt_duration);
3651	sched_set_normal(current, nice: 0);
3652	stutter_wait(title: "rcu_torture_preempt");
3653	} while (!torture_must_stop());
3654	torture_kthread_stopping(title: "rcu_torture_preempt");
3655	return 0;
3656	}
3657
3658	static enum cpuhp_state rcutor_hp;
3659
3660	static struct hrtimer gpwrap_lag_timer;
3661	static bool gpwrap_lag_active;
3662
3663	/* Timer handler for toggling RCU grace-period sequence overflow test lag value */
3664	static enum hrtimer_restart rcu_gpwrap_lag_timer(struct hrtimer *timer)
3665	{
3666	ktime_t next_delay;
3667
3668	if (gpwrap_lag_active) {
3669	pr_alert("rcu-torture: Disabling gpwrap lag (value=0)\n");
3670	cur_ops->set_gpwrap_lag(0);
3671	gpwrap_lag_active = false;
3672	next_delay = ktime_set(secs: (gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, nsecs: 0);
3673	} else {
3674	pr_alert("rcu-torture: Enabling gpwrap lag (value=%d)\n", gpwrap_lag_gps);
3675	cur_ops->set_gpwrap_lag(gpwrap_lag_gps);
3676	gpwrap_lag_active = true;
3677	next_delay = ktime_set(secs: gpwrap_lag_active_mins * 60, nsecs: 0);
3678	}
3679
3680	if (torture_must_stop_irq())
3681	return HRTIMER_NORESTART;
3682
3683	hrtimer_forward_now(timer, interval: next_delay);
3684	return HRTIMER_RESTART;
3685	}
3686
3687	static int rcu_gpwrap_lag_init(void)
3688	{
3689	if (!gpwrap_lag)
3690	return 0;
3691
3692	if (gpwrap_lag_cycle_mins <= 0 || gpwrap_lag_active_mins <= 0) {
3693	pr_alert("rcu-torture: lag timing parameters must be positive\n");
3694	return -EINVAL;
3695	}
3696
3697	hrtimer_setup(timer: &gpwrap_lag_timer, function: rcu_gpwrap_lag_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
3698	gpwrap_lag_active = false;
3699	hrtimer_start(timer: &gpwrap_lag_timer,
3700	tim: ktime_set(secs: (gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, nsecs: 0), mode: HRTIMER_MODE_REL);
3701
3702	return 0;
3703	}
3704
3705	static void rcu_gpwrap_lag_cleanup(void)
3706	{
3707	hrtimer_cancel(timer: &gpwrap_lag_timer);
3708	cur_ops->set_gpwrap_lag(0);
3709	gpwrap_lag_active = false;
3710	}
3711	static void
3712	rcu_torture_cleanup(void)
3713	{
3714	int firsttime;
3715	int flags = 0;
3716	unsigned long gp_seq = 0;
3717	int i;
3718	int j;
3719
3720	if (torture_cleanup_begin()) {
3721	if (cur_ops->cb_barrier != NULL) {
3722	pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
3723	cur_ops->cb_barrier();
3724	}
3725	if (cur_ops->gp_slow_unregister)
3726	cur_ops->gp_slow_unregister(NULL);
3727	return;
3728	}
3729	if (!cur_ops) {
3730	torture_cleanup_end();
3731	return;
3732	}
3733
3734	rcutorture_test_nmis(n: test_nmis);
3735
3736	if (cur_ops->gp_kthread_dbg)
3737	cur_ops->gp_kthread_dbg();
3738	torture_stop_kthread(rcu_torture_preempt, preempt_task);
3739	rcu_torture_read_exit_cleanup();
3740	rcu_torture_barrier_cleanup();
3741	rcu_torture_fwd_prog_cleanup();
3742	torture_stop_kthread(rcu_torture_stall, stall_task);
3743	torture_stop_kthread(rcu_torture_writer, writer_task);
3744
3745	if (nocb_tasks) {
3746	for (i = 0; i < nrealnocbers; i++)
3747	torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]);
3748	kfree(objp: nocb_tasks);
3749	nocb_tasks = NULL;
3750	}
3751
3752	if (reader_tasks) {
3753	for (i = 0; i < nrealreaders; i++)
3754	torture_stop_kthread(rcu_torture_reader,
3755	reader_tasks[i]);
3756	kfree(objp: reader_tasks);
3757	reader_tasks = NULL;
3758	}
3759	kfree(objp: rcu_torture_reader_mbchk);
3760	rcu_torture_reader_mbchk = NULL;
3761
3762	if (fakewriter_tasks) {
3763	for (i = 0; i < nrealfakewriters; i++)
3764	torture_stop_kthread(rcu_torture_fakewriter,
3765	fakewriter_tasks[i]);
3766	kfree(objp: fakewriter_tasks);
3767	fakewriter_tasks = NULL;
3768	}
3769
3770	if (cur_ops->get_gp_data)
3771	cur_ops->get_gp_data(&flags, &gp_seq);
3772	pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n",
3773	cur_ops->name, (long)gp_seq, flags,
3774	rcutorture_seq_diff(gp_seq, start_gp_seq));
3775	torture_stop_kthread(rcu_torture_stats, stats_task);
3776	torture_stop_kthread(rcu_torture_fqs, fqs_task);
3777	if (rcu_torture_can_boost() && rcutor_hp >= 0)
3778	cpuhp_remove_state(state: rcutor_hp);
3779
3780	/*
3781	* Wait for all RCU callbacks to fire, then do torture-type-specific
3782	* cleanup operations.
3783	*/
3784	if (cur_ops->cb_barrier != NULL) {
3785	pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
3786	cur_ops->cb_barrier();
3787	}
3788	if (cur_ops->cleanup != NULL)
3789	cur_ops->cleanup();
3790
3791	rcu_torture_mem_dump_obj();
3792
3793	rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
3794
3795	if (err_segs_recorded) {
3796	pr_alert("Failure/close-call rcutorture reader segments:\n");
3797	if (rt_read_nsegs == 0)
3798	pr_alert("\t: No segments recorded!!!\n");
3799	firsttime = 1;
3800	for (i = 0; i < rt_read_nsegs; i++) {
3801	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP))
3802	pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL));
3803	else
3804	pr_alert("\t");
3805	pr_cont("%d: %#4x", i, err_segs[i].rt_readstate);
3806	if (err_segs[i].rt_delay_jiffies != 0) {
3807	pr_cont("%s%ldjiffies", firsttime ? "" : "+",
3808	err_segs[i].rt_delay_jiffies);
3809	firsttime = 0;
3810	}
3811	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) {
3812	pr_cont(" CPU %2d", err_segs[i].rt_cpu);
3813	if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu)
3814	pr_cont("->%-2d", err_segs[i].rt_end_cpu);
3815	else
3816	pr_cont(" ...");
3817	}
3818	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) &&
3819	cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) {
3820	char buf1[20+1];
3821	char buf2[20+1];
3822	char sepchar = '-';
3823
3824	cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq,
3825	buf1, ARRAY_SIZE(buf1));
3826	cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end,
3827	buf2, ARRAY_SIZE(buf2));
3828	if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) {
3829	if (buf2[0]) {
3830	for (j = 0; buf2[j]; j++)
3831	buf2[j] = '.';
3832	if (j)
3833	buf2[j - 1] = ' ';
3834	}
3835	sepchar = ' ';
3836	}
3837	pr_cont(" %s%c%s", buf1, sepchar, buf2);
3838	}
3839	if (err_segs[i].rt_delay_ms != 0) {
3840	pr_cont(" %s%ldms", firsttime ? "" : "+",
3841	err_segs[i].rt_delay_ms);
3842	firsttime = 0;
3843	}
3844	if (err_segs[i].rt_delay_us != 0) {
3845	pr_cont(" %s%ldus", firsttime ? "" : "+",
3846	err_segs[i].rt_delay_us);
3847	firsttime = 0;
3848	}
3849	pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : "");
3850	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH)
3851	pr_cont(" BH");
3852	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ)
3853	pr_cont(" IRQ");
3854	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT)
3855	pr_cont(" PREEMPT");
3856	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH)
3857	pr_cont(" RBH");
3858	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED)
3859	pr_cont(" SCHED");
3860	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1)
3861	pr_cont(" RCU_1");
3862	if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2)
3863	pr_cont(" RCU_2");
3864	pr_cont("\n");
3865
3866	}
3867	if (rt_read_preempted)
3868	pr_alert("\tReader was preempted.\n");
3869	}
3870	if (atomic_read(v: &n_rcu_torture_error) || n_rcu_torture_barrier_error)
3871	rcu_torture_print_module_parms(cur_ops, tag: "End of test: FAILURE");
3872	else if (torture_onoff_failures())
3873	rcu_torture_print_module_parms(cur_ops,
3874	tag: "End of test: RCU_HOTPLUG");
3875	else
3876	rcu_torture_print_module_parms(cur_ops, tag: "End of test: SUCCESS");
3877	torture_cleanup_end();
3878	if (cur_ops->gp_slow_unregister)
3879	cur_ops->gp_slow_unregister(NULL);
3880
3881	if (gpwrap_lag && cur_ops->set_gpwrap_lag)
3882	rcu_gpwrap_lag_cleanup();
3883	}
3884
3885	static void rcu_torture_leak_cb(struct rcu_head *rhp)
3886	{
3887	}
3888
3889	static void rcu_torture_err_cb(struct rcu_head *rhp)
3890	{
3891	/*
3892	* This -might- happen due to race conditions, but is unlikely.
3893	* The scenario that leads to this happening is that the
3894	* first of the pair of duplicate callbacks is queued,
3895	* someone else starts a grace period that includes that
3896	* callback, then the second of the pair must wait for the
3897	* next grace period. Unlikely, but can happen. If it
3898	* does happen, the debug-objects subsystem won't have splatted.
3899	*/
3900	pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME);
3901	}
3902
3903	/*
3904	* Verify that double-free causes debug-objects to complain, but only
3905	* if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test
3906	* cannot be carried out.
3907	*/
3908	static void rcu_test_debug_objects(void)
3909	{
3910	struct rcu_head rh1;
3911	struct rcu_head rh2;
3912	int idx;
3913
3914	if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) {
3915	pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n",
3916	KBUILD_MODNAME, cur_ops->name);
3917	return;
3918	}
3919
3920	if (WARN_ON_ONCE(cur_ops->debug_objects &&
3921	(!cur_ops->call || !cur_ops->cb_barrier)))
3922	return;
3923
3924	struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
3925
3926	init_rcu_head_on_stack(head: &rh1);
3927	init_rcu_head_on_stack(head: &rh2);
3928	pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name);
3929
3930	/* Try to queue the rh2 pair of callbacks for the same grace period. */
3931	idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */
3932	cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */
3933	cur_ops->call(&rh2, rcu_torture_leak_cb);
3934	cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
3935	if (rhp) {
3936	cur_ops->call(rhp, rcu_torture_leak_cb);
3937	cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */
3938	}
3939	cur_ops->readunlock(idx);
3940
3941	/* Wait for them all to get done so we can safely return. */
3942	cur_ops->cb_barrier();
3943	pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name);
3944	destroy_rcu_head_on_stack(head: &rh1);
3945	destroy_rcu_head_on_stack(head: &rh2);
3946	kfree(objp: rhp);
3947	}
3948
3949	static void rcutorture_sync(void)
3950	{
3951	static unsigned long n;
3952
3953	if (cur_ops->sync && !(++n & 0xfff))
3954	cur_ops->sync();
3955	}
3956
3957	static DEFINE_MUTEX(mut0);
3958	static DEFINE_MUTEX(mut1);
3959	static DEFINE_MUTEX(mut2);
3960	static DEFINE_MUTEX(mut3);
3961	static DEFINE_MUTEX(mut4);
3962	static DEFINE_MUTEX(mut5);
3963	static DEFINE_MUTEX(mut6);
3964	static DEFINE_MUTEX(mut7);
3965	static DEFINE_MUTEX(mut8);
3966	static DEFINE_MUTEX(mut9);
3967
3968	static DECLARE_RWSEM(rwsem0);
3969	static DECLARE_RWSEM(rwsem1);
3970	static DECLARE_RWSEM(rwsem2);
3971	static DECLARE_RWSEM(rwsem3);
3972	static DECLARE_RWSEM(rwsem4);
3973	static DECLARE_RWSEM(rwsem5);
3974	static DECLARE_RWSEM(rwsem6);
3975	static DECLARE_RWSEM(rwsem7);
3976	static DECLARE_RWSEM(rwsem8);
3977	static DECLARE_RWSEM(rwsem9);
3978
3979	DEFINE_STATIC_SRCU(srcu0);
3980	DEFINE_STATIC_SRCU(srcu1);
3981	DEFINE_STATIC_SRCU(srcu2);
3982	DEFINE_STATIC_SRCU(srcu3);
3983	DEFINE_STATIC_SRCU(srcu4);
3984	DEFINE_STATIC_SRCU(srcu5);
3985	DEFINE_STATIC_SRCU(srcu6);
3986	DEFINE_STATIC_SRCU(srcu7);
3987	DEFINE_STATIC_SRCU(srcu8);
3988	DEFINE_STATIC_SRCU(srcu9);
3989
3990	static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i,
3991	int cyclelen, int deadlock)
3992	{
3993	int j = i + 1;
3994
3995	if (j >= cyclelen)
3996	j = deadlock ? 0 : -1;
3997	if (j >= 0)
3998	pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i);
3999	else
4000	pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i);
4001	return j;
4002	}
4003
4004	// Test lockdep on SRCU-based deadlock scenarios.
4005	static void rcu_torture_init_srcu_lockdep(void)
4006	{
4007	int cyclelen;
4008	int deadlock;
4009	bool err = false;
4010	int i;
4011	int j;
4012	int idx;
4013	struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4,
4014	&mut5, &mut6, &mut7, &mut8, &mut9 };
4015	struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4,
4016	&rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 };
4017	struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4,
4018	&srcu5, &srcu6, &srcu7, &srcu8, &srcu9 };
4019	int testtype;
4020
4021	if (!test_srcu_lockdep)
4022	return;
4023
4024	deadlock = test_srcu_lockdep / 1000;
4025	testtype = (test_srcu_lockdep / 10) % 100;
4026	cyclelen = test_srcu_lockdep % 10;
4027	WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus));
4028	if (WARN_ONCE(deadlock != !!deadlock,
4029	"%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n",
4030	__func__, test_srcu_lockdep, deadlock))
4031	err = true;
4032	if (WARN_ONCE(cyclelen <= 0,
4033	"%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n",
4034	__func__, test_srcu_lockdep, cyclelen))
4035	err = true;
4036	if (err)
4037	goto err_out;
4038
4039	if (testtype == 0) {
4040	pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n",
4041	__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4042	if (deadlock && cyclelen == 1)
4043	pr_info("%s: Expect hang.\n", __func__);
4044	for (i = 0; i < cyclelen; i++) {
4045	j = srcu_lockdep_next(f: __func__, fl: "srcu_read_lock", fs: "synchronize_srcu",
4046	fu: "srcu_read_unlock", i, cyclelen, deadlock);
4047	idx = srcu_read_lock(ssp: srcus[i]);
4048	if (j >= 0)
4049	synchronize_srcu(ssp: srcus[j]);
4050	srcu_read_unlock(ssp: srcus[i], idx);
4051	}
4052	return;
4053	}
4054
4055	if (testtype == 1) {
4056	pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n",
4057	__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4058	for (i = 0; i < cyclelen; i++) {
4059	pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n",
4060	__func__, i, i, i, i);
4061	idx = srcu_read_lock(ssp: srcus[i]);
4062	mutex_lock(muts[i]);
4063	mutex_unlock(lock: muts[i]);
4064	srcu_read_unlock(ssp: srcus[i], idx);
4065
4066	j = srcu_lockdep_next(f: __func__, fl: "mutex_lock", fs: "synchronize_srcu",
4067	fu: "mutex_unlock", i, cyclelen, deadlock);
4068	mutex_lock(muts[i]);
4069	if (j >= 0)
4070	synchronize_srcu(ssp: srcus[j]);
4071	mutex_unlock(lock: muts[i]);
4072	}
4073	return;
4074	}
4075
4076	if (testtype == 2) {
4077	pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n",
4078	__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4079	for (i = 0; i < cyclelen; i++) {
4080	pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n",
4081	__func__, i, i, i, i);
4082	idx = srcu_read_lock(ssp: srcus[i]);
4083	down_read(sem: rwsems[i]);
4084	up_read(sem: rwsems[i]);
4085	srcu_read_unlock(ssp: srcus[i], idx);
4086
4087	j = srcu_lockdep_next(f: __func__, fl: "down_write", fs: "synchronize_srcu",
4088	fu: "up_write", i, cyclelen, deadlock);
4089	down_write(sem: rwsems[i]);
4090	if (j >= 0)
4091	synchronize_srcu(ssp: srcus[j]);
4092	up_write(sem: rwsems[i]);
4093	}
4094	return;
4095	}
4096
4097	#ifdef CONFIG_TASKS_TRACE_RCU
4098	if (testtype == 3) {
4099	pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n",
4100	__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4101	if (deadlock && cyclelen == 1)
4102	pr_info("%s: Expect hang.\n", __func__);
4103	for (i = 0; i < cyclelen; i++) {
4104	char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock";
4105	char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace"
4106	: "synchronize_srcu";
4107	char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock";
4108
4109	j = srcu_lockdep_next(f: __func__, fl, fs, fu, i, cyclelen, deadlock);
4110	if (i == 0)
4111	rcu_read_lock_trace();
4112	else
4113	idx = srcu_read_lock(ssp: srcus[i]);
4114	if (j >= 0) {
4115	if (i == cyclelen - 1)
4116	synchronize_rcu_tasks_trace();
4117	else
4118	synchronize_srcu(ssp: srcus[j]);
4119	}
4120	if (i == 0)
4121	rcu_read_unlock_trace();
4122	else
4123	srcu_read_unlock(ssp: srcus[i], idx);
4124	}
4125	return;
4126	}
4127	#endif // #ifdef CONFIG_TASKS_TRACE_RCU
4128
4129	err_out:
4130	pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep);
4131	pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__);
4132	pr_info("%s: D: Deadlock if nonzero.\n", __func__);
4133	pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__);
4134	pr_info("%s: L: Cycle length.\n", __func__);
4135	if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU))
4136	pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__);
4137	}
4138
4139	static int __init
4140	rcu_torture_init(void)
4141	{
4142	long i;
4143	int cpu;
4144	int firsterr = 0;
4145	int flags = 0;
4146	unsigned long gp_seq = 0;
4147	static struct rcu_torture_ops *torture_ops[] = {
4148	&rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops,
4149	TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS
4150	&trivial_ops,
4151	};
4152
4153	if (!torture_init_begin(ttype: torture_type, v: verbose))
4154	return -EBUSY;
4155
4156	/* Process args and tell the world that the torturer is on the job. */
4157	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
4158	cur_ops = torture_ops[i];
4159	if (strcmp(torture_type, cur_ops->name) == 0)
4160	break;
4161	}
4162	if (i == ARRAY_SIZE(torture_ops)) {
4163	pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
4164	torture_type);
4165	pr_alert("rcu-torture types:");
4166	for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
4167	pr_cont(" %s", torture_ops[i]->name);
4168	pr_cont("\n");
4169	firsterr = -EINVAL;
4170	cur_ops = NULL;
4171	goto unwind;
4172	}
4173	if (cur_ops->fqs == NULL && fqs_duration != 0) {
4174	pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
4175	fqs_duration = 0;
4176	}
4177	if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops ||
4178	!IS_ENABLED(CONFIG_RCU_NOCB_CPU))) {
4179	pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n",
4180	cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU));
4181	nocbs_nthreads = 0;
4182	}
4183	if (cur_ops->init)
4184	cur_ops->init();
4185
4186	rcu_torture_init_srcu_lockdep();
4187
4188	if (nfakewriters >= 0) {
4189	nrealfakewriters = nfakewriters;
4190	} else {
4191	nrealfakewriters = num_online_cpus() - 2 - nfakewriters;
4192	if (nrealfakewriters <= 0)
4193	nrealfakewriters = 1;
4194	}
4195
4196	if (nreaders >= 0) {
4197	nrealreaders = nreaders;
4198	} else {
4199	nrealreaders = num_online_cpus() - 2 - nreaders;
4200	if (nrealreaders <= 0)
4201	nrealreaders = 1;
4202	}
4203	rcu_torture_print_module_parms(cur_ops, tag: "Start of test");
4204	if (cur_ops->get_gp_data)
4205	cur_ops->get_gp_data(&flags, &gp_seq);
4206	start_gp_seq = gp_seq;
4207	pr_alert("%s: Start-test grace-period state: g%ld f%#x\n",
4208	cur_ops->name, (long)gp_seq, flags);
4209
4210	/* Set up the freelist. */
4211
4212	INIT_LIST_HEAD(list: &rcu_torture_freelist);
4213	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
4214	rcu_tortures[i].rtort_mbtest = 0;
4215	list_add_tail(new: &rcu_tortures[i].rtort_free,
4216	head: &rcu_torture_freelist);
4217	}
4218
4219	/* Initialize the statistics so that each run gets its own numbers. */
4220
4221	rcu_torture_current = NULL;
4222	rcu_torture_current_version = 0;
4223	atomic_set(v: &n_rcu_torture_alloc, i: 0);
4224	atomic_set(v: &n_rcu_torture_alloc_fail, i: 0);
4225	atomic_set(v: &n_rcu_torture_free, i: 0);
4226	atomic_set(v: &n_rcu_torture_mberror, i: 0);
4227	atomic_set(v: &n_rcu_torture_mbchk_fail, i: 0);
4228	atomic_set(v: &n_rcu_torture_mbchk_tries, i: 0);
4229	atomic_set(v: &n_rcu_torture_error, i: 0);
4230	n_rcu_torture_barrier_error = 0;
4231	n_rcu_torture_boost_ktrerror = 0;
4232	n_rcu_torture_boost_failure = 0;
4233	n_rcu_torture_boosts = 0;
4234	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
4235	atomic_set(v: &rcu_torture_wcount[i], i: 0);
4236	for_each_possible_cpu(cpu) {
4237	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
4238	per_cpu(rcu_torture_count, cpu)[i] = 0;
4239	per_cpu(rcu_torture_batch, cpu)[i] = 0;
4240	}
4241	}
4242	err_segs_recorded = 0;
4243	rt_read_nsegs = 0;
4244
4245	/* Start up the kthreads. */
4246
4247	rcu_torture_write_types();
4248	firsterr = torture_create_kthread(rcu_torture_writer, NULL,
4249	writer_task);
4250	if (torture_init_error(firsterr))
4251	goto unwind;
4252
4253	if (nrealfakewriters > 0) {
4254	fakewriter_tasks = kcalloc(nrealfakewriters,
4255	sizeof(fakewriter_tasks[0]),
4256	GFP_KERNEL);
4257	if (fakewriter_tasks == NULL) {
4258	TOROUT_ERRSTRING("out of memory");
4259	firsterr = -ENOMEM;
4260	goto unwind;
4261	}
4262	}
4263	for (i = 0; i < nrealfakewriters; i++) {
4264	firsterr = torture_create_kthread(rcu_torture_fakewriter,
4265	NULL, fakewriter_tasks[i]);
4266	if (torture_init_error(firsterr))
4267	goto unwind;
4268	}
4269	reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
4270	GFP_KERNEL);
4271	rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk),
4272	GFP_KERNEL);
4273	if (!reader_tasks || !rcu_torture_reader_mbchk) {
4274	TOROUT_ERRSTRING("out of memory");
4275	firsterr = -ENOMEM;
4276	goto unwind;
4277	}
4278	for (i = 0; i < nrealreaders; i++) {
4279	rcu_torture_reader_mbchk[i].rtc_chkrdr = -1;
4280	firsterr = torture_create_kthread(rcu_torture_reader, (void *)i,
4281	reader_tasks[i]);
4282	if (torture_init_error(firsterr))
4283	goto unwind;
4284	}
4285	nrealnocbers = nocbs_nthreads;
4286	if (WARN_ON(nrealnocbers < 0))
4287	nrealnocbers = 1;
4288	if (WARN_ON(nocbs_toggle < 0))
4289	nocbs_toggle = HZ;
4290	if (nrealnocbers > 0) {
4291	nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL);
4292	if (nocb_tasks == NULL) {
4293	TOROUT_ERRSTRING("out of memory");
4294	firsterr = -ENOMEM;
4295	goto unwind;
4296	}
4297	} else {
4298	nocb_tasks = NULL;
4299	}
4300	for (i = 0; i < nrealnocbers; i++) {
4301	firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]);
4302	if (torture_init_error(firsterr))
4303	goto unwind;
4304	}
4305	if (stat_interval > 0) {
4306	firsterr = torture_create_kthread(rcu_torture_stats, NULL,
4307	stats_task);
4308	if (torture_init_error(firsterr))
4309	goto unwind;
4310	}
4311	if (test_no_idle_hz && shuffle_interval > 0) {
4312	firsterr = torture_shuffle_init(shuffint: shuffle_interval * HZ);
4313	if (torture_init_error(firsterr))
4314	goto unwind;
4315	}
4316	if (stutter < 0)
4317	stutter = 0;
4318	if (stutter) {
4319	int t;
4320
4321	t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;
4322	firsterr = torture_stutter_init(s: stutter * HZ, sgap: t);
4323	if (torture_init_error(firsterr))
4324	goto unwind;
4325	}
4326	if (fqs_duration < 0)
4327	fqs_duration = 0;
4328	if (fqs_holdoff < 0)
4329	fqs_holdoff = 0;
4330	if (fqs_duration && fqs_holdoff) {
4331	/* Create the fqs thread */
4332	firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
4333	fqs_task);
4334	if (torture_init_error(firsterr))
4335	goto unwind;
4336	}
4337	if (test_boost_interval < 1)
4338	test_boost_interval = 1;
4339	if (test_boost_duration < 2)
4340	test_boost_duration = 2;
4341	if (rcu_torture_can_boost()) {
4342
4343	boost_starttime = jiffies + test_boost_interval * HZ;
4344
4345	firsterr = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "RCU_TORTURE",
4346	startup: rcutorture_booster_init,
4347	teardown: rcutorture_booster_cleanup);
4348	rcutor_hp = firsterr;
4349	if (torture_init_error(firsterr))
4350	goto unwind;
4351	}
4352	shutdown_jiffies = jiffies + shutdown_secs * HZ;
4353	firsterr = torture_shutdown_init(ssecs: shutdown_secs, cleanup: rcu_torture_cleanup);
4354	if (torture_init_error(firsterr))
4355	goto unwind;
4356	firsterr = torture_onoff_init(ooholdoff: onoff_holdoff * HZ, oointerval: onoff_interval,
4357	f: rcutorture_sync);
4358	if (torture_init_error(firsterr))
4359	goto unwind;
4360	firsterr = rcu_torture_stall_init();
4361	if (torture_init_error(firsterr))
4362	goto unwind;
4363	firsterr = rcu_torture_fwd_prog_init();
4364	if (torture_init_error(firsterr))
4365	goto unwind;
4366	firsterr = rcu_torture_barrier_init();
4367	if (torture_init_error(firsterr))
4368	goto unwind;
4369	firsterr = rcu_torture_read_exit_init();
4370	if (torture_init_error(firsterr))
4371	goto unwind;
4372	if (preempt_duration > 0) {
4373	firsterr = torture_create_kthread(rcu_torture_preempt, NULL, preempt_task);
4374	if (torture_init_error(firsterr))
4375	goto unwind;
4376	}
4377	if (object_debug)
4378	rcu_test_debug_objects();
4379
4380	if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister))
4381	cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay);
4382
4383	if (gpwrap_lag && cur_ops->set_gpwrap_lag) {
4384	firsterr = rcu_gpwrap_lag_init();
4385	if (torture_init_error(firsterr))
4386	goto unwind;
4387	}
4388
4389	torture_init_end();
4390	return 0;
4391
4392	unwind:
4393	torture_init_end();
4394	rcu_torture_cleanup();
4395	if (shutdown_secs) {
4396	WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST));
4397	kernel_power_off();
4398	}
4399	return firsterr;
4400	}
4401
4402	module_init(rcu_torture_init);
4403	module_exit(rcu_torture_cleanup);
4404