1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // |
3 | // Scalability test comparing RCU vs other mechanisms |
4 | // for acquiring references on objects. |
5 | // |
6 | // Copyright (C) Google, 2020. |
7 | // |
8 | // Author: Joel Fernandes <joel@joelfernandes.org> |
9 | |
10 | #define pr_fmt(fmt) fmt |
11 | |
12 | #include <linux/atomic.h> |
13 | #include <linux/bitops.h> |
14 | #include <linux/completion.h> |
15 | #include <linux/cpu.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/err.h> |
18 | #include <linux/init.h> |
19 | #include <linux/interrupt.h> |
20 | #include <linux/kthread.h> |
21 | #include <linux/kernel.h> |
22 | #include <linux/mm.h> |
23 | #include <linux/module.h> |
24 | #include <linux/moduleparam.h> |
25 | #include <linux/notifier.h> |
26 | #include <linux/percpu.h> |
27 | #include <linux/rcupdate.h> |
28 | #include <linux/rcupdate_trace.h> |
29 | #include <linux/reboot.h> |
30 | #include <linux/sched.h> |
31 | #include <linux/spinlock.h> |
32 | #include <linux/smp.h> |
33 | #include <linux/stat.h> |
34 | #include <linux/srcu.h> |
35 | #include <linux/slab.h> |
36 | #include <linux/torture.h> |
37 | #include <linux/types.h> |
38 | |
39 | #include "rcu.h" |
40 | |
41 | #define SCALE_FLAG "-ref-scale: " |
42 | |
43 | #define SCALEOUT(s, x...) \ |
44 | pr_alert("%s" SCALE_FLAG s, scale_type, ## x) |
45 | |
46 | #define VERBOSE_SCALEOUT(s, x...) \ |
47 | do { \ |
48 | if (verbose) \ |
49 | pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ |
50 | } while (0) |
51 | |
52 | static atomic_t verbose_batch_ctr; |
53 | |
54 | #define VERBOSE_SCALEOUT_BATCH(s, x...) \ |
55 | do { \ |
56 | if (verbose && \ |
57 | (verbose_batched <= 0 || \ |
58 | !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ |
59 | schedule_timeout_uninterruptible(1); \ |
60 | pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ |
61 | } \ |
62 | } while (0) |
63 | |
64 | #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x) |
65 | |
66 | MODULE_LICENSE("GPL" ); |
67 | MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>" ); |
68 | |
69 | static char *scale_type = "rcu" ; |
70 | module_param(scale_type, charp, 0444); |
71 | MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock." ); |
72 | |
73 | torture_param(int, verbose, 0, "Enable verbose debugging printk()s" ); |
74 | torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s" ); |
75 | |
76 | // Wait until there are multiple CPUs before starting test. |
77 | torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, |
78 | "Holdoff time before test start (s)" ); |
79 | // Number of typesafe_lookup structures, that is, the degree of concurrency. |
80 | torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures." ); |
81 | // Number of loops per experiment, all readers execute operations concurrently. |
82 | torture_param(long, loops, 10000, "Number of loops per experiment." ); |
83 | // Number of readers, with -1 defaulting to about 75% of the CPUs. |
84 | torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs." ); |
85 | // Number of runs. |
86 | torture_param(int, nruns, 30, "Number of experiments to run." ); |
87 | // Reader delay in nanoseconds, 0 for no delay. |
88 | torture_param(int, readdelay, 0, "Read-side delay in nanoseconds." ); |
89 | |
90 | #ifdef MODULE |
91 | # define REFSCALE_SHUTDOWN 0 |
92 | #else |
93 | # define REFSCALE_SHUTDOWN 1 |
94 | #endif |
95 | |
96 | torture_param(bool, shutdown, REFSCALE_SHUTDOWN, |
97 | "Shutdown at end of scalability tests." ); |
98 | |
99 | struct reader_task { |
100 | struct task_struct *task; |
101 | int start_reader; |
102 | wait_queue_head_t wq; |
103 | u64 last_duration_ns; |
104 | }; |
105 | |
106 | static struct task_struct *shutdown_task; |
107 | static wait_queue_head_t shutdown_wq; |
108 | |
109 | static struct task_struct *main_task; |
110 | static wait_queue_head_t main_wq; |
111 | static int shutdown_start; |
112 | |
113 | static struct reader_task *reader_tasks; |
114 | |
115 | // Number of readers that are part of the current experiment. |
116 | static atomic_t nreaders_exp; |
117 | |
118 | // Use to wait for all threads to start. |
119 | static atomic_t n_init; |
120 | static atomic_t n_started; |
121 | static atomic_t n_warmedup; |
122 | static atomic_t n_cooleddown; |
123 | |
124 | // Track which experiment is currently running. |
125 | static int exp_idx; |
126 | |
127 | // Operations vector for selecting different types of tests. |
128 | struct ref_scale_ops { |
129 | bool (*init)(void); |
130 | void (*cleanup)(void); |
131 | void (*readsection)(const int nloops); |
132 | void (*delaysection)(const int nloops, const int udl, const int ndl); |
133 | const char *name; |
134 | }; |
135 | |
136 | static struct ref_scale_ops *cur_ops; |
137 | |
138 | static void un_delay(const int udl, const int ndl) |
139 | { |
140 | if (udl) |
141 | udelay(udl); |
142 | if (ndl) |
143 | ndelay(ndl); |
144 | } |
145 | |
146 | static void ref_rcu_read_section(const int nloops) |
147 | { |
148 | int i; |
149 | |
150 | for (i = nloops; i >= 0; i--) { |
151 | rcu_read_lock(); |
152 | rcu_read_unlock(); |
153 | } |
154 | } |
155 | |
156 | static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) |
157 | { |
158 | int i; |
159 | |
160 | for (i = nloops; i >= 0; i--) { |
161 | rcu_read_lock(); |
162 | un_delay(udl, ndl); |
163 | rcu_read_unlock(); |
164 | } |
165 | } |
166 | |
167 | static bool rcu_sync_scale_init(void) |
168 | { |
169 | return true; |
170 | } |
171 | |
172 | static struct ref_scale_ops rcu_ops = { |
173 | .init = rcu_sync_scale_init, |
174 | .readsection = ref_rcu_read_section, |
175 | .delaysection = ref_rcu_delay_section, |
176 | .name = "rcu" |
177 | }; |
178 | |
179 | // Definitions for SRCU ref scale testing. |
180 | DEFINE_STATIC_SRCU(srcu_refctl_scale); |
181 | static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; |
182 | |
183 | static void srcu_ref_scale_read_section(const int nloops) |
184 | { |
185 | int i; |
186 | int idx; |
187 | |
188 | for (i = nloops; i >= 0; i--) { |
189 | idx = srcu_read_lock(ssp: srcu_ctlp); |
190 | srcu_read_unlock(ssp: srcu_ctlp, idx); |
191 | } |
192 | } |
193 | |
194 | static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
195 | { |
196 | int i; |
197 | int idx; |
198 | |
199 | for (i = nloops; i >= 0; i--) { |
200 | idx = srcu_read_lock(ssp: srcu_ctlp); |
201 | un_delay(udl, ndl); |
202 | srcu_read_unlock(ssp: srcu_ctlp, idx); |
203 | } |
204 | } |
205 | |
206 | static struct ref_scale_ops srcu_ops = { |
207 | .init = rcu_sync_scale_init, |
208 | .readsection = srcu_ref_scale_read_section, |
209 | .delaysection = srcu_ref_scale_delay_section, |
210 | .name = "srcu" |
211 | }; |
212 | |
213 | #ifdef CONFIG_TASKS_RCU |
214 | |
215 | // Definitions for RCU Tasks ref scale testing: Empty read markers. |
216 | // These definitions also work for RCU Rude readers. |
217 | static void rcu_tasks_ref_scale_read_section(const int nloops) |
218 | { |
219 | int i; |
220 | |
221 | for (i = nloops; i >= 0; i--) |
222 | continue; |
223 | } |
224 | |
225 | static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
226 | { |
227 | int i; |
228 | |
229 | for (i = nloops; i >= 0; i--) |
230 | un_delay(udl, ndl); |
231 | } |
232 | |
233 | static struct ref_scale_ops rcu_tasks_ops = { |
234 | .init = rcu_sync_scale_init, |
235 | .readsection = rcu_tasks_ref_scale_read_section, |
236 | .delaysection = rcu_tasks_ref_scale_delay_section, |
237 | .name = "rcu-tasks" |
238 | }; |
239 | |
240 | #define RCU_TASKS_OPS &rcu_tasks_ops, |
241 | |
242 | #else // #ifdef CONFIG_TASKS_RCU |
243 | |
244 | #define RCU_TASKS_OPS |
245 | |
246 | #endif // #else // #ifdef CONFIG_TASKS_RCU |
247 | |
248 | #ifdef CONFIG_TASKS_TRACE_RCU |
249 | |
250 | // Definitions for RCU Tasks Trace ref scale testing. |
251 | static void rcu_trace_ref_scale_read_section(const int nloops) |
252 | { |
253 | int i; |
254 | |
255 | for (i = nloops; i >= 0; i--) { |
256 | rcu_read_lock_trace(); |
257 | rcu_read_unlock_trace(); |
258 | } |
259 | } |
260 | |
261 | static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
262 | { |
263 | int i; |
264 | |
265 | for (i = nloops; i >= 0; i--) { |
266 | rcu_read_lock_trace(); |
267 | un_delay(udl, ndl); |
268 | rcu_read_unlock_trace(); |
269 | } |
270 | } |
271 | |
272 | static struct ref_scale_ops rcu_trace_ops = { |
273 | .init = rcu_sync_scale_init, |
274 | .readsection = rcu_trace_ref_scale_read_section, |
275 | .delaysection = rcu_trace_ref_scale_delay_section, |
276 | .name = "rcu-trace" |
277 | }; |
278 | |
279 | #define RCU_TRACE_OPS &rcu_trace_ops, |
280 | |
281 | #else // #ifdef CONFIG_TASKS_TRACE_RCU |
282 | |
283 | #define RCU_TRACE_OPS |
284 | |
285 | #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU |
286 | |
287 | // Definitions for reference count |
288 | static atomic_t refcnt; |
289 | |
290 | static void ref_refcnt_section(const int nloops) |
291 | { |
292 | int i; |
293 | |
294 | for (i = nloops; i >= 0; i--) { |
295 | atomic_inc(v: &refcnt); |
296 | atomic_dec(v: &refcnt); |
297 | } |
298 | } |
299 | |
300 | static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) |
301 | { |
302 | int i; |
303 | |
304 | for (i = nloops; i >= 0; i--) { |
305 | atomic_inc(v: &refcnt); |
306 | un_delay(udl, ndl); |
307 | atomic_dec(v: &refcnt); |
308 | } |
309 | } |
310 | |
311 | static struct ref_scale_ops refcnt_ops = { |
312 | .init = rcu_sync_scale_init, |
313 | .readsection = ref_refcnt_section, |
314 | .delaysection = ref_refcnt_delay_section, |
315 | .name = "refcnt" |
316 | }; |
317 | |
318 | // Definitions for rwlock |
319 | static rwlock_t test_rwlock; |
320 | |
321 | static bool ref_rwlock_init(void) |
322 | { |
323 | rwlock_init(&test_rwlock); |
324 | return true; |
325 | } |
326 | |
327 | static void ref_rwlock_section(const int nloops) |
328 | { |
329 | int i; |
330 | |
331 | for (i = nloops; i >= 0; i--) { |
332 | read_lock(&test_rwlock); |
333 | read_unlock(&test_rwlock); |
334 | } |
335 | } |
336 | |
337 | static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) |
338 | { |
339 | int i; |
340 | |
341 | for (i = nloops; i >= 0; i--) { |
342 | read_lock(&test_rwlock); |
343 | un_delay(udl, ndl); |
344 | read_unlock(&test_rwlock); |
345 | } |
346 | } |
347 | |
348 | static struct ref_scale_ops rwlock_ops = { |
349 | .init = ref_rwlock_init, |
350 | .readsection = ref_rwlock_section, |
351 | .delaysection = ref_rwlock_delay_section, |
352 | .name = "rwlock" |
353 | }; |
354 | |
355 | // Definitions for rwsem |
356 | static struct rw_semaphore test_rwsem; |
357 | |
358 | static bool ref_rwsem_init(void) |
359 | { |
360 | init_rwsem(&test_rwsem); |
361 | return true; |
362 | } |
363 | |
364 | static void ref_rwsem_section(const int nloops) |
365 | { |
366 | int i; |
367 | |
368 | for (i = nloops; i >= 0; i--) { |
369 | down_read(sem: &test_rwsem); |
370 | up_read(sem: &test_rwsem); |
371 | } |
372 | } |
373 | |
374 | static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) |
375 | { |
376 | int i; |
377 | |
378 | for (i = nloops; i >= 0; i--) { |
379 | down_read(sem: &test_rwsem); |
380 | un_delay(udl, ndl); |
381 | up_read(sem: &test_rwsem); |
382 | } |
383 | } |
384 | |
385 | static struct ref_scale_ops rwsem_ops = { |
386 | .init = ref_rwsem_init, |
387 | .readsection = ref_rwsem_section, |
388 | .delaysection = ref_rwsem_delay_section, |
389 | .name = "rwsem" |
390 | }; |
391 | |
392 | // Definitions for global spinlock |
393 | static DEFINE_RAW_SPINLOCK(test_lock); |
394 | |
395 | static void ref_lock_section(const int nloops) |
396 | { |
397 | int i; |
398 | |
399 | preempt_disable(); |
400 | for (i = nloops; i >= 0; i--) { |
401 | raw_spin_lock(&test_lock); |
402 | raw_spin_unlock(&test_lock); |
403 | } |
404 | preempt_enable(); |
405 | } |
406 | |
407 | static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) |
408 | { |
409 | int i; |
410 | |
411 | preempt_disable(); |
412 | for (i = nloops; i >= 0; i--) { |
413 | raw_spin_lock(&test_lock); |
414 | un_delay(udl, ndl); |
415 | raw_spin_unlock(&test_lock); |
416 | } |
417 | preempt_enable(); |
418 | } |
419 | |
420 | static struct ref_scale_ops lock_ops = { |
421 | .readsection = ref_lock_section, |
422 | .delaysection = ref_lock_delay_section, |
423 | .name = "lock" |
424 | }; |
425 | |
426 | // Definitions for global irq-save spinlock |
427 | |
428 | static void ref_lock_irq_section(const int nloops) |
429 | { |
430 | unsigned long flags; |
431 | int i; |
432 | |
433 | preempt_disable(); |
434 | for (i = nloops; i >= 0; i--) { |
435 | raw_spin_lock_irqsave(&test_lock, flags); |
436 | raw_spin_unlock_irqrestore(&test_lock, flags); |
437 | } |
438 | preempt_enable(); |
439 | } |
440 | |
441 | static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) |
442 | { |
443 | unsigned long flags; |
444 | int i; |
445 | |
446 | preempt_disable(); |
447 | for (i = nloops; i >= 0; i--) { |
448 | raw_spin_lock_irqsave(&test_lock, flags); |
449 | un_delay(udl, ndl); |
450 | raw_spin_unlock_irqrestore(&test_lock, flags); |
451 | } |
452 | preempt_enable(); |
453 | } |
454 | |
455 | static struct ref_scale_ops lock_irq_ops = { |
456 | .readsection = ref_lock_irq_section, |
457 | .delaysection = ref_lock_irq_delay_section, |
458 | .name = "lock-irq" |
459 | }; |
460 | |
461 | // Definitions acquire-release. |
462 | static DEFINE_PER_CPU(unsigned long, test_acqrel); |
463 | |
464 | static void ref_acqrel_section(const int nloops) |
465 | { |
466 | unsigned long x; |
467 | int i; |
468 | |
469 | preempt_disable(); |
470 | for (i = nloops; i >= 0; i--) { |
471 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); |
472 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); |
473 | } |
474 | preempt_enable(); |
475 | } |
476 | |
477 | static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) |
478 | { |
479 | unsigned long x; |
480 | int i; |
481 | |
482 | preempt_disable(); |
483 | for (i = nloops; i >= 0; i--) { |
484 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); |
485 | un_delay(udl, ndl); |
486 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); |
487 | } |
488 | preempt_enable(); |
489 | } |
490 | |
491 | static struct ref_scale_ops acqrel_ops = { |
492 | .readsection = ref_acqrel_section, |
493 | .delaysection = ref_acqrel_delay_section, |
494 | .name = "acqrel" |
495 | }; |
496 | |
497 | static volatile u64 stopopts; |
498 | |
499 | static void ref_clock_section(const int nloops) |
500 | { |
501 | u64 x = 0; |
502 | int i; |
503 | |
504 | preempt_disable(); |
505 | for (i = nloops; i >= 0; i--) |
506 | x += ktime_get_real_fast_ns(); |
507 | preempt_enable(); |
508 | stopopts = x; |
509 | } |
510 | |
511 | static void ref_clock_delay_section(const int nloops, const int udl, const int ndl) |
512 | { |
513 | u64 x = 0; |
514 | int i; |
515 | |
516 | preempt_disable(); |
517 | for (i = nloops; i >= 0; i--) { |
518 | x += ktime_get_real_fast_ns(); |
519 | un_delay(udl, ndl); |
520 | } |
521 | preempt_enable(); |
522 | stopopts = x; |
523 | } |
524 | |
525 | static struct ref_scale_ops clock_ops = { |
526 | .readsection = ref_clock_section, |
527 | .delaysection = ref_clock_delay_section, |
528 | .name = "clock" |
529 | }; |
530 | |
531 | static void ref_jiffies_section(const int nloops) |
532 | { |
533 | u64 x = 0; |
534 | int i; |
535 | |
536 | preempt_disable(); |
537 | for (i = nloops; i >= 0; i--) |
538 | x += jiffies; |
539 | preempt_enable(); |
540 | stopopts = x; |
541 | } |
542 | |
543 | static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl) |
544 | { |
545 | u64 x = 0; |
546 | int i; |
547 | |
548 | preempt_disable(); |
549 | for (i = nloops; i >= 0; i--) { |
550 | x += jiffies; |
551 | un_delay(udl, ndl); |
552 | } |
553 | preempt_enable(); |
554 | stopopts = x; |
555 | } |
556 | |
557 | static struct ref_scale_ops jiffies_ops = { |
558 | .readsection = ref_jiffies_section, |
559 | .delaysection = ref_jiffies_delay_section, |
560 | .name = "jiffies" |
561 | }; |
562 | |
563 | //////////////////////////////////////////////////////////////////////// |
564 | // |
565 | // Methods leveraging SLAB_TYPESAFE_BY_RCU. |
566 | // |
567 | |
568 | // Item to look up in a typesafe manner. Array of pointers to these. |
569 | struct refscale_typesafe { |
570 | atomic_t rts_refctr; // Used by all flavors |
571 | spinlock_t rts_lock; |
572 | seqlock_t rts_seqlock; |
573 | unsigned int a; |
574 | unsigned int b; |
575 | }; |
576 | |
577 | static struct kmem_cache *typesafe_kmem_cachep; |
578 | static struct refscale_typesafe **rtsarray; |
579 | static long rtsarray_size; |
580 | static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand); |
581 | static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start); |
582 | static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start); |
583 | |
584 | // Conditionally acquire an explicit in-structure reference count. |
585 | static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start) |
586 | { |
587 | return atomic_inc_not_zero(v: &rtsp->rts_refctr); |
588 | } |
589 | |
590 | // Unconditionally release an explicit in-structure reference count. |
591 | static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start) |
592 | { |
593 | if (!atomic_dec_return(v: &rtsp->rts_refctr)) { |
594 | WRITE_ONCE(rtsp->a, rtsp->a + 1); |
595 | kmem_cache_free(s: typesafe_kmem_cachep, objp: rtsp); |
596 | } |
597 | return true; |
598 | } |
599 | |
600 | // Unconditionally acquire an explicit in-structure spinlock. |
601 | static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start) |
602 | { |
603 | spin_lock(lock: &rtsp->rts_lock); |
604 | return true; |
605 | } |
606 | |
607 | // Unconditionally release an explicit in-structure spinlock. |
608 | static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start) |
609 | { |
610 | spin_unlock(lock: &rtsp->rts_lock); |
611 | return true; |
612 | } |
613 | |
614 | // Unconditionally acquire an explicit in-structure sequence lock. |
615 | static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start) |
616 | { |
617 | *start = read_seqbegin(sl: &rtsp->rts_seqlock); |
618 | return true; |
619 | } |
620 | |
621 | // Conditionally release an explicit in-structure sequence lock. Return |
622 | // true if this release was successful, that is, if no retry is required. |
623 | static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start) |
624 | { |
625 | return !read_seqretry(sl: &rtsp->rts_seqlock, start); |
626 | } |
627 | |
628 | // Do a read-side critical section with the specified delay in |
629 | // microseconds and nanoseconds inserted so as to increase probability |
630 | // of failure. |
631 | static void typesafe_delay_section(const int nloops, const int udl, const int ndl) |
632 | { |
633 | unsigned int a; |
634 | unsigned int b; |
635 | int i; |
636 | long idx; |
637 | struct refscale_typesafe *rtsp; |
638 | unsigned int start; |
639 | |
640 | for (i = nloops; i >= 0; i--) { |
641 | preempt_disable(); |
642 | idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size; |
643 | preempt_enable(); |
644 | retry: |
645 | rcu_read_lock(); |
646 | rtsp = rcu_dereference(rtsarray[idx]); |
647 | a = READ_ONCE(rtsp->a); |
648 | if (!rts_acquire(rtsp, &start)) { |
649 | rcu_read_unlock(); |
650 | goto retry; |
651 | } |
652 | if (a != READ_ONCE(rtsp->a)) { |
653 | (void)rts_release(rtsp, start); |
654 | rcu_read_unlock(); |
655 | goto retry; |
656 | } |
657 | un_delay(udl, ndl); |
658 | b = READ_ONCE(rtsp->a); |
659 | // Remember, seqlock read-side release can fail. |
660 | if (!rts_release(rtsp, start)) { |
661 | rcu_read_unlock(); |
662 | goto retry; |
663 | } |
664 | WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n" , a, b); |
665 | b = rtsp->b; |
666 | rcu_read_unlock(); |
667 | WARN_ON_ONCE(a * a != b); |
668 | } |
669 | } |
670 | |
671 | // Because the acquisition and release methods are expensive, there |
672 | // is no point in optimizing away the un_delay() function's two checks. |
673 | // Thus simply define typesafe_read_section() as a simple wrapper around |
674 | // typesafe_delay_section(). |
675 | static void typesafe_read_section(const int nloops) |
676 | { |
677 | typesafe_delay_section(nloops, udl: 0, ndl: 0); |
678 | } |
679 | |
680 | // Allocate and initialize one refscale_typesafe structure. |
681 | static struct refscale_typesafe *typesafe_alloc_one(void) |
682 | { |
683 | struct refscale_typesafe *rtsp; |
684 | |
685 | rtsp = kmem_cache_alloc(cachep: typesafe_kmem_cachep, GFP_KERNEL); |
686 | if (!rtsp) |
687 | return NULL; |
688 | atomic_set(v: &rtsp->rts_refctr, i: 1); |
689 | WRITE_ONCE(rtsp->a, rtsp->a + 1); |
690 | WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a); |
691 | return rtsp; |
692 | } |
693 | |
694 | // Slab-allocator constructor for refscale_typesafe structures created |
695 | // out of a new slab of system memory. |
696 | static void refscale_typesafe_ctor(void *rtsp_in) |
697 | { |
698 | struct refscale_typesafe *rtsp = rtsp_in; |
699 | |
700 | spin_lock_init(&rtsp->rts_lock); |
701 | seqlock_init(&rtsp->rts_seqlock); |
702 | preempt_disable(); |
703 | rtsp->a = torture_random(this_cpu_ptr(&refscale_rand)); |
704 | preempt_enable(); |
705 | } |
706 | |
707 | static struct ref_scale_ops typesafe_ref_ops; |
708 | static struct ref_scale_ops typesafe_lock_ops; |
709 | static struct ref_scale_ops typesafe_seqlock_ops; |
710 | |
711 | // Initialize for a typesafe test. |
712 | static bool typesafe_init(void) |
713 | { |
714 | long idx; |
715 | long si = lookup_instances; |
716 | |
717 | typesafe_kmem_cachep = kmem_cache_create(name: "refscale_typesafe" , |
718 | size: sizeof(struct refscale_typesafe), align: sizeof(void *), |
719 | SLAB_TYPESAFE_BY_RCU, ctor: refscale_typesafe_ctor); |
720 | if (!typesafe_kmem_cachep) |
721 | return false; |
722 | if (si < 0) |
723 | si = -si * nr_cpu_ids; |
724 | else if (si == 0) |
725 | si = nr_cpu_ids; |
726 | rtsarray_size = si; |
727 | rtsarray = kcalloc(n: si, size: sizeof(*rtsarray), GFP_KERNEL); |
728 | if (!rtsarray) |
729 | return false; |
730 | for (idx = 0; idx < rtsarray_size; idx++) { |
731 | rtsarray[idx] = typesafe_alloc_one(); |
732 | if (!rtsarray[idx]) |
733 | return false; |
734 | } |
735 | if (cur_ops == &typesafe_ref_ops) { |
736 | rts_acquire = typesafe_ref_acquire; |
737 | rts_release = typesafe_ref_release; |
738 | } else if (cur_ops == &typesafe_lock_ops) { |
739 | rts_acquire = typesafe_lock_acquire; |
740 | rts_release = typesafe_lock_release; |
741 | } else if (cur_ops == &typesafe_seqlock_ops) { |
742 | rts_acquire = typesafe_seqlock_acquire; |
743 | rts_release = typesafe_seqlock_release; |
744 | } else { |
745 | WARN_ON_ONCE(1); |
746 | return false; |
747 | } |
748 | return true; |
749 | } |
750 | |
751 | // Clean up after a typesafe test. |
752 | static void typesafe_cleanup(void) |
753 | { |
754 | long idx; |
755 | |
756 | if (rtsarray) { |
757 | for (idx = 0; idx < rtsarray_size; idx++) |
758 | kmem_cache_free(s: typesafe_kmem_cachep, objp: rtsarray[idx]); |
759 | kfree(objp: rtsarray); |
760 | rtsarray = NULL; |
761 | rtsarray_size = 0; |
762 | } |
763 | kmem_cache_destroy(s: typesafe_kmem_cachep); |
764 | typesafe_kmem_cachep = NULL; |
765 | rts_acquire = NULL; |
766 | rts_release = NULL; |
767 | } |
768 | |
769 | // The typesafe_init() function distinguishes these structures by address. |
770 | static struct ref_scale_ops typesafe_ref_ops = { |
771 | .init = typesafe_init, |
772 | .cleanup = typesafe_cleanup, |
773 | .readsection = typesafe_read_section, |
774 | .delaysection = typesafe_delay_section, |
775 | .name = "typesafe_ref" |
776 | }; |
777 | |
778 | static struct ref_scale_ops typesafe_lock_ops = { |
779 | .init = typesafe_init, |
780 | .cleanup = typesafe_cleanup, |
781 | .readsection = typesafe_read_section, |
782 | .delaysection = typesafe_delay_section, |
783 | .name = "typesafe_lock" |
784 | }; |
785 | |
786 | static struct ref_scale_ops typesafe_seqlock_ops = { |
787 | .init = typesafe_init, |
788 | .cleanup = typesafe_cleanup, |
789 | .readsection = typesafe_read_section, |
790 | .delaysection = typesafe_delay_section, |
791 | .name = "typesafe_seqlock" |
792 | }; |
793 | |
794 | static void rcu_scale_one_reader(void) |
795 | { |
796 | if (readdelay <= 0) |
797 | cur_ops->readsection(loops); |
798 | else |
799 | cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); |
800 | } |
801 | |
802 | // Reader kthread. Repeatedly does empty RCU read-side |
803 | // critical section, minimizing update-side interference. |
804 | static int |
805 | ref_scale_reader(void *arg) |
806 | { |
807 | unsigned long flags; |
808 | long me = (long)arg; |
809 | struct reader_task *rt = &(reader_tasks[me]); |
810 | u64 start; |
811 | s64 duration; |
812 | |
813 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started" , me); |
814 | WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids))); |
815 | set_user_nice(current, MAX_NICE); |
816 | atomic_inc(v: &n_init); |
817 | if (holdoff) |
818 | schedule_timeout_interruptible(timeout: holdoff * HZ); |
819 | repeat: |
820 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d" , me, raw_smp_processor_id()); |
821 | |
822 | // Wait for signal that this reader can start. |
823 | wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || |
824 | torture_must_stop()); |
825 | |
826 | if (torture_must_stop()) |
827 | goto end; |
828 | |
829 | // Make sure that the CPU is affinitized appropriately during testing. |
830 | WARN_ON_ONCE(raw_smp_processor_id() != me); |
831 | |
832 | WRITE_ONCE(rt->start_reader, 0); |
833 | if (!atomic_dec_return(v: &n_started)) |
834 | while (atomic_read_acquire(v: &n_started)) |
835 | cpu_relax(); |
836 | |
837 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started" , me, exp_idx); |
838 | |
839 | |
840 | // To reduce noise, do an initial cache-warming invocation, check |
841 | // in, and then keep warming until everyone has checked in. |
842 | rcu_scale_one_reader(); |
843 | if (!atomic_dec_return(v: &n_warmedup)) |
844 | while (atomic_read_acquire(v: &n_warmedup)) |
845 | rcu_scale_one_reader(); |
846 | // Also keep interrupts disabled. This also has the effect |
847 | // of preventing entries into slow path for rcu_read_unlock(). |
848 | local_irq_save(flags); |
849 | start = ktime_get_mono_fast_ns(); |
850 | |
851 | rcu_scale_one_reader(); |
852 | |
853 | duration = ktime_get_mono_fast_ns() - start; |
854 | local_irq_restore(flags); |
855 | |
856 | rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; |
857 | // To reduce runtime-skew noise, do maintain-load invocations until |
858 | // everyone is done. |
859 | if (!atomic_dec_return(v: &n_cooleddown)) |
860 | while (atomic_read_acquire(v: &n_cooleddown)) |
861 | rcu_scale_one_reader(); |
862 | |
863 | if (atomic_dec_and_test(v: &nreaders_exp)) |
864 | wake_up(&main_wq); |
865 | |
866 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)" , |
867 | me, exp_idx, atomic_read(&nreaders_exp)); |
868 | |
869 | if (!torture_must_stop()) |
870 | goto repeat; |
871 | end: |
872 | torture_kthread_stopping(title: "ref_scale_reader" ); |
873 | return 0; |
874 | } |
875 | |
876 | static void reset_readers(void) |
877 | { |
878 | int i; |
879 | struct reader_task *rt; |
880 | |
881 | for (i = 0; i < nreaders; i++) { |
882 | rt = &(reader_tasks[i]); |
883 | |
884 | rt->last_duration_ns = 0; |
885 | } |
886 | } |
887 | |
888 | // Print the results of each reader and return the sum of all their durations. |
889 | static u64 process_durations(int n) |
890 | { |
891 | int i; |
892 | struct reader_task *rt; |
893 | char buf1[64]; |
894 | char *buf; |
895 | u64 sum = 0; |
896 | |
897 | buf = kmalloc(size: 800 + 64, GFP_KERNEL); |
898 | if (!buf) |
899 | return 0; |
900 | buf[0] = 0; |
901 | sprintf(buf, fmt: "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)" , |
902 | exp_idx); |
903 | |
904 | for (i = 0; i < n && !torture_must_stop(); i++) { |
905 | rt = &(reader_tasks[i]); |
906 | sprintf(buf: buf1, fmt: "%d: %llu\t" , i, rt->last_duration_ns); |
907 | |
908 | if (i % 5 == 0) |
909 | strcat(p: buf, q: "\n" ); |
910 | if (strlen(buf) >= 800) { |
911 | pr_alert("%s" , buf); |
912 | buf[0] = 0; |
913 | } |
914 | strcat(p: buf, q: buf1); |
915 | |
916 | sum += rt->last_duration_ns; |
917 | } |
918 | pr_alert("%s\n" , buf); |
919 | |
920 | kfree(objp: buf); |
921 | return sum; |
922 | } |
923 | |
924 | // The main_func is the main orchestrator, it performs a bunch of |
925 | // experiments. For every experiment, it orders all the readers |
926 | // involved to start and waits for them to finish the experiment. It |
927 | // then reads their timestamps and starts the next experiment. Each |
928 | // experiment progresses from 1 concurrent reader to N of them at which |
929 | // point all the timestamps are printed. |
930 | static int main_func(void *arg) |
931 | { |
932 | int exp, r; |
933 | char buf1[64]; |
934 | char *buf; |
935 | u64 *result_avg; |
936 | |
937 | set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); |
938 | set_user_nice(current, MAX_NICE); |
939 | |
940 | VERBOSE_SCALEOUT("main_func task started" ); |
941 | result_avg = kzalloc(size: nruns * sizeof(*result_avg), GFP_KERNEL); |
942 | buf = kzalloc(size: 800 + 64, GFP_KERNEL); |
943 | if (!result_avg || !buf) { |
944 | SCALEOUT_ERRSTRING("out of memory" ); |
945 | goto oom_exit; |
946 | } |
947 | if (holdoff) |
948 | schedule_timeout_interruptible(timeout: holdoff * HZ); |
949 | |
950 | // Wait for all threads to start. |
951 | atomic_inc(v: &n_init); |
952 | while (atomic_read(v: &n_init) < nreaders + 1) |
953 | schedule_timeout_uninterruptible(timeout: 1); |
954 | |
955 | // Start exp readers up per experiment |
956 | for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { |
957 | if (torture_must_stop()) |
958 | goto end; |
959 | |
960 | reset_readers(); |
961 | atomic_set(v: &nreaders_exp, i: nreaders); |
962 | atomic_set(v: &n_started, i: nreaders); |
963 | atomic_set(v: &n_warmedup, i: nreaders); |
964 | atomic_set(v: &n_cooleddown, i: nreaders); |
965 | |
966 | exp_idx = exp; |
967 | |
968 | for (r = 0; r < nreaders; r++) { |
969 | smp_store_release(&reader_tasks[r].start_reader, 1); |
970 | wake_up(&reader_tasks[r].wq); |
971 | } |
972 | |
973 | VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers" , |
974 | nreaders); |
975 | |
976 | wait_event(main_wq, |
977 | !atomic_read(&nreaders_exp) || torture_must_stop()); |
978 | |
979 | VERBOSE_SCALEOUT("main_func: experiment ended" ); |
980 | |
981 | if (torture_must_stop()) |
982 | goto end; |
983 | |
984 | result_avg[exp] = div_u64(dividend: 1000 * process_durations(n: nreaders), divisor: nreaders * loops); |
985 | } |
986 | |
987 | // Print the average of all experiments |
988 | SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n" ); |
989 | |
990 | pr_alert("Runs\tTime(ns)\n" ); |
991 | for (exp = 0; exp < nruns; exp++) { |
992 | u64 avg; |
993 | u32 rem; |
994 | |
995 | avg = div_u64_rem(dividend: result_avg[exp], divisor: 1000, remainder: &rem); |
996 | sprintf(buf: buf1, fmt: "%d\t%llu.%03u\n" , exp + 1, avg, rem); |
997 | strcat(p: buf, q: buf1); |
998 | if (strlen(buf) >= 800) { |
999 | pr_alert("%s" , buf); |
1000 | buf[0] = 0; |
1001 | } |
1002 | } |
1003 | |
1004 | pr_alert("%s" , buf); |
1005 | |
1006 | oom_exit: |
1007 | // This will shutdown everything including us. |
1008 | if (shutdown) { |
1009 | shutdown_start = 1; |
1010 | wake_up(&shutdown_wq); |
1011 | } |
1012 | |
1013 | // Wait for torture to stop us |
1014 | while (!torture_must_stop()) |
1015 | schedule_timeout_uninterruptible(timeout: 1); |
1016 | |
1017 | end: |
1018 | torture_kthread_stopping(title: "main_func" ); |
1019 | kfree(objp: result_avg); |
1020 | kfree(objp: buf); |
1021 | return 0; |
1022 | } |
1023 | |
1024 | static void |
1025 | ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) |
1026 | { |
1027 | pr_alert("%s" SCALE_FLAG |
1028 | "--- %s: verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nreaders=%d nruns=%d readdelay=%d\n" , scale_type, tag, |
1029 | verbose, verbose_batched, shutdown, holdoff, lookup_instances, loops, nreaders, nruns, readdelay); |
1030 | } |
1031 | |
1032 | static void |
1033 | ref_scale_cleanup(void) |
1034 | { |
1035 | int i; |
1036 | |
1037 | if (torture_cleanup_begin()) |
1038 | return; |
1039 | |
1040 | if (!cur_ops) { |
1041 | torture_cleanup_end(); |
1042 | return; |
1043 | } |
1044 | |
1045 | if (reader_tasks) { |
1046 | for (i = 0; i < nreaders; i++) |
1047 | torture_stop_kthread("ref_scale_reader" , |
1048 | reader_tasks[i].task); |
1049 | } |
1050 | kfree(objp: reader_tasks); |
1051 | |
1052 | torture_stop_kthread("main_task" , main_task); |
1053 | kfree(objp: main_task); |
1054 | |
1055 | // Do scale-type-specific cleanup operations. |
1056 | if (cur_ops->cleanup != NULL) |
1057 | cur_ops->cleanup(); |
1058 | |
1059 | torture_cleanup_end(); |
1060 | } |
1061 | |
1062 | // Shutdown kthread. Just waits to be awakened, then shuts down system. |
1063 | static int |
1064 | ref_scale_shutdown(void *arg) |
1065 | { |
1066 | wait_event_idle(shutdown_wq, shutdown_start); |
1067 | |
1068 | smp_mb(); // Wake before output. |
1069 | ref_scale_cleanup(); |
1070 | kernel_power_off(); |
1071 | |
1072 | return -EINVAL; |
1073 | } |
1074 | |
1075 | static int __init |
1076 | ref_scale_init(void) |
1077 | { |
1078 | long i; |
1079 | int firsterr = 0; |
1080 | static struct ref_scale_ops *scale_ops[] = { |
1081 | &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, |
1082 | &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops, |
1083 | &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, |
1084 | }; |
1085 | |
1086 | if (!torture_init_begin(ttype: scale_type, v: verbose)) |
1087 | return -EBUSY; |
1088 | |
1089 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { |
1090 | cur_ops = scale_ops[i]; |
1091 | if (strcmp(scale_type, cur_ops->name) == 0) |
1092 | break; |
1093 | } |
1094 | if (i == ARRAY_SIZE(scale_ops)) { |
1095 | pr_alert("rcu-scale: invalid scale type: \"%s\"\n" , scale_type); |
1096 | pr_alert("rcu-scale types:" ); |
1097 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) |
1098 | pr_cont(" %s" , scale_ops[i]->name); |
1099 | pr_cont("\n" ); |
1100 | firsterr = -EINVAL; |
1101 | cur_ops = NULL; |
1102 | goto unwind; |
1103 | } |
1104 | if (cur_ops->init) |
1105 | if (!cur_ops->init()) { |
1106 | firsterr = -EUCLEAN; |
1107 | goto unwind; |
1108 | } |
1109 | |
1110 | ref_scale_print_module_parms(cur_ops, tag: "Start of test" ); |
1111 | |
1112 | // Shutdown task |
1113 | if (shutdown) { |
1114 | init_waitqueue_head(&shutdown_wq); |
1115 | firsterr = torture_create_kthread(ref_scale_shutdown, NULL, |
1116 | shutdown_task); |
1117 | if (torture_init_error(firsterr)) |
1118 | goto unwind; |
1119 | schedule_timeout_uninterruptible(timeout: 1); |
1120 | } |
1121 | |
1122 | // Reader tasks (default to ~75% of online CPUs). |
1123 | if (nreaders < 0) |
1124 | nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); |
1125 | if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n" , __func__, loops)) |
1126 | loops = 1; |
1127 | if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n" , __func__, nreaders)) |
1128 | nreaders = 1; |
1129 | if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n" , __func__, nruns)) |
1130 | nruns = 1; |
1131 | reader_tasks = kcalloc(n: nreaders, size: sizeof(reader_tasks[0]), |
1132 | GFP_KERNEL); |
1133 | if (!reader_tasks) { |
1134 | SCALEOUT_ERRSTRING("out of memory" ); |
1135 | firsterr = -ENOMEM; |
1136 | goto unwind; |
1137 | } |
1138 | |
1139 | VERBOSE_SCALEOUT("Starting %d reader threads" , nreaders); |
1140 | |
1141 | for (i = 0; i < nreaders; i++) { |
1142 | init_waitqueue_head(&reader_tasks[i].wq); |
1143 | firsterr = torture_create_kthread(ref_scale_reader, (void *)i, |
1144 | reader_tasks[i].task); |
1145 | if (torture_init_error(firsterr)) |
1146 | goto unwind; |
1147 | } |
1148 | |
1149 | // Main Task |
1150 | init_waitqueue_head(&main_wq); |
1151 | firsterr = torture_create_kthread(main_func, NULL, main_task); |
1152 | if (torture_init_error(firsterr)) |
1153 | goto unwind; |
1154 | |
1155 | torture_init_end(); |
1156 | return 0; |
1157 | |
1158 | unwind: |
1159 | torture_init_end(); |
1160 | ref_scale_cleanup(); |
1161 | if (shutdown) { |
1162 | WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); |
1163 | kernel_power_off(); |
1164 | } |
1165 | return firsterr; |
1166 | } |
1167 | |
1168 | module_init(ref_scale_init); |
1169 | module_exit(ref_scale_cleanup); |
1170 | |