1// SPDX-License-Identifier: GPL-2.0+
2//
3// Scalability test comparing RCU vs other mechanisms
4// for acquiring references on objects.
5//
6// Copyright (C) Google, 2020.
7//
8// Author: Joel Fernandes <joel@joelfernandes.org>
9
10#define pr_fmt(fmt) fmt
11
12#include <linux/atomic.h>
13#include <linux/bitops.h>
14#include <linux/completion.h>
15#include <linux/cpu.h>
16#include <linux/delay.h>
17#include <linux/err.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/kthread.h>
21#include <linux/kernel.h>
22#include <linux/mm.h>
23#include <linux/module.h>
24#include <linux/moduleparam.h>
25#include <linux/notifier.h>
26#include <linux/percpu.h>
27#include <linux/rcupdate.h>
28#include <linux/rcupdate_trace.h>
29#include <linux/reboot.h>
30#include <linux/sched.h>
31#include <linux/spinlock.h>
32#include <linux/smp.h>
33#include <linux/stat.h>
34#include <linux/srcu.h>
35#include <linux/slab.h>
36#include <linux/torture.h>
37#include <linux/types.h>
38
39#include "rcu.h"
40
41#define SCALE_FLAG "-ref-scale: "
42
43#define SCALEOUT(s, x...) \
44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
45
46#define VERBOSE_SCALEOUT(s, x...) \
47 do { \
48 if (verbose) \
49 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
50 } while (0)
51
52static atomic_t verbose_batch_ctr;
53
54#define VERBOSE_SCALEOUT_BATCH(s, x...) \
55do { \
56 if (verbose && \
57 (verbose_batched <= 0 || \
58 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
59 schedule_timeout_uninterruptible(1); \
60 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
61 } \
62} while (0)
63
64#define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
65
66MODULE_LICENSE("GPL");
67MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
68
69static char *scale_type = "rcu";
70module_param(scale_type, charp, 0444);
71MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
72
73torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
74torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
75
76// Wait until there are multiple CPUs before starting test.
77torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
78 "Holdoff time before test start (s)");
79// Number of typesafe_lookup structures, that is, the degree of concurrency.
80torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
81// Number of loops per experiment, all readers execute operations concurrently.
82torture_param(long, loops, 10000, "Number of loops per experiment.");
83// Number of readers, with -1 defaulting to about 75% of the CPUs.
84torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
85// Number of runs.
86torture_param(int, nruns, 30, "Number of experiments to run.");
87// Reader delay in nanoseconds, 0 for no delay.
88torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
89
90#ifdef MODULE
91# define REFSCALE_SHUTDOWN 0
92#else
93# define REFSCALE_SHUTDOWN 1
94#endif
95
96torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
97 "Shutdown at end of scalability tests.");
98
99struct reader_task {
100 struct task_struct *task;
101 int start_reader;
102 wait_queue_head_t wq;
103 u64 last_duration_ns;
104};
105
106static struct task_struct *shutdown_task;
107static wait_queue_head_t shutdown_wq;
108
109static struct task_struct *main_task;
110static wait_queue_head_t main_wq;
111static int shutdown_start;
112
113static struct reader_task *reader_tasks;
114
115// Number of readers that are part of the current experiment.
116static atomic_t nreaders_exp;
117
118// Use to wait for all threads to start.
119static atomic_t n_init;
120static atomic_t n_started;
121static atomic_t n_warmedup;
122static atomic_t n_cooleddown;
123
124// Track which experiment is currently running.
125static int exp_idx;
126
127// Operations vector for selecting different types of tests.
128struct ref_scale_ops {
129 bool (*init)(void);
130 void (*cleanup)(void);
131 void (*readsection)(const int nloops);
132 void (*delaysection)(const int nloops, const int udl, const int ndl);
133 const char *name;
134};
135
136static struct ref_scale_ops *cur_ops;
137
138static void un_delay(const int udl, const int ndl)
139{
140 if (udl)
141 udelay(udl);
142 if (ndl)
143 ndelay(ndl);
144}
145
146static void ref_rcu_read_section(const int nloops)
147{
148 int i;
149
150 for (i = nloops; i >= 0; i--) {
151 rcu_read_lock();
152 rcu_read_unlock();
153 }
154}
155
156static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
157{
158 int i;
159
160 for (i = nloops; i >= 0; i--) {
161 rcu_read_lock();
162 un_delay(udl, ndl);
163 rcu_read_unlock();
164 }
165}
166
167static bool rcu_sync_scale_init(void)
168{
169 return true;
170}
171
172static struct ref_scale_ops rcu_ops = {
173 .init = rcu_sync_scale_init,
174 .readsection = ref_rcu_read_section,
175 .delaysection = ref_rcu_delay_section,
176 .name = "rcu"
177};
178
179// Definitions for SRCU ref scale testing.
180DEFINE_STATIC_SRCU(srcu_refctl_scale);
181static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
182
183static void srcu_ref_scale_read_section(const int nloops)
184{
185 int i;
186 int idx;
187
188 for (i = nloops; i >= 0; i--) {
189 idx = srcu_read_lock(ssp: srcu_ctlp);
190 srcu_read_unlock(ssp: srcu_ctlp, idx);
191 }
192}
193
194static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
195{
196 int i;
197 int idx;
198
199 for (i = nloops; i >= 0; i--) {
200 idx = srcu_read_lock(ssp: srcu_ctlp);
201 un_delay(udl, ndl);
202 srcu_read_unlock(ssp: srcu_ctlp, idx);
203 }
204}
205
206static struct ref_scale_ops srcu_ops = {
207 .init = rcu_sync_scale_init,
208 .readsection = srcu_ref_scale_read_section,
209 .delaysection = srcu_ref_scale_delay_section,
210 .name = "srcu"
211};
212
213#ifdef CONFIG_TASKS_RCU
214
215// Definitions for RCU Tasks ref scale testing: Empty read markers.
216// These definitions also work for RCU Rude readers.
217static void rcu_tasks_ref_scale_read_section(const int nloops)
218{
219 int i;
220
221 for (i = nloops; i >= 0; i--)
222 continue;
223}
224
225static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
226{
227 int i;
228
229 for (i = nloops; i >= 0; i--)
230 un_delay(udl, ndl);
231}
232
233static struct ref_scale_ops rcu_tasks_ops = {
234 .init = rcu_sync_scale_init,
235 .readsection = rcu_tasks_ref_scale_read_section,
236 .delaysection = rcu_tasks_ref_scale_delay_section,
237 .name = "rcu-tasks"
238};
239
240#define RCU_TASKS_OPS &rcu_tasks_ops,
241
242#else // #ifdef CONFIG_TASKS_RCU
243
244#define RCU_TASKS_OPS
245
246#endif // #else // #ifdef CONFIG_TASKS_RCU
247
248#ifdef CONFIG_TASKS_TRACE_RCU
249
250// Definitions for RCU Tasks Trace ref scale testing.
251static void rcu_trace_ref_scale_read_section(const int nloops)
252{
253 int i;
254
255 for (i = nloops; i >= 0; i--) {
256 rcu_read_lock_trace();
257 rcu_read_unlock_trace();
258 }
259}
260
261static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
262{
263 int i;
264
265 for (i = nloops; i >= 0; i--) {
266 rcu_read_lock_trace();
267 un_delay(udl, ndl);
268 rcu_read_unlock_trace();
269 }
270}
271
272static struct ref_scale_ops rcu_trace_ops = {
273 .init = rcu_sync_scale_init,
274 .readsection = rcu_trace_ref_scale_read_section,
275 .delaysection = rcu_trace_ref_scale_delay_section,
276 .name = "rcu-trace"
277};
278
279#define RCU_TRACE_OPS &rcu_trace_ops,
280
281#else // #ifdef CONFIG_TASKS_TRACE_RCU
282
283#define RCU_TRACE_OPS
284
285#endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
286
287// Definitions for reference count
288static atomic_t refcnt;
289
290static void ref_refcnt_section(const int nloops)
291{
292 int i;
293
294 for (i = nloops; i >= 0; i--) {
295 atomic_inc(v: &refcnt);
296 atomic_dec(v: &refcnt);
297 }
298}
299
300static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
301{
302 int i;
303
304 for (i = nloops; i >= 0; i--) {
305 atomic_inc(v: &refcnt);
306 un_delay(udl, ndl);
307 atomic_dec(v: &refcnt);
308 }
309}
310
311static struct ref_scale_ops refcnt_ops = {
312 .init = rcu_sync_scale_init,
313 .readsection = ref_refcnt_section,
314 .delaysection = ref_refcnt_delay_section,
315 .name = "refcnt"
316};
317
318// Definitions for rwlock
319static rwlock_t test_rwlock;
320
321static bool ref_rwlock_init(void)
322{
323 rwlock_init(&test_rwlock);
324 return true;
325}
326
327static void ref_rwlock_section(const int nloops)
328{
329 int i;
330
331 for (i = nloops; i >= 0; i--) {
332 read_lock(&test_rwlock);
333 read_unlock(&test_rwlock);
334 }
335}
336
337static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
338{
339 int i;
340
341 for (i = nloops; i >= 0; i--) {
342 read_lock(&test_rwlock);
343 un_delay(udl, ndl);
344 read_unlock(&test_rwlock);
345 }
346}
347
348static struct ref_scale_ops rwlock_ops = {
349 .init = ref_rwlock_init,
350 .readsection = ref_rwlock_section,
351 .delaysection = ref_rwlock_delay_section,
352 .name = "rwlock"
353};
354
355// Definitions for rwsem
356static struct rw_semaphore test_rwsem;
357
358static bool ref_rwsem_init(void)
359{
360 init_rwsem(&test_rwsem);
361 return true;
362}
363
364static void ref_rwsem_section(const int nloops)
365{
366 int i;
367
368 for (i = nloops; i >= 0; i--) {
369 down_read(sem: &test_rwsem);
370 up_read(sem: &test_rwsem);
371 }
372}
373
374static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
375{
376 int i;
377
378 for (i = nloops; i >= 0; i--) {
379 down_read(sem: &test_rwsem);
380 un_delay(udl, ndl);
381 up_read(sem: &test_rwsem);
382 }
383}
384
385static struct ref_scale_ops rwsem_ops = {
386 .init = ref_rwsem_init,
387 .readsection = ref_rwsem_section,
388 .delaysection = ref_rwsem_delay_section,
389 .name = "rwsem"
390};
391
392// Definitions for global spinlock
393static DEFINE_RAW_SPINLOCK(test_lock);
394
395static void ref_lock_section(const int nloops)
396{
397 int i;
398
399 preempt_disable();
400 for (i = nloops; i >= 0; i--) {
401 raw_spin_lock(&test_lock);
402 raw_spin_unlock(&test_lock);
403 }
404 preempt_enable();
405}
406
407static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
408{
409 int i;
410
411 preempt_disable();
412 for (i = nloops; i >= 0; i--) {
413 raw_spin_lock(&test_lock);
414 un_delay(udl, ndl);
415 raw_spin_unlock(&test_lock);
416 }
417 preempt_enable();
418}
419
420static struct ref_scale_ops lock_ops = {
421 .readsection = ref_lock_section,
422 .delaysection = ref_lock_delay_section,
423 .name = "lock"
424};
425
426// Definitions for global irq-save spinlock
427
428static void ref_lock_irq_section(const int nloops)
429{
430 unsigned long flags;
431 int i;
432
433 preempt_disable();
434 for (i = nloops; i >= 0; i--) {
435 raw_spin_lock_irqsave(&test_lock, flags);
436 raw_spin_unlock_irqrestore(&test_lock, flags);
437 }
438 preempt_enable();
439}
440
441static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
442{
443 unsigned long flags;
444 int i;
445
446 preempt_disable();
447 for (i = nloops; i >= 0; i--) {
448 raw_spin_lock_irqsave(&test_lock, flags);
449 un_delay(udl, ndl);
450 raw_spin_unlock_irqrestore(&test_lock, flags);
451 }
452 preempt_enable();
453}
454
455static struct ref_scale_ops lock_irq_ops = {
456 .readsection = ref_lock_irq_section,
457 .delaysection = ref_lock_irq_delay_section,
458 .name = "lock-irq"
459};
460
461// Definitions acquire-release.
462static DEFINE_PER_CPU(unsigned long, test_acqrel);
463
464static void ref_acqrel_section(const int nloops)
465{
466 unsigned long x;
467 int i;
468
469 preempt_disable();
470 for (i = nloops; i >= 0; i--) {
471 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
472 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
473 }
474 preempt_enable();
475}
476
477static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
478{
479 unsigned long x;
480 int i;
481
482 preempt_disable();
483 for (i = nloops; i >= 0; i--) {
484 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
485 un_delay(udl, ndl);
486 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
487 }
488 preempt_enable();
489}
490
491static struct ref_scale_ops acqrel_ops = {
492 .readsection = ref_acqrel_section,
493 .delaysection = ref_acqrel_delay_section,
494 .name = "acqrel"
495};
496
497static volatile u64 stopopts;
498
499static void ref_clock_section(const int nloops)
500{
501 u64 x = 0;
502 int i;
503
504 preempt_disable();
505 for (i = nloops; i >= 0; i--)
506 x += ktime_get_real_fast_ns();
507 preempt_enable();
508 stopopts = x;
509}
510
511static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
512{
513 u64 x = 0;
514 int i;
515
516 preempt_disable();
517 for (i = nloops; i >= 0; i--) {
518 x += ktime_get_real_fast_ns();
519 un_delay(udl, ndl);
520 }
521 preempt_enable();
522 stopopts = x;
523}
524
525static struct ref_scale_ops clock_ops = {
526 .readsection = ref_clock_section,
527 .delaysection = ref_clock_delay_section,
528 .name = "clock"
529};
530
531static void ref_jiffies_section(const int nloops)
532{
533 u64 x = 0;
534 int i;
535
536 preempt_disable();
537 for (i = nloops; i >= 0; i--)
538 x += jiffies;
539 preempt_enable();
540 stopopts = x;
541}
542
543static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl)
544{
545 u64 x = 0;
546 int i;
547
548 preempt_disable();
549 for (i = nloops; i >= 0; i--) {
550 x += jiffies;
551 un_delay(udl, ndl);
552 }
553 preempt_enable();
554 stopopts = x;
555}
556
557static struct ref_scale_ops jiffies_ops = {
558 .readsection = ref_jiffies_section,
559 .delaysection = ref_jiffies_delay_section,
560 .name = "jiffies"
561};
562
563////////////////////////////////////////////////////////////////////////
564//
565// Methods leveraging SLAB_TYPESAFE_BY_RCU.
566//
567
568// Item to look up in a typesafe manner. Array of pointers to these.
569struct refscale_typesafe {
570 atomic_t rts_refctr; // Used by all flavors
571 spinlock_t rts_lock;
572 seqlock_t rts_seqlock;
573 unsigned int a;
574 unsigned int b;
575};
576
577static struct kmem_cache *typesafe_kmem_cachep;
578static struct refscale_typesafe **rtsarray;
579static long rtsarray_size;
580static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
581static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
582static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
583
584// Conditionally acquire an explicit in-structure reference count.
585static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
586{
587 return atomic_inc_not_zero(v: &rtsp->rts_refctr);
588}
589
590// Unconditionally release an explicit in-structure reference count.
591static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
592{
593 if (!atomic_dec_return(v: &rtsp->rts_refctr)) {
594 WRITE_ONCE(rtsp->a, rtsp->a + 1);
595 kmem_cache_free(s: typesafe_kmem_cachep, objp: rtsp);
596 }
597 return true;
598}
599
600// Unconditionally acquire an explicit in-structure spinlock.
601static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
602{
603 spin_lock(lock: &rtsp->rts_lock);
604 return true;
605}
606
607// Unconditionally release an explicit in-structure spinlock.
608static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
609{
610 spin_unlock(lock: &rtsp->rts_lock);
611 return true;
612}
613
614// Unconditionally acquire an explicit in-structure sequence lock.
615static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
616{
617 *start = read_seqbegin(sl: &rtsp->rts_seqlock);
618 return true;
619}
620
621// Conditionally release an explicit in-structure sequence lock. Return
622// true if this release was successful, that is, if no retry is required.
623static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
624{
625 return !read_seqretry(sl: &rtsp->rts_seqlock, start);
626}
627
628// Do a read-side critical section with the specified delay in
629// microseconds and nanoseconds inserted so as to increase probability
630// of failure.
631static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
632{
633 unsigned int a;
634 unsigned int b;
635 int i;
636 long idx;
637 struct refscale_typesafe *rtsp;
638 unsigned int start;
639
640 for (i = nloops; i >= 0; i--) {
641 preempt_disable();
642 idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
643 preempt_enable();
644retry:
645 rcu_read_lock();
646 rtsp = rcu_dereference(rtsarray[idx]);
647 a = READ_ONCE(rtsp->a);
648 if (!rts_acquire(rtsp, &start)) {
649 rcu_read_unlock();
650 goto retry;
651 }
652 if (a != READ_ONCE(rtsp->a)) {
653 (void)rts_release(rtsp, start);
654 rcu_read_unlock();
655 goto retry;
656 }
657 un_delay(udl, ndl);
658 b = READ_ONCE(rtsp->a);
659 // Remember, seqlock read-side release can fail.
660 if (!rts_release(rtsp, start)) {
661 rcu_read_unlock();
662 goto retry;
663 }
664 WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
665 b = rtsp->b;
666 rcu_read_unlock();
667 WARN_ON_ONCE(a * a != b);
668 }
669}
670
671// Because the acquisition and release methods are expensive, there
672// is no point in optimizing away the un_delay() function's two checks.
673// Thus simply define typesafe_read_section() as a simple wrapper around
674// typesafe_delay_section().
675static void typesafe_read_section(const int nloops)
676{
677 typesafe_delay_section(nloops, udl: 0, ndl: 0);
678}
679
680// Allocate and initialize one refscale_typesafe structure.
681static struct refscale_typesafe *typesafe_alloc_one(void)
682{
683 struct refscale_typesafe *rtsp;
684
685 rtsp = kmem_cache_alloc(cachep: typesafe_kmem_cachep, GFP_KERNEL);
686 if (!rtsp)
687 return NULL;
688 atomic_set(v: &rtsp->rts_refctr, i: 1);
689 WRITE_ONCE(rtsp->a, rtsp->a + 1);
690 WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
691 return rtsp;
692}
693
694// Slab-allocator constructor for refscale_typesafe structures created
695// out of a new slab of system memory.
696static void refscale_typesafe_ctor(void *rtsp_in)
697{
698 struct refscale_typesafe *rtsp = rtsp_in;
699
700 spin_lock_init(&rtsp->rts_lock);
701 seqlock_init(&rtsp->rts_seqlock);
702 preempt_disable();
703 rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
704 preempt_enable();
705}
706
707static struct ref_scale_ops typesafe_ref_ops;
708static struct ref_scale_ops typesafe_lock_ops;
709static struct ref_scale_ops typesafe_seqlock_ops;
710
711// Initialize for a typesafe test.
712static bool typesafe_init(void)
713{
714 long idx;
715 long si = lookup_instances;
716
717 typesafe_kmem_cachep = kmem_cache_create(name: "refscale_typesafe",
718 size: sizeof(struct refscale_typesafe), align: sizeof(void *),
719 SLAB_TYPESAFE_BY_RCU, ctor: refscale_typesafe_ctor);
720 if (!typesafe_kmem_cachep)
721 return false;
722 if (si < 0)
723 si = -si * nr_cpu_ids;
724 else if (si == 0)
725 si = nr_cpu_ids;
726 rtsarray_size = si;
727 rtsarray = kcalloc(n: si, size: sizeof(*rtsarray), GFP_KERNEL);
728 if (!rtsarray)
729 return false;
730 for (idx = 0; idx < rtsarray_size; idx++) {
731 rtsarray[idx] = typesafe_alloc_one();
732 if (!rtsarray[idx])
733 return false;
734 }
735 if (cur_ops == &typesafe_ref_ops) {
736 rts_acquire = typesafe_ref_acquire;
737 rts_release = typesafe_ref_release;
738 } else if (cur_ops == &typesafe_lock_ops) {
739 rts_acquire = typesafe_lock_acquire;
740 rts_release = typesafe_lock_release;
741 } else if (cur_ops == &typesafe_seqlock_ops) {
742 rts_acquire = typesafe_seqlock_acquire;
743 rts_release = typesafe_seqlock_release;
744 } else {
745 WARN_ON_ONCE(1);
746 return false;
747 }
748 return true;
749}
750
751// Clean up after a typesafe test.
752static void typesafe_cleanup(void)
753{
754 long idx;
755
756 if (rtsarray) {
757 for (idx = 0; idx < rtsarray_size; idx++)
758 kmem_cache_free(s: typesafe_kmem_cachep, objp: rtsarray[idx]);
759 kfree(objp: rtsarray);
760 rtsarray = NULL;
761 rtsarray_size = 0;
762 }
763 kmem_cache_destroy(s: typesafe_kmem_cachep);
764 typesafe_kmem_cachep = NULL;
765 rts_acquire = NULL;
766 rts_release = NULL;
767}
768
769// The typesafe_init() function distinguishes these structures by address.
770static struct ref_scale_ops typesafe_ref_ops = {
771 .init = typesafe_init,
772 .cleanup = typesafe_cleanup,
773 .readsection = typesafe_read_section,
774 .delaysection = typesafe_delay_section,
775 .name = "typesafe_ref"
776};
777
778static struct ref_scale_ops typesafe_lock_ops = {
779 .init = typesafe_init,
780 .cleanup = typesafe_cleanup,
781 .readsection = typesafe_read_section,
782 .delaysection = typesafe_delay_section,
783 .name = "typesafe_lock"
784};
785
786static struct ref_scale_ops typesafe_seqlock_ops = {
787 .init = typesafe_init,
788 .cleanup = typesafe_cleanup,
789 .readsection = typesafe_read_section,
790 .delaysection = typesafe_delay_section,
791 .name = "typesafe_seqlock"
792};
793
794static void rcu_scale_one_reader(void)
795{
796 if (readdelay <= 0)
797 cur_ops->readsection(loops);
798 else
799 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
800}
801
802// Reader kthread. Repeatedly does empty RCU read-side
803// critical section, minimizing update-side interference.
804static int
805ref_scale_reader(void *arg)
806{
807 unsigned long flags;
808 long me = (long)arg;
809 struct reader_task *rt = &(reader_tasks[me]);
810 u64 start;
811 s64 duration;
812
813 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
814 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
815 set_user_nice(current, MAX_NICE);
816 atomic_inc(v: &n_init);
817 if (holdoff)
818 schedule_timeout_interruptible(timeout: holdoff * HZ);
819repeat:
820 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
821
822 // Wait for signal that this reader can start.
823 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
824 torture_must_stop());
825
826 if (torture_must_stop())
827 goto end;
828
829 // Make sure that the CPU is affinitized appropriately during testing.
830 WARN_ON_ONCE(raw_smp_processor_id() != me);
831
832 WRITE_ONCE(rt->start_reader, 0);
833 if (!atomic_dec_return(v: &n_started))
834 while (atomic_read_acquire(v: &n_started))
835 cpu_relax();
836
837 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
838
839
840 // To reduce noise, do an initial cache-warming invocation, check
841 // in, and then keep warming until everyone has checked in.
842 rcu_scale_one_reader();
843 if (!atomic_dec_return(v: &n_warmedup))
844 while (atomic_read_acquire(v: &n_warmedup))
845 rcu_scale_one_reader();
846 // Also keep interrupts disabled. This also has the effect
847 // of preventing entries into slow path for rcu_read_unlock().
848 local_irq_save(flags);
849 start = ktime_get_mono_fast_ns();
850
851 rcu_scale_one_reader();
852
853 duration = ktime_get_mono_fast_ns() - start;
854 local_irq_restore(flags);
855
856 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
857 // To reduce runtime-skew noise, do maintain-load invocations until
858 // everyone is done.
859 if (!atomic_dec_return(v: &n_cooleddown))
860 while (atomic_read_acquire(v: &n_cooleddown))
861 rcu_scale_one_reader();
862
863 if (atomic_dec_and_test(v: &nreaders_exp))
864 wake_up(&main_wq);
865
866 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
867 me, exp_idx, atomic_read(&nreaders_exp));
868
869 if (!torture_must_stop())
870 goto repeat;
871end:
872 torture_kthread_stopping(title: "ref_scale_reader");
873 return 0;
874}
875
876static void reset_readers(void)
877{
878 int i;
879 struct reader_task *rt;
880
881 for (i = 0; i < nreaders; i++) {
882 rt = &(reader_tasks[i]);
883
884 rt->last_duration_ns = 0;
885 }
886}
887
888// Print the results of each reader and return the sum of all their durations.
889static u64 process_durations(int n)
890{
891 int i;
892 struct reader_task *rt;
893 char buf1[64];
894 char *buf;
895 u64 sum = 0;
896
897 buf = kmalloc(size: 800 + 64, GFP_KERNEL);
898 if (!buf)
899 return 0;
900 buf[0] = 0;
901 sprintf(buf, fmt: "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
902 exp_idx);
903
904 for (i = 0; i < n && !torture_must_stop(); i++) {
905 rt = &(reader_tasks[i]);
906 sprintf(buf: buf1, fmt: "%d: %llu\t", i, rt->last_duration_ns);
907
908 if (i % 5 == 0)
909 strcat(p: buf, q: "\n");
910 if (strlen(buf) >= 800) {
911 pr_alert("%s", buf);
912 buf[0] = 0;
913 }
914 strcat(p: buf, q: buf1);
915
916 sum += rt->last_duration_ns;
917 }
918 pr_alert("%s\n", buf);
919
920 kfree(objp: buf);
921 return sum;
922}
923
924// The main_func is the main orchestrator, it performs a bunch of
925// experiments. For every experiment, it orders all the readers
926// involved to start and waits for them to finish the experiment. It
927// then reads their timestamps and starts the next experiment. Each
928// experiment progresses from 1 concurrent reader to N of them at which
929// point all the timestamps are printed.
930static int main_func(void *arg)
931{
932 int exp, r;
933 char buf1[64];
934 char *buf;
935 u64 *result_avg;
936
937 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
938 set_user_nice(current, MAX_NICE);
939
940 VERBOSE_SCALEOUT("main_func task started");
941 result_avg = kzalloc(size: nruns * sizeof(*result_avg), GFP_KERNEL);
942 buf = kzalloc(size: 800 + 64, GFP_KERNEL);
943 if (!result_avg || !buf) {
944 SCALEOUT_ERRSTRING("out of memory");
945 goto oom_exit;
946 }
947 if (holdoff)
948 schedule_timeout_interruptible(timeout: holdoff * HZ);
949
950 // Wait for all threads to start.
951 atomic_inc(v: &n_init);
952 while (atomic_read(v: &n_init) < nreaders + 1)
953 schedule_timeout_uninterruptible(timeout: 1);
954
955 // Start exp readers up per experiment
956 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
957 if (torture_must_stop())
958 goto end;
959
960 reset_readers();
961 atomic_set(v: &nreaders_exp, i: nreaders);
962 atomic_set(v: &n_started, i: nreaders);
963 atomic_set(v: &n_warmedup, i: nreaders);
964 atomic_set(v: &n_cooleddown, i: nreaders);
965
966 exp_idx = exp;
967
968 for (r = 0; r < nreaders; r++) {
969 smp_store_release(&reader_tasks[r].start_reader, 1);
970 wake_up(&reader_tasks[r].wq);
971 }
972
973 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
974 nreaders);
975
976 wait_event(main_wq,
977 !atomic_read(&nreaders_exp) || torture_must_stop());
978
979 VERBOSE_SCALEOUT("main_func: experiment ended");
980
981 if (torture_must_stop())
982 goto end;
983
984 result_avg[exp] = div_u64(dividend: 1000 * process_durations(n: nreaders), divisor: nreaders * loops);
985 }
986
987 // Print the average of all experiments
988 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
989
990 pr_alert("Runs\tTime(ns)\n");
991 for (exp = 0; exp < nruns; exp++) {
992 u64 avg;
993 u32 rem;
994
995 avg = div_u64_rem(dividend: result_avg[exp], divisor: 1000, remainder: &rem);
996 sprintf(buf: buf1, fmt: "%d\t%llu.%03u\n", exp + 1, avg, rem);
997 strcat(p: buf, q: buf1);
998 if (strlen(buf) >= 800) {
999 pr_alert("%s", buf);
1000 buf[0] = 0;
1001 }
1002 }
1003
1004 pr_alert("%s", buf);
1005
1006oom_exit:
1007 // This will shutdown everything including us.
1008 if (shutdown) {
1009 shutdown_start = 1;
1010 wake_up(&shutdown_wq);
1011 }
1012
1013 // Wait for torture to stop us
1014 while (!torture_must_stop())
1015 schedule_timeout_uninterruptible(timeout: 1);
1016
1017end:
1018 torture_kthread_stopping(title: "main_func");
1019 kfree(objp: result_avg);
1020 kfree(objp: buf);
1021 return 0;
1022}
1023
1024static void
1025ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
1026{
1027 pr_alert("%s" SCALE_FLAG
1028 "--- %s: verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
1029 verbose, verbose_batched, shutdown, holdoff, lookup_instances, loops, nreaders, nruns, readdelay);
1030}
1031
1032static void
1033ref_scale_cleanup(void)
1034{
1035 int i;
1036
1037 if (torture_cleanup_begin())
1038 return;
1039
1040 if (!cur_ops) {
1041 torture_cleanup_end();
1042 return;
1043 }
1044
1045 if (reader_tasks) {
1046 for (i = 0; i < nreaders; i++)
1047 torture_stop_kthread("ref_scale_reader",
1048 reader_tasks[i].task);
1049 }
1050 kfree(objp: reader_tasks);
1051
1052 torture_stop_kthread("main_task", main_task);
1053 kfree(objp: main_task);
1054
1055 // Do scale-type-specific cleanup operations.
1056 if (cur_ops->cleanup != NULL)
1057 cur_ops->cleanup();
1058
1059 torture_cleanup_end();
1060}
1061
1062// Shutdown kthread. Just waits to be awakened, then shuts down system.
1063static int
1064ref_scale_shutdown(void *arg)
1065{
1066 wait_event_idle(shutdown_wq, shutdown_start);
1067
1068 smp_mb(); // Wake before output.
1069 ref_scale_cleanup();
1070 kernel_power_off();
1071
1072 return -EINVAL;
1073}
1074
1075static int __init
1076ref_scale_init(void)
1077{
1078 long i;
1079 int firsterr = 0;
1080 static struct ref_scale_ops *scale_ops[] = {
1081 &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
1082 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops,
1083 &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
1084 };
1085
1086 if (!torture_init_begin(ttype: scale_type, v: verbose))
1087 return -EBUSY;
1088
1089 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
1090 cur_ops = scale_ops[i];
1091 if (strcmp(scale_type, cur_ops->name) == 0)
1092 break;
1093 }
1094 if (i == ARRAY_SIZE(scale_ops)) {
1095 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
1096 pr_alert("rcu-scale types:");
1097 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
1098 pr_cont(" %s", scale_ops[i]->name);
1099 pr_cont("\n");
1100 firsterr = -EINVAL;
1101 cur_ops = NULL;
1102 goto unwind;
1103 }
1104 if (cur_ops->init)
1105 if (!cur_ops->init()) {
1106 firsterr = -EUCLEAN;
1107 goto unwind;
1108 }
1109
1110 ref_scale_print_module_parms(cur_ops, tag: "Start of test");
1111
1112 // Shutdown task
1113 if (shutdown) {
1114 init_waitqueue_head(&shutdown_wq);
1115 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
1116 shutdown_task);
1117 if (torture_init_error(firsterr))
1118 goto unwind;
1119 schedule_timeout_uninterruptible(timeout: 1);
1120 }
1121
1122 // Reader tasks (default to ~75% of online CPUs).
1123 if (nreaders < 0)
1124 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
1125 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
1126 loops = 1;
1127 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
1128 nreaders = 1;
1129 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
1130 nruns = 1;
1131 reader_tasks = kcalloc(n: nreaders, size: sizeof(reader_tasks[0]),
1132 GFP_KERNEL);
1133 if (!reader_tasks) {
1134 SCALEOUT_ERRSTRING("out of memory");
1135 firsterr = -ENOMEM;
1136 goto unwind;
1137 }
1138
1139 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
1140
1141 for (i = 0; i < nreaders; i++) {
1142 init_waitqueue_head(&reader_tasks[i].wq);
1143 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
1144 reader_tasks[i].task);
1145 if (torture_init_error(firsterr))
1146 goto unwind;
1147 }
1148
1149 // Main Task
1150 init_waitqueue_head(&main_wq);
1151 firsterr = torture_create_kthread(main_func, NULL, main_task);
1152 if (torture_init_error(firsterr))
1153 goto unwind;
1154
1155 torture_init_end();
1156 return 0;
1157
1158unwind:
1159 torture_init_end();
1160 ref_scale_cleanup();
1161 if (shutdown) {
1162 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
1163 kernel_power_off();
1164 }
1165 return firsterr;
1166}
1167
1168module_init(ref_scale_init);
1169module_exit(ref_scale_cleanup);
1170

source code of linux/kernel/rcu/refscale.c