1// SPDX-License-Identifier: MIT
2
3/*
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/delay.h>
8#include <linux/dma-fence.h>
9#include <linux/dma-fence-chain.h>
10#include <linux/kernel.h>
11#include <linux/kthread.h>
12#include <linux/mm.h>
13#include <linux/sched/signal.h>
14#include <linux/slab.h>
15#include <linux/spinlock.h>
16#include <linux/random.h>
17
18#include "selftest.h"
19
20#define CHAIN_SZ (4 << 10)
21
22static struct kmem_cache *slab_fences;
23
24static inline struct mock_fence {
25 struct dma_fence base;
26 spinlock_t lock;
27} *to_mock_fence(struct dma_fence *f) {
28 return container_of(f, struct mock_fence, base);
29}
30
31static const char *mock_name(struct dma_fence *f)
32{
33 return "mock";
34}
35
36static void mock_fence_release(struct dma_fence *f)
37{
38 kmem_cache_free(s: slab_fences, objp: to_mock_fence(f));
39}
40
41static const struct dma_fence_ops mock_ops = {
42 .get_driver_name = mock_name,
43 .get_timeline_name = mock_name,
44 .release = mock_fence_release,
45};
46
47static struct dma_fence *mock_fence(void)
48{
49 struct mock_fence *f;
50
51 f = kmem_cache_alloc(cachep: slab_fences, GFP_KERNEL);
52 if (!f)
53 return NULL;
54
55 spin_lock_init(&f->lock);
56 dma_fence_init(fence: &f->base, ops: &mock_ops, lock: &f->lock, context: 0, seqno: 0);
57
58 return &f->base;
59}
60
61static struct dma_fence *mock_chain(struct dma_fence *prev,
62 struct dma_fence *fence,
63 u64 seqno)
64{
65 struct dma_fence_chain *f;
66
67 f = dma_fence_chain_alloc();
68 if (!f)
69 return NULL;
70
71 dma_fence_chain_init(chain: f, prev: dma_fence_get(fence: prev), fence: dma_fence_get(fence),
72 seqno);
73
74 return &f->base;
75}
76
77static int sanitycheck(void *arg)
78{
79 struct dma_fence *f, *chain;
80 int err = 0;
81
82 f = mock_fence();
83 if (!f)
84 return -ENOMEM;
85
86 chain = mock_chain(NULL, fence: f, seqno: 1);
87 if (!chain)
88 err = -ENOMEM;
89
90 dma_fence_enable_sw_signaling(fence: chain);
91
92 dma_fence_signal(fence: f);
93 dma_fence_put(fence: f);
94
95 dma_fence_put(fence: chain);
96
97 return err;
98}
99
100struct fence_chains {
101 unsigned int chain_length;
102 struct dma_fence **fences;
103 struct dma_fence **chains;
104
105 struct dma_fence *tail;
106};
107
108static uint64_t seqno_inc(unsigned int i)
109{
110 return i + 1;
111}
112
113static int fence_chains_init(struct fence_chains *fc, unsigned int count,
114 uint64_t (*seqno_fn)(unsigned int))
115{
116 unsigned int i;
117 int err = 0;
118
119 fc->chains = kvmalloc_array(n: count, size: sizeof(*fc->chains),
120 GFP_KERNEL | __GFP_ZERO);
121 if (!fc->chains)
122 return -ENOMEM;
123
124 fc->fences = kvmalloc_array(n: count, size: sizeof(*fc->fences),
125 GFP_KERNEL | __GFP_ZERO);
126 if (!fc->fences) {
127 err = -ENOMEM;
128 goto err_chains;
129 }
130
131 fc->tail = NULL;
132 for (i = 0; i < count; i++) {
133 fc->fences[i] = mock_fence();
134 if (!fc->fences[i]) {
135 err = -ENOMEM;
136 goto unwind;
137 }
138
139 fc->chains[i] = mock_chain(prev: fc->tail,
140 fence: fc->fences[i],
141 seqno: seqno_fn(i));
142 if (!fc->chains[i]) {
143 err = -ENOMEM;
144 goto unwind;
145 }
146
147 fc->tail = fc->chains[i];
148
149 dma_fence_enable_sw_signaling(fence: fc->chains[i]);
150 }
151
152 fc->chain_length = i;
153 return 0;
154
155unwind:
156 for (i = 0; i < count; i++) {
157 dma_fence_put(fence: fc->fences[i]);
158 dma_fence_put(fence: fc->chains[i]);
159 }
160 kvfree(addr: fc->fences);
161err_chains:
162 kvfree(addr: fc->chains);
163 return err;
164}
165
166static void fence_chains_fini(struct fence_chains *fc)
167{
168 unsigned int i;
169
170 for (i = 0; i < fc->chain_length; i++) {
171 dma_fence_signal(fence: fc->fences[i]);
172 dma_fence_put(fence: fc->fences[i]);
173 }
174 kvfree(addr: fc->fences);
175
176 for (i = 0; i < fc->chain_length; i++)
177 dma_fence_put(fence: fc->chains[i]);
178 kvfree(addr: fc->chains);
179}
180
181static int find_seqno(void *arg)
182{
183 struct fence_chains fc;
184 struct dma_fence *fence;
185 int err;
186 int i;
187
188 err = fence_chains_init(fc: &fc, count: 64, seqno_fn: seqno_inc);
189 if (err)
190 return err;
191
192 fence = dma_fence_get(fence: fc.tail);
193 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: 0);
194 dma_fence_put(fence);
195 if (err) {
196 pr_err("Reported %d for find_seqno(0)!\n", err);
197 goto err;
198 }
199
200 for (i = 0; i < fc.chain_length; i++) {
201 fence = dma_fence_get(fence: fc.tail);
202 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: i + 1);
203 dma_fence_put(fence);
204 if (err) {
205 pr_err("Reported %d for find_seqno(%d:%d)!\n",
206 err, fc.chain_length + 1, i + 1);
207 goto err;
208 }
209 if (fence != fc.chains[i]) {
210 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
211 fc.chain_length + 1, i + 1);
212 err = -EINVAL;
213 goto err;
214 }
215
216 dma_fence_get(fence);
217 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: i + 1);
218 dma_fence_put(fence);
219 if (err) {
220 pr_err("Error reported for finding self\n");
221 goto err;
222 }
223 if (fence != fc.chains[i]) {
224 pr_err("Incorrect fence reported by find self\n");
225 err = -EINVAL;
226 goto err;
227 }
228
229 dma_fence_get(fence);
230 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: i + 2);
231 dma_fence_put(fence);
232 if (!err) {
233 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
234 i + 1, i + 2);
235 err = -EINVAL;
236 goto err;
237 }
238
239 dma_fence_get(fence);
240 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: i);
241 dma_fence_put(fence);
242 if (err) {
243 pr_err("Error reported for previous fence!\n");
244 goto err;
245 }
246 if (i > 0 && fence != fc.chains[i - 1]) {
247 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
248 i + 1, i);
249 err = -EINVAL;
250 goto err;
251 }
252 }
253
254err:
255 fence_chains_fini(fc: &fc);
256 return err;
257}
258
259static int find_signaled(void *arg)
260{
261 struct fence_chains fc;
262 struct dma_fence *fence;
263 int err;
264
265 err = fence_chains_init(fc: &fc, count: 2, seqno_fn: seqno_inc);
266 if (err)
267 return err;
268
269 dma_fence_signal(fence: fc.fences[0]);
270
271 fence = dma_fence_get(fence: fc.tail);
272 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: 1);
273 dma_fence_put(fence);
274 if (err) {
275 pr_err("Reported %d for find_seqno()!\n", err);
276 goto err;
277 }
278
279 if (fence && fence != fc.chains[0]) {
280 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
281 fence->seqno);
282
283 dma_fence_get(fence);
284 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: 1);
285 dma_fence_put(fence);
286 if (err)
287 pr_err("Reported %d for finding self!\n", err);
288
289 err = -EINVAL;
290 }
291
292err:
293 fence_chains_fini(fc: &fc);
294 return err;
295}
296
297static int find_out_of_order(void *arg)
298{
299 struct fence_chains fc;
300 struct dma_fence *fence;
301 int err;
302
303 err = fence_chains_init(fc: &fc, count: 3, seqno_fn: seqno_inc);
304 if (err)
305 return err;
306
307 dma_fence_signal(fence: fc.fences[1]);
308
309 fence = dma_fence_get(fence: fc.tail);
310 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: 2);
311 dma_fence_put(fence);
312 if (err) {
313 pr_err("Reported %d for find_seqno()!\n", err);
314 goto err;
315 }
316
317 /*
318 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior
319 * of the dma-fence-chain is to make us wait for all the fences up to
320 * the point we want. Since fence 1 is still not signaled, this what
321 * we should get as fence to wait upon (fence 2 being garbage
322 * collected during the traversal of the chain).
323 */
324 if (fence != fc.chains[0]) {
325 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
326 fence ? fence->seqno : 0);
327
328 err = -EINVAL;
329 }
330
331err:
332 fence_chains_fini(fc: &fc);
333 return err;
334}
335
336static uint64_t seqno_inc2(unsigned int i)
337{
338 return 2 * i + 2;
339}
340
341static int find_gap(void *arg)
342{
343 struct fence_chains fc;
344 struct dma_fence *fence;
345 int err;
346 int i;
347
348 err = fence_chains_init(fc: &fc, count: 64, seqno_fn: seqno_inc2);
349 if (err)
350 return err;
351
352 for (i = 0; i < fc.chain_length; i++) {
353 fence = dma_fence_get(fence: fc.tail);
354 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: 2 * i + 1);
355 dma_fence_put(fence);
356 if (err) {
357 pr_err("Reported %d for find_seqno(%d:%d)!\n",
358 err, fc.chain_length + 1, 2 * i + 1);
359 goto err;
360 }
361 if (fence != fc.chains[i]) {
362 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
363 fence->seqno,
364 fc.chain_length + 1,
365 2 * i + 1);
366 err = -EINVAL;
367 goto err;
368 }
369
370 dma_fence_get(fence);
371 err = dma_fence_chain_find_seqno(pfence: &fence, seqno: 2 * i + 2);
372 dma_fence_put(fence);
373 if (err) {
374 pr_err("Error reported for finding self\n");
375 goto err;
376 }
377 if (fence != fc.chains[i]) {
378 pr_err("Incorrect fence reported by find self\n");
379 err = -EINVAL;
380 goto err;
381 }
382 }
383
384err:
385 fence_chains_fini(fc: &fc);
386 return err;
387}
388
389struct find_race {
390 struct fence_chains fc;
391 atomic_t children;
392};
393
394static int __find_race(void *arg)
395{
396 struct find_race *data = arg;
397 int err = 0;
398
399 while (!kthread_should_stop()) {
400 struct dma_fence *fence = dma_fence_get(fence: data->fc.tail);
401 int seqno;
402
403 seqno = get_random_u32_inclusive(floor: 1, ceil: data->fc.chain_length);
404
405 err = dma_fence_chain_find_seqno(pfence: &fence, seqno);
406 if (err) {
407 pr_err("Failed to find fence seqno:%d\n",
408 seqno);
409 dma_fence_put(fence);
410 break;
411 }
412 if (!fence)
413 goto signal;
414
415 /*
416 * We can only find ourselves if we are on fence we were
417 * looking for.
418 */
419 if (fence->seqno == seqno) {
420 err = dma_fence_chain_find_seqno(pfence: &fence, seqno);
421 if (err) {
422 pr_err("Reported an invalid fence for find-self:%d\n",
423 seqno);
424 dma_fence_put(fence);
425 break;
426 }
427 }
428
429 dma_fence_put(fence);
430
431signal:
432 seqno = get_random_u32_below(ceil: data->fc.chain_length - 1);
433 dma_fence_signal(fence: data->fc.fences[seqno]);
434 cond_resched();
435 }
436
437 if (atomic_dec_and_test(v: &data->children))
438 wake_up_var(var: &data->children);
439 return err;
440}
441
442static int find_race(void *arg)
443{
444 struct find_race data;
445 int ncpus = num_online_cpus();
446 struct task_struct **threads;
447 unsigned long count;
448 int err;
449 int i;
450
451 err = fence_chains_init(fc: &data.fc, CHAIN_SZ, seqno_fn: seqno_inc);
452 if (err)
453 return err;
454
455 threads = kmalloc_array(n: ncpus, size: sizeof(*threads), GFP_KERNEL);
456 if (!threads) {
457 err = -ENOMEM;
458 goto err;
459 }
460
461 atomic_set(v: &data.children, i: 0);
462 for (i = 0; i < ncpus; i++) {
463 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);
464 if (IS_ERR(ptr: threads[i])) {
465 ncpus = i;
466 break;
467 }
468 atomic_inc(v: &data.children);
469 get_task_struct(t: threads[i]);
470 }
471
472 wait_var_event_timeout(&data.children,
473 !atomic_read(&data.children),
474 5 * HZ);
475
476 for (i = 0; i < ncpus; i++) {
477 int ret;
478
479 ret = kthread_stop_put(k: threads[i]);
480 if (ret && !err)
481 err = ret;
482 }
483 kfree(objp: threads);
484
485 count = 0;
486 for (i = 0; i < data.fc.chain_length; i++)
487 if (dma_fence_is_signaled(fence: data.fc.fences[i]))
488 count++;
489 pr_info("Completed %lu cycles\n", count);
490
491err:
492 fence_chains_fini(fc: &data.fc);
493 return err;
494}
495
496static int signal_forward(void *arg)
497{
498 struct fence_chains fc;
499 int err;
500 int i;
501
502 err = fence_chains_init(fc: &fc, count: 64, seqno_fn: seqno_inc);
503 if (err)
504 return err;
505
506 for (i = 0; i < fc.chain_length; i++) {
507 dma_fence_signal(fence: fc.fences[i]);
508
509 if (!dma_fence_is_signaled(fence: fc.chains[i])) {
510 pr_err("chain[%d] not signaled!\n", i);
511 err = -EINVAL;
512 goto err;
513 }
514
515 if (i + 1 < fc.chain_length &&
516 dma_fence_is_signaled(fence: fc.chains[i + 1])) {
517 pr_err("chain[%d] is signaled!\n", i);
518 err = -EINVAL;
519 goto err;
520 }
521 }
522
523err:
524 fence_chains_fini(fc: &fc);
525 return err;
526}
527
528static int signal_backward(void *arg)
529{
530 struct fence_chains fc;
531 int err;
532 int i;
533
534 err = fence_chains_init(fc: &fc, count: 64, seqno_fn: seqno_inc);
535 if (err)
536 return err;
537
538 for (i = fc.chain_length; i--; ) {
539 dma_fence_signal(fence: fc.fences[i]);
540
541 if (i > 0 && dma_fence_is_signaled(fence: fc.chains[i])) {
542 pr_err("chain[%d] is signaled!\n", i);
543 err = -EINVAL;
544 goto err;
545 }
546 }
547
548 for (i = 0; i < fc.chain_length; i++) {
549 if (!dma_fence_is_signaled(fence: fc.chains[i])) {
550 pr_err("chain[%d] was not signaled!\n", i);
551 err = -EINVAL;
552 goto err;
553 }
554 }
555
556err:
557 fence_chains_fini(fc: &fc);
558 return err;
559}
560
561static int __wait_fence_chains(void *arg)
562{
563 struct fence_chains *fc = arg;
564
565 if (dma_fence_wait(fence: fc->tail, intr: false))
566 return -EIO;
567
568 return 0;
569}
570
571static int wait_forward(void *arg)
572{
573 struct fence_chains fc;
574 struct task_struct *tsk;
575 int err;
576 int i;
577
578 err = fence_chains_init(fc: &fc, CHAIN_SZ, seqno_fn: seqno_inc);
579 if (err)
580 return err;
581
582 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
583 if (IS_ERR(ptr: tsk)) {
584 err = PTR_ERR(ptr: tsk);
585 goto err;
586 }
587 get_task_struct(t: tsk);
588 yield_to(p: tsk, preempt: true);
589
590 for (i = 0; i < fc.chain_length; i++)
591 dma_fence_signal(fence: fc.fences[i]);
592
593 err = kthread_stop_put(k: tsk);
594
595err:
596 fence_chains_fini(fc: &fc);
597 return err;
598}
599
600static int wait_backward(void *arg)
601{
602 struct fence_chains fc;
603 struct task_struct *tsk;
604 int err;
605 int i;
606
607 err = fence_chains_init(fc: &fc, CHAIN_SZ, seqno_fn: seqno_inc);
608 if (err)
609 return err;
610
611 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
612 if (IS_ERR(ptr: tsk)) {
613 err = PTR_ERR(ptr: tsk);
614 goto err;
615 }
616 get_task_struct(t: tsk);
617 yield_to(p: tsk, preempt: true);
618
619 for (i = fc.chain_length; i--; )
620 dma_fence_signal(fence: fc.fences[i]);
621
622 err = kthread_stop_put(k: tsk);
623
624err:
625 fence_chains_fini(fc: &fc);
626 return err;
627}
628
629static void randomise_fences(struct fence_chains *fc)
630{
631 unsigned int count = fc->chain_length;
632
633 /* Fisher-Yates shuffle courtesy of Knuth */
634 while (--count) {
635 unsigned int swp;
636
637 swp = get_random_u32_below(ceil: count + 1);
638 if (swp == count)
639 continue;
640
641 swap(fc->fences[count], fc->fences[swp]);
642 }
643}
644
645static int wait_random(void *arg)
646{
647 struct fence_chains fc;
648 struct task_struct *tsk;
649 int err;
650 int i;
651
652 err = fence_chains_init(fc: &fc, CHAIN_SZ, seqno_fn: seqno_inc);
653 if (err)
654 return err;
655
656 randomise_fences(fc: &fc);
657
658 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
659 if (IS_ERR(ptr: tsk)) {
660 err = PTR_ERR(ptr: tsk);
661 goto err;
662 }
663 get_task_struct(t: tsk);
664 yield_to(p: tsk, preempt: true);
665
666 for (i = 0; i < fc.chain_length; i++)
667 dma_fence_signal(fence: fc.fences[i]);
668
669 err = kthread_stop_put(k: tsk);
670
671err:
672 fence_chains_fini(fc: &fc);
673 return err;
674}
675
676int dma_fence_chain(void)
677{
678 static const struct subtest tests[] = {
679 SUBTEST(sanitycheck),
680 SUBTEST(find_seqno),
681 SUBTEST(find_signaled),
682 SUBTEST(find_out_of_order),
683 SUBTEST(find_gap),
684 SUBTEST(find_race),
685 SUBTEST(signal_forward),
686 SUBTEST(signal_backward),
687 SUBTEST(wait_forward),
688 SUBTEST(wait_backward),
689 SUBTEST(wait_random),
690 };
691 int ret;
692
693 pr_info("sizeof(dma_fence_chain)=%zu\n",
694 sizeof(struct dma_fence_chain));
695
696 slab_fences = KMEM_CACHE(mock_fence,
697 SLAB_TYPESAFE_BY_RCU |
698 SLAB_HWCACHE_ALIGN);
699 if (!slab_fences)
700 return -ENOMEM;
701
702 ret = subtests(tests, NULL);
703
704 kmem_cache_destroy(s: slab_fences);
705 return ret;
706}
707

source code of linux/drivers/dma-buf/st-dma-fence-chain.c