1/* SPDX-License-Identifier: MIT */
2
3/*
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/delay.h>
8#include <linux/dma-fence.h>
9#include <linux/kernel.h>
10#include <linux/kthread.h>
11#include <linux/sched/signal.h>
12#include <linux/slab.h>
13#include <linux/spinlock.h>
14
15#include "selftest.h"
16
17static struct kmem_cache *slab_fences;
18
19static struct mock_fence {
20 struct dma_fence base;
21 struct spinlock lock;
22} *to_mock_fence(struct dma_fence *f) {
23 return container_of(f, struct mock_fence, base);
24}
25
26static const char *mock_name(struct dma_fence *f)
27{
28 return "mock";
29}
30
31static void mock_fence_release(struct dma_fence *f)
32{
33 kmem_cache_free(s: slab_fences, objp: to_mock_fence(f));
34}
35
36struct wait_cb {
37 struct dma_fence_cb cb;
38 struct task_struct *task;
39};
40
41static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
42{
43 wake_up_process(container_of(cb, struct wait_cb, cb)->task);
44}
45
46static long mock_wait(struct dma_fence *f, bool intr, long timeout)
47{
48 const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
49 struct wait_cb cb = { .task = current };
50
51 if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: mock_wakeup))
52 return timeout;
53
54 while (timeout) {
55 set_current_state(state);
56
57 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
58 break;
59
60 if (signal_pending_state(state, current))
61 break;
62
63 timeout = schedule_timeout(timeout);
64 }
65 __set_current_state(TASK_RUNNING);
66
67 if (!dma_fence_remove_callback(fence: f, cb: &cb.cb))
68 return timeout;
69
70 if (signal_pending_state(state, current))
71 return -ERESTARTSYS;
72
73 return -ETIME;
74}
75
76static const struct dma_fence_ops mock_ops = {
77 .get_driver_name = mock_name,
78 .get_timeline_name = mock_name,
79 .wait = mock_wait,
80 .release = mock_fence_release,
81};
82
83static struct dma_fence *mock_fence(void)
84{
85 struct mock_fence *f;
86
87 f = kmem_cache_alloc(cachep: slab_fences, GFP_KERNEL);
88 if (!f)
89 return NULL;
90
91 spin_lock_init(&f->lock);
92 dma_fence_init(fence: &f->base, ops: &mock_ops, lock: &f->lock, context: 0, seqno: 0);
93
94 return &f->base;
95}
96
97static int sanitycheck(void *arg)
98{
99 struct dma_fence *f;
100
101 f = mock_fence();
102 if (!f)
103 return -ENOMEM;
104
105 dma_fence_enable_sw_signaling(fence: f);
106
107 dma_fence_signal(fence: f);
108 dma_fence_put(fence: f);
109
110 return 0;
111}
112
113static int test_signaling(void *arg)
114{
115 struct dma_fence *f;
116 int err = -EINVAL;
117
118 f = mock_fence();
119 if (!f)
120 return -ENOMEM;
121
122 dma_fence_enable_sw_signaling(fence: f);
123
124 if (dma_fence_is_signaled(fence: f)) {
125 pr_err("Fence unexpectedly signaled on creation\n");
126 goto err_free;
127 }
128
129 if (dma_fence_signal(fence: f)) {
130 pr_err("Fence reported being already signaled\n");
131 goto err_free;
132 }
133
134 if (!dma_fence_is_signaled(fence: f)) {
135 pr_err("Fence not reporting signaled\n");
136 goto err_free;
137 }
138
139 if (!dma_fence_signal(fence: f)) {
140 pr_err("Fence reported not being already signaled\n");
141 goto err_free;
142 }
143
144 err = 0;
145err_free:
146 dma_fence_put(fence: f);
147 return err;
148}
149
150struct simple_cb {
151 struct dma_fence_cb cb;
152 bool seen;
153};
154
155static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
156{
157 smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
158}
159
160static int test_add_callback(void *arg)
161{
162 struct simple_cb cb = {};
163 struct dma_fence *f;
164 int err = -EINVAL;
165
166 f = mock_fence();
167 if (!f)
168 return -ENOMEM;
169
170 if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) {
171 pr_err("Failed to add callback, fence already signaled!\n");
172 goto err_free;
173 }
174
175 dma_fence_signal(fence: f);
176 if (!cb.seen) {
177 pr_err("Callback failed!\n");
178 goto err_free;
179 }
180
181 err = 0;
182err_free:
183 dma_fence_put(fence: f);
184 return err;
185}
186
187static int test_late_add_callback(void *arg)
188{
189 struct simple_cb cb = {};
190 struct dma_fence *f;
191 int err = -EINVAL;
192
193 f = mock_fence();
194 if (!f)
195 return -ENOMEM;
196
197 dma_fence_enable_sw_signaling(fence: f);
198
199 dma_fence_signal(fence: f);
200
201 if (!dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) {
202 pr_err("Added callback, but fence was already signaled!\n");
203 goto err_free;
204 }
205
206 dma_fence_signal(fence: f);
207 if (cb.seen) {
208 pr_err("Callback called after failed attachment !\n");
209 goto err_free;
210 }
211
212 err = 0;
213err_free:
214 dma_fence_put(fence: f);
215 return err;
216}
217
218static int test_rm_callback(void *arg)
219{
220 struct simple_cb cb = {};
221 struct dma_fence *f;
222 int err = -EINVAL;
223
224 f = mock_fence();
225 if (!f)
226 return -ENOMEM;
227
228 if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) {
229 pr_err("Failed to add callback, fence already signaled!\n");
230 goto err_free;
231 }
232
233 if (!dma_fence_remove_callback(fence: f, cb: &cb.cb)) {
234 pr_err("Failed to remove callback!\n");
235 goto err_free;
236 }
237
238 dma_fence_signal(fence: f);
239 if (cb.seen) {
240 pr_err("Callback still signaled after removal!\n");
241 goto err_free;
242 }
243
244 err = 0;
245err_free:
246 dma_fence_put(fence: f);
247 return err;
248}
249
250static int test_late_rm_callback(void *arg)
251{
252 struct simple_cb cb = {};
253 struct dma_fence *f;
254 int err = -EINVAL;
255
256 f = mock_fence();
257 if (!f)
258 return -ENOMEM;
259
260 if (dma_fence_add_callback(fence: f, cb: &cb.cb, func: simple_callback)) {
261 pr_err("Failed to add callback, fence already signaled!\n");
262 goto err_free;
263 }
264
265 dma_fence_signal(fence: f);
266 if (!cb.seen) {
267 pr_err("Callback failed!\n");
268 goto err_free;
269 }
270
271 if (dma_fence_remove_callback(fence: f, cb: &cb.cb)) {
272 pr_err("Callback removal succeed after being executed!\n");
273 goto err_free;
274 }
275
276 err = 0;
277err_free:
278 dma_fence_put(fence: f);
279 return err;
280}
281
282static int test_status(void *arg)
283{
284 struct dma_fence *f;
285 int err = -EINVAL;
286
287 f = mock_fence();
288 if (!f)
289 return -ENOMEM;
290
291 dma_fence_enable_sw_signaling(fence: f);
292
293 if (dma_fence_get_status(fence: f)) {
294 pr_err("Fence unexpectedly has signaled status on creation\n");
295 goto err_free;
296 }
297
298 dma_fence_signal(fence: f);
299 if (!dma_fence_get_status(fence: f)) {
300 pr_err("Fence not reporting signaled status\n");
301 goto err_free;
302 }
303
304 err = 0;
305err_free:
306 dma_fence_put(fence: f);
307 return err;
308}
309
310static int test_error(void *arg)
311{
312 struct dma_fence *f;
313 int err = -EINVAL;
314
315 f = mock_fence();
316 if (!f)
317 return -ENOMEM;
318
319 dma_fence_enable_sw_signaling(fence: f);
320
321 dma_fence_set_error(fence: f, error: -EIO);
322
323 if (dma_fence_get_status(fence: f)) {
324 pr_err("Fence unexpectedly has error status before signal\n");
325 goto err_free;
326 }
327
328 dma_fence_signal(fence: f);
329 if (dma_fence_get_status(fence: f) != -EIO) {
330 pr_err("Fence not reporting error status, got %d\n",
331 dma_fence_get_status(f));
332 goto err_free;
333 }
334
335 err = 0;
336err_free:
337 dma_fence_put(fence: f);
338 return err;
339}
340
341static int test_wait(void *arg)
342{
343 struct dma_fence *f;
344 int err = -EINVAL;
345
346 f = mock_fence();
347 if (!f)
348 return -ENOMEM;
349
350 dma_fence_enable_sw_signaling(fence: f);
351
352 if (dma_fence_wait_timeout(f, intr: false, timeout: 0) != -ETIME) {
353 pr_err("Wait reported complete before being signaled\n");
354 goto err_free;
355 }
356
357 dma_fence_signal(fence: f);
358
359 if (dma_fence_wait_timeout(f, intr: false, timeout: 0) != 0) {
360 pr_err("Wait reported incomplete after being signaled\n");
361 goto err_free;
362 }
363
364 err = 0;
365err_free:
366 dma_fence_signal(fence: f);
367 dma_fence_put(fence: f);
368 return err;
369}
370
371struct wait_timer {
372 struct timer_list timer;
373 struct dma_fence *f;
374};
375
376static void wait_timer(struct timer_list *timer)
377{
378 struct wait_timer *wt = from_timer(wt, timer, timer);
379
380 dma_fence_signal(fence: wt->f);
381}
382
383static int test_wait_timeout(void *arg)
384{
385 struct wait_timer wt;
386 int err = -EINVAL;
387
388 timer_setup_on_stack(&wt.timer, wait_timer, 0);
389
390 wt.f = mock_fence();
391 if (!wt.f)
392 return -ENOMEM;
393
394 dma_fence_enable_sw_signaling(fence: wt.f);
395
396 if (dma_fence_wait_timeout(wt.f, intr: false, timeout: 1) != -ETIME) {
397 pr_err("Wait reported complete before being signaled\n");
398 goto err_free;
399 }
400
401 mod_timer(timer: &wt.timer, expires: jiffies + 1);
402
403 if (dma_fence_wait_timeout(wt.f, intr: false, timeout: 2) == -ETIME) {
404 if (timer_pending(timer: &wt.timer)) {
405 pr_notice("Timer did not fire within the jiffie!\n");
406 err = 0; /* not our fault! */
407 } else {
408 pr_err("Wait reported incomplete after timeout\n");
409 }
410 goto err_free;
411 }
412
413 err = 0;
414err_free:
415 del_timer_sync(timer: &wt.timer);
416 destroy_timer_on_stack(timer: &wt.timer);
417 dma_fence_signal(fence: wt.f);
418 dma_fence_put(fence: wt.f);
419 return err;
420}
421
422static int test_stub(void *arg)
423{
424 struct dma_fence *f[64];
425 int err = -EINVAL;
426 int i;
427
428 for (i = 0; i < ARRAY_SIZE(f); i++) {
429 f[i] = dma_fence_get_stub();
430 if (!dma_fence_is_signaled(fence: f[i])) {
431 pr_err("Obtained unsignaled stub fence!\n");
432 goto err;
433 }
434 }
435
436 err = 0;
437err:
438 while (i--)
439 dma_fence_put(fence: f[i]);
440 return err;
441}
442
443/* Now off to the races! */
444
445struct race_thread {
446 struct dma_fence __rcu **fences;
447 struct task_struct *task;
448 bool before;
449 int id;
450};
451
452static void __wait_for_callbacks(struct dma_fence *f)
453{
454 spin_lock_irq(lock: f->lock);
455 spin_unlock_irq(lock: f->lock);
456}
457
458static int thread_signal_callback(void *arg)
459{
460 const struct race_thread *t = arg;
461 unsigned long pass = 0;
462 unsigned long miss = 0;
463 int err = 0;
464
465 while (!err && !kthread_should_stop()) {
466 struct dma_fence *f1, *f2;
467 struct simple_cb cb;
468
469 f1 = mock_fence();
470 if (!f1) {
471 err = -ENOMEM;
472 break;
473 }
474
475 dma_fence_enable_sw_signaling(fence: f1);
476
477 rcu_assign_pointer(t->fences[t->id], f1);
478 smp_wmb();
479
480 rcu_read_lock();
481 do {
482 f2 = dma_fence_get_rcu_safe(fencep: &t->fences[!t->id]);
483 } while (!f2 && !kthread_should_stop());
484 rcu_read_unlock();
485
486 if (t->before)
487 dma_fence_signal(fence: f1);
488
489 smp_store_mb(cb.seen, false);
490 if (!f2 ||
491 dma_fence_add_callback(fence: f2, cb: &cb.cb, func: simple_callback)) {
492 miss++;
493 cb.seen = true;
494 }
495
496 if (!t->before)
497 dma_fence_signal(fence: f1);
498
499 if (!cb.seen) {
500 dma_fence_wait(fence: f2, intr: false);
501 __wait_for_callbacks(f: f2);
502 }
503
504 if (!READ_ONCE(cb.seen)) {
505 pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",
506 t->id, pass, miss,
507 t->before ? "before" : "after",
508 dma_fence_is_signaled(f2) ? "yes" : "no");
509 err = -EINVAL;
510 }
511
512 dma_fence_put(fence: f2);
513
514 rcu_assign_pointer(t->fences[t->id], NULL);
515 smp_wmb();
516
517 dma_fence_put(fence: f1);
518
519 pass++;
520 }
521
522 pr_info("%s[%d] completed %lu passes, %lu misses\n",
523 __func__, t->id, pass, miss);
524 return err;
525}
526
527static int race_signal_callback(void *arg)
528{
529 struct dma_fence __rcu *f[2] = {};
530 int ret = 0;
531 int pass;
532
533 for (pass = 0; !ret && pass <= 1; pass++) {
534 struct race_thread t[2];
535 int i;
536
537 for (i = 0; i < ARRAY_SIZE(t); i++) {
538 t[i].fences = f;
539 t[i].id = i;
540 t[i].before = pass;
541 t[i].task = kthread_run(thread_signal_callback, &t[i],
542 "dma-fence:%d", i);
543 get_task_struct(t: t[i].task);
544 }
545
546 msleep(msecs: 50);
547
548 for (i = 0; i < ARRAY_SIZE(t); i++) {
549 int err;
550
551 err = kthread_stop_put(k: t[i].task);
552 if (err && !ret)
553 ret = err;
554 }
555 }
556
557 return ret;
558}
559
560int dma_fence(void)
561{
562 static const struct subtest tests[] = {
563 SUBTEST(sanitycheck),
564 SUBTEST(test_signaling),
565 SUBTEST(test_add_callback),
566 SUBTEST(test_late_add_callback),
567 SUBTEST(test_rm_callback),
568 SUBTEST(test_late_rm_callback),
569 SUBTEST(test_status),
570 SUBTEST(test_error),
571 SUBTEST(test_wait),
572 SUBTEST(test_wait_timeout),
573 SUBTEST(test_stub),
574 SUBTEST(race_signal_callback),
575 };
576 int ret;
577
578 pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
579
580 slab_fences = KMEM_CACHE(mock_fence,
581 SLAB_TYPESAFE_BY_RCU |
582 SLAB_HWCACHE_ALIGN);
583 if (!slab_fences)
584 return -ENOMEM;
585
586 ret = subtests(tests, NULL);
587
588 kmem_cache_destroy(s: slab_fences);
589
590 return ret;
591}
592

source code of linux/drivers/dma-buf/st-dma-fence.c