1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * (C) Copyright 2016 Intel Corporation |
5 | */ |
6 | |
7 | #include <linux/slab.h> |
8 | #include <linux/dma-fence.h> |
9 | #include <linux/irq_work.h> |
10 | #include <linux/dma-resv.h> |
11 | |
12 | #include "i915_sw_fence.h" |
13 | #include "i915_selftest.h" |
14 | |
15 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) |
16 | #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) |
17 | #else |
18 | #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) |
19 | #endif |
20 | |
21 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
22 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
23 | #endif |
24 | |
25 | #define WQ_FLAG_BITS \ |
26 | BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags)) |
27 | |
28 | /* after WQ_FLAG_* for safety */ |
29 | #define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1) |
30 | #define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2) |
31 | |
32 | enum { |
33 | DEBUG_FENCE_IDLE = 0, |
34 | DEBUG_FENCE_NOTIFY, |
35 | }; |
36 | |
37 | static void *i915_sw_fence_debug_hint(void *addr) |
38 | { |
39 | return (void *)(((struct i915_sw_fence *)addr)->fn); |
40 | } |
41 | |
42 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
43 | |
44 | static const struct debug_obj_descr i915_sw_fence_debug_descr = { |
45 | .name = "i915_sw_fence" , |
46 | .debug_hint = i915_sw_fence_debug_hint, |
47 | }; |
48 | |
49 | static inline void debug_fence_init(struct i915_sw_fence *fence) |
50 | { |
51 | debug_object_init(addr: fence, descr: &i915_sw_fence_debug_descr); |
52 | } |
53 | |
54 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
55 | { |
56 | debug_object_init_on_stack(addr: fence, descr: &i915_sw_fence_debug_descr); |
57 | } |
58 | |
59 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
60 | { |
61 | debug_object_activate(addr: fence, descr: &i915_sw_fence_debug_descr); |
62 | } |
63 | |
64 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, |
65 | int old, int new) |
66 | { |
67 | debug_object_active_state(addr: fence, descr: &i915_sw_fence_debug_descr, expect: old, next: new); |
68 | } |
69 | |
70 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) |
71 | { |
72 | debug_object_deactivate(addr: fence, descr: &i915_sw_fence_debug_descr); |
73 | } |
74 | |
75 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) |
76 | { |
77 | debug_object_destroy(addr: fence, descr: &i915_sw_fence_debug_descr); |
78 | } |
79 | |
80 | static inline void debug_fence_free(struct i915_sw_fence *fence) |
81 | { |
82 | debug_object_free(addr: fence, descr: &i915_sw_fence_debug_descr); |
83 | smp_wmb(); /* flush the change in state before reallocation */ |
84 | } |
85 | |
86 | static inline void debug_fence_assert(struct i915_sw_fence *fence) |
87 | { |
88 | debug_object_assert_init(addr: fence, descr: &i915_sw_fence_debug_descr); |
89 | } |
90 | |
91 | #else |
92 | |
93 | static inline void debug_fence_init(struct i915_sw_fence *fence) |
94 | { |
95 | } |
96 | |
97 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
98 | { |
99 | } |
100 | |
101 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
102 | { |
103 | } |
104 | |
105 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, |
106 | int old, int new) |
107 | { |
108 | } |
109 | |
110 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) |
111 | { |
112 | } |
113 | |
114 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) |
115 | { |
116 | } |
117 | |
118 | static inline void debug_fence_free(struct i915_sw_fence *fence) |
119 | { |
120 | } |
121 | |
122 | static inline void debug_fence_assert(struct i915_sw_fence *fence) |
123 | { |
124 | } |
125 | |
126 | #endif |
127 | |
128 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
129 | enum i915_sw_fence_notify state) |
130 | { |
131 | return fence->fn(fence, state); |
132 | } |
133 | |
134 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
135 | void i915_sw_fence_fini(struct i915_sw_fence *fence) |
136 | { |
137 | debug_fence_free(fence); |
138 | } |
139 | #endif |
140 | |
141 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
142 | struct list_head *continuation) |
143 | { |
144 | wait_queue_head_t *x = &fence->wait; |
145 | wait_queue_entry_t *pos, *next; |
146 | unsigned long flags; |
147 | |
148 | debug_fence_deactivate(fence); |
149 | atomic_set_release(v: &fence->pending, i: -1); /* 0 -> -1 [done] */ |
150 | |
151 | /* |
152 | * To prevent unbounded recursion as we traverse the graph of |
153 | * i915_sw_fences, we move the entry list from this, the next ready |
154 | * fence, to the tail of the original fence's entry list |
155 | * (and so added to the list to be woken). |
156 | */ |
157 | |
158 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); |
159 | if (continuation) { |
160 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
161 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) |
162 | list_move_tail(list: &pos->entry, head: continuation); |
163 | else |
164 | pos->func(pos, TASK_NORMAL, 0, continuation); |
165 | } |
166 | } else { |
167 | LIST_HEAD(); |
168 | |
169 | do { |
170 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
171 | int wake_flags; |
172 | |
173 | wake_flags = 0; |
174 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) |
175 | wake_flags = fence->error; |
176 | |
177 | pos->func(pos, TASK_NORMAL, wake_flags, &extra); |
178 | } |
179 | |
180 | if (list_empty(head: &extra)) |
181 | break; |
182 | |
183 | list_splice_tail_init(list: &extra, head: &x->head); |
184 | } while (1); |
185 | } |
186 | spin_unlock_irqrestore(lock: &x->lock, flags); |
187 | |
188 | debug_fence_assert(fence); |
189 | } |
190 | |
191 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, |
192 | struct list_head *continuation) |
193 | { |
194 | debug_fence_assert(fence); |
195 | |
196 | if (!atomic_dec_and_test(v: &fence->pending)) |
197 | return; |
198 | |
199 | debug_fence_set_state(fence, old: DEBUG_FENCE_IDLE, new: DEBUG_FENCE_NOTIFY); |
200 | |
201 | if (__i915_sw_fence_notify(fence, state: FENCE_COMPLETE) != NOTIFY_DONE) |
202 | return; |
203 | |
204 | debug_fence_set_state(fence, old: DEBUG_FENCE_NOTIFY, new: DEBUG_FENCE_IDLE); |
205 | |
206 | __i915_sw_fence_wake_up_all(fence, continuation); |
207 | |
208 | debug_fence_destroy(fence); |
209 | __i915_sw_fence_notify(fence, state: FENCE_FREE); |
210 | } |
211 | |
212 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
213 | { |
214 | debug_fence_assert(fence); |
215 | |
216 | if (WARN_ON(i915_sw_fence_done(fence))) |
217 | return; |
218 | |
219 | __i915_sw_fence_complete(fence, NULL); |
220 | } |
221 | |
222 | bool i915_sw_fence_await(struct i915_sw_fence *fence) |
223 | { |
224 | int pending; |
225 | |
226 | /* |
227 | * It is only safe to add a new await to the fence while it has |
228 | * not yet been signaled (i.e. there are still existing signalers). |
229 | */ |
230 | pending = atomic_read(v: &fence->pending); |
231 | do { |
232 | if (pending < 1) |
233 | return false; |
234 | } while (!atomic_try_cmpxchg(v: &fence->pending, old: &pending, new: pending + 1)); |
235 | |
236 | return true; |
237 | } |
238 | |
239 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
240 | i915_sw_fence_notify_t fn, |
241 | const char *name, |
242 | struct lock_class_key *key) |
243 | { |
244 | __init_waitqueue_head(wq_head: &fence->wait, name, key); |
245 | fence->fn = fn; |
246 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
247 | fence->flags = 0; |
248 | #endif |
249 | |
250 | i915_sw_fence_reinit(fence); |
251 | } |
252 | |
253 | void i915_sw_fence_reinit(struct i915_sw_fence *fence) |
254 | { |
255 | debug_fence_init(fence); |
256 | |
257 | atomic_set(v: &fence->pending, i: 1); |
258 | fence->error = 0; |
259 | |
260 | I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); |
261 | } |
262 | |
263 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
264 | { |
265 | debug_fence_activate(fence); |
266 | i915_sw_fence_complete(fence); |
267 | } |
268 | |
269 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
270 | { |
271 | i915_sw_fence_set_error_once(fence: wq->private, error: flags); |
272 | |
273 | list_del(entry: &wq->entry); |
274 | __i915_sw_fence_complete(fence: wq->private, continuation: key); |
275 | |
276 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
277 | kfree(objp: wq); |
278 | return 0; |
279 | } |
280 | |
281 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
282 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, |
283 | const struct i915_sw_fence * const signaler) |
284 | { |
285 | wait_queue_entry_t *wq; |
286 | |
287 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) |
288 | return false; |
289 | |
290 | if (fence == signaler) |
291 | return true; |
292 | |
293 | list_for_each_entry(wq, &fence->wait.head, entry) { |
294 | if (wq->func != i915_sw_fence_wake) |
295 | continue; |
296 | |
297 | if (__i915_sw_fence_check_if_after(fence: wq->private, signaler)) |
298 | return true; |
299 | } |
300 | |
301 | return false; |
302 | } |
303 | |
304 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) |
305 | { |
306 | wait_queue_entry_t *wq; |
307 | |
308 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) |
309 | return; |
310 | |
311 | list_for_each_entry(wq, &fence->wait.head, entry) { |
312 | if (wq->func != i915_sw_fence_wake) |
313 | continue; |
314 | |
315 | __i915_sw_fence_clear_checked_bit(fence: wq->private); |
316 | } |
317 | } |
318 | |
319 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, |
320 | const struct i915_sw_fence * const signaler) |
321 | { |
322 | unsigned long flags; |
323 | bool err; |
324 | |
325 | spin_lock_irqsave(&i915_sw_fence_lock, flags); |
326 | err = __i915_sw_fence_check_if_after(fence, signaler); |
327 | __i915_sw_fence_clear_checked_bit(fence); |
328 | spin_unlock_irqrestore(lock: &i915_sw_fence_lock, flags); |
329 | |
330 | return err; |
331 | } |
332 | #else |
333 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, |
334 | const struct i915_sw_fence * const signaler) |
335 | { |
336 | return false; |
337 | } |
338 | #endif |
339 | |
340 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
341 | struct i915_sw_fence *signaler, |
342 | wait_queue_entry_t *wq, gfp_t gfp) |
343 | { |
344 | unsigned int pending; |
345 | unsigned long flags; |
346 | |
347 | debug_fence_assert(fence); |
348 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
349 | |
350 | if (i915_sw_fence_done(fence: signaler)) { |
351 | i915_sw_fence_set_error_once(fence, error: signaler->error); |
352 | return 0; |
353 | } |
354 | |
355 | debug_fence_assert(fence: signaler); |
356 | |
357 | /* The dependency graph must be acyclic. */ |
358 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) |
359 | return -EINVAL; |
360 | |
361 | pending = I915_SW_FENCE_FLAG_FENCE; |
362 | if (!wq) { |
363 | wq = kmalloc(size: sizeof(*wq), flags: gfp); |
364 | if (!wq) { |
365 | if (!gfpflags_allow_blocking(gfp_flags: gfp)) |
366 | return -ENOMEM; |
367 | |
368 | i915_sw_fence_wait(fence: signaler); |
369 | i915_sw_fence_set_error_once(fence, error: signaler->error); |
370 | return 0; |
371 | } |
372 | |
373 | pending |= I915_SW_FENCE_FLAG_ALLOC; |
374 | } |
375 | |
376 | INIT_LIST_HEAD(list: &wq->entry); |
377 | wq->flags = pending; |
378 | wq->func = i915_sw_fence_wake; |
379 | wq->private = fence; |
380 | |
381 | i915_sw_fence_await(fence); |
382 | |
383 | spin_lock_irqsave(&signaler->wait.lock, flags); |
384 | if (likely(!i915_sw_fence_done(signaler))) { |
385 | __add_wait_queue_entry_tail(wq_head: &signaler->wait, wq_entry: wq); |
386 | pending = 1; |
387 | } else { |
388 | i915_sw_fence_wake(wq, mode: 0, flags: signaler->error, NULL); |
389 | pending = 0; |
390 | } |
391 | spin_unlock_irqrestore(lock: &signaler->wait.lock, flags); |
392 | |
393 | return pending; |
394 | } |
395 | |
396 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
397 | struct i915_sw_fence *signaler, |
398 | wait_queue_entry_t *wq) |
399 | { |
400 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, gfp: 0); |
401 | } |
402 | |
403 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, |
404 | struct i915_sw_fence *signaler, |
405 | gfp_t gfp) |
406 | { |
407 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); |
408 | } |
409 | |
410 | struct i915_sw_dma_fence_cb_timer { |
411 | struct i915_sw_dma_fence_cb base; |
412 | struct dma_fence *dma; |
413 | struct timer_list timer; |
414 | struct irq_work work; |
415 | struct rcu_head rcu; |
416 | }; |
417 | |
418 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
419 | struct dma_fence_cb *data) |
420 | { |
421 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); |
422 | |
423 | i915_sw_fence_set_error_once(fence: cb->fence, error: dma->error); |
424 | i915_sw_fence_complete(fence: cb->fence); |
425 | kfree(objp: cb); |
426 | } |
427 | |
428 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
429 | { |
430 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
431 | struct i915_sw_fence *fence; |
432 | |
433 | fence = xchg(&cb->base.fence, NULL); |
434 | if (!fence) |
435 | return; |
436 | |
437 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n" , |
438 | cb->dma->ops->get_driver_name(cb->dma), |
439 | cb->dma->ops->get_timeline_name(cb->dma), |
440 | cb->dma->seqno, |
441 | i915_sw_fence_debug_hint(fence)); |
442 | |
443 | i915_sw_fence_set_error_once(fence, error: -ETIMEDOUT); |
444 | i915_sw_fence_complete(fence); |
445 | } |
446 | |
447 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
448 | struct dma_fence_cb *data) |
449 | { |
450 | struct i915_sw_dma_fence_cb_timer *cb = |
451 | container_of(data, typeof(*cb), base.base); |
452 | struct i915_sw_fence *fence; |
453 | |
454 | fence = xchg(&cb->base.fence, NULL); |
455 | if (fence) { |
456 | i915_sw_fence_set_error_once(fence, error: dma->error); |
457 | i915_sw_fence_complete(fence); |
458 | } |
459 | |
460 | irq_work_queue(work: &cb->work); |
461 | } |
462 | |
463 | static void irq_i915_sw_fence_work(struct irq_work *wrk) |
464 | { |
465 | struct i915_sw_dma_fence_cb_timer *cb = |
466 | container_of(wrk, typeof(*cb), work); |
467 | |
468 | timer_shutdown_sync(timer: &cb->timer); |
469 | dma_fence_put(fence: cb->dma); |
470 | |
471 | kfree_rcu(cb, rcu); |
472 | } |
473 | |
474 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, |
475 | struct dma_fence *dma, |
476 | unsigned long timeout, |
477 | gfp_t gfp) |
478 | { |
479 | struct i915_sw_dma_fence_cb *cb; |
480 | dma_fence_func_t func; |
481 | int ret; |
482 | |
483 | debug_fence_assert(fence); |
484 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
485 | |
486 | if (dma_fence_is_signaled(fence: dma)) { |
487 | i915_sw_fence_set_error_once(fence, error: dma->error); |
488 | return 0; |
489 | } |
490 | |
491 | cb = kmalloc(size: timeout ? |
492 | sizeof(struct i915_sw_dma_fence_cb_timer) : |
493 | sizeof(struct i915_sw_dma_fence_cb), |
494 | flags: gfp); |
495 | if (!cb) { |
496 | if (!gfpflags_allow_blocking(gfp_flags: gfp)) |
497 | return -ENOMEM; |
498 | |
499 | ret = dma_fence_wait(fence: dma, intr: false); |
500 | if (ret) |
501 | return ret; |
502 | |
503 | i915_sw_fence_set_error_once(fence, error: dma->error); |
504 | return 0; |
505 | } |
506 | |
507 | cb->fence = fence; |
508 | i915_sw_fence_await(fence); |
509 | |
510 | func = dma_i915_sw_fence_wake; |
511 | if (timeout) { |
512 | struct i915_sw_dma_fence_cb_timer *timer = |
513 | container_of(cb, typeof(*timer), base); |
514 | |
515 | timer->dma = dma_fence_get(fence: dma); |
516 | init_irq_work(work: &timer->work, func: irq_i915_sw_fence_work); |
517 | |
518 | timer_setup(&timer->timer, |
519 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
520 | mod_timer(timer: &timer->timer, expires: round_jiffies_up(j: jiffies + timeout)); |
521 | |
522 | func = dma_i915_sw_fence_wake_timer; |
523 | } |
524 | |
525 | ret = dma_fence_add_callback(fence: dma, cb: &cb->base, func); |
526 | if (ret == 0) { |
527 | ret = 1; |
528 | } else { |
529 | func(dma, &cb->base); |
530 | if (ret == -ENOENT) /* fence already signaled */ |
531 | ret = 0; |
532 | } |
533 | |
534 | return ret; |
535 | } |
536 | |
537 | static void __dma_i915_sw_fence_wake(struct dma_fence *dma, |
538 | struct dma_fence_cb *data) |
539 | { |
540 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); |
541 | |
542 | i915_sw_fence_set_error_once(fence: cb->fence, error: dma->error); |
543 | i915_sw_fence_complete(fence: cb->fence); |
544 | } |
545 | |
546 | int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, |
547 | struct dma_fence *dma, |
548 | struct i915_sw_dma_fence_cb *cb) |
549 | { |
550 | int ret; |
551 | |
552 | debug_fence_assert(fence); |
553 | |
554 | if (dma_fence_is_signaled(fence: dma)) { |
555 | i915_sw_fence_set_error_once(fence, error: dma->error); |
556 | return 0; |
557 | } |
558 | |
559 | cb->fence = fence; |
560 | i915_sw_fence_await(fence); |
561 | |
562 | ret = 1; |
563 | if (dma_fence_add_callback(fence: dma, cb: &cb->base, func: __dma_i915_sw_fence_wake)) { |
564 | /* fence already signaled */ |
565 | __dma_i915_sw_fence_wake(dma, data: &cb->base); |
566 | ret = 0; |
567 | } |
568 | |
569 | return ret; |
570 | } |
571 | |
572 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, |
573 | struct dma_resv *resv, |
574 | bool write, |
575 | unsigned long timeout, |
576 | gfp_t gfp) |
577 | { |
578 | struct dma_resv_iter cursor; |
579 | struct dma_fence *f; |
580 | int ret = 0, pending; |
581 | |
582 | debug_fence_assert(fence); |
583 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
584 | |
585 | dma_resv_iter_begin(cursor: &cursor, obj: resv, usage: dma_resv_usage_rw(write)); |
586 | dma_resv_for_each_fence_unlocked(&cursor, f) { |
587 | pending = i915_sw_fence_await_dma_fence(fence, dma: f, timeout, |
588 | gfp); |
589 | if (pending < 0) { |
590 | ret = pending; |
591 | break; |
592 | } |
593 | |
594 | ret |= pending; |
595 | } |
596 | dma_resv_iter_end(cursor: &cursor); |
597 | return ret; |
598 | } |
599 | |
600 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
601 | #include "selftests/lib_sw_fence.c" |
602 | #include "selftests/i915_sw_fence.c" |
603 | #endif |
604 | |