1 | /* |
2 | * Copyright (C) 2007 Ben Skeggs. |
3 | * All Rights Reserved. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining |
6 | * a copy of this software and associated documentation files (the |
7 | * "Software"), to deal in the Software without restriction, including |
8 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * distribute, sublicense, and/or sell copies of the Software, and to |
10 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * the following conditions: |
12 | * |
13 | * The above copyright notice and this permission notice (including the |
14 | * next paragraph) shall be included in all copies or substantial |
15 | * portions of the Software. |
16 | * |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | * |
25 | */ |
26 | |
27 | #include <linux/ktime.h> |
28 | #include <linux/hrtimer.h> |
29 | #include <linux/sched/signal.h> |
30 | #include <trace/events/dma_fence.h> |
31 | |
32 | #include <nvif/if0020.h> |
33 | |
34 | #include "nouveau_drv.h" |
35 | #include "nouveau_dma.h" |
36 | #include "nouveau_fence.h" |
37 | |
38 | static const struct dma_fence_ops nouveau_fence_ops_uevent; |
39 | static const struct dma_fence_ops nouveau_fence_ops_legacy; |
40 | |
41 | static inline struct nouveau_fence * |
42 | from_fence(struct dma_fence *fence) |
43 | { |
44 | return container_of(fence, struct nouveau_fence, base); |
45 | } |
46 | |
47 | static inline struct nouveau_fence_chan * |
48 | nouveau_fctx(struct nouveau_fence *fence) |
49 | { |
50 | return container_of(fence->base.lock, struct nouveau_fence_chan, lock); |
51 | } |
52 | |
53 | static int |
54 | nouveau_fence_signal(struct nouveau_fence *fence) |
55 | { |
56 | int drop = 0; |
57 | |
58 | dma_fence_signal_locked(fence: &fence->base); |
59 | list_del(entry: &fence->head); |
60 | rcu_assign_pointer(fence->channel, NULL); |
61 | |
62 | if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) { |
63 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
64 | |
65 | if (!--fctx->notify_ref) |
66 | drop = 1; |
67 | } |
68 | |
69 | dma_fence_put(fence: &fence->base); |
70 | return drop; |
71 | } |
72 | |
73 | static struct nouveau_fence * |
74 | nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) |
75 | { |
76 | if (fence->ops != &nouveau_fence_ops_legacy && |
77 | fence->ops != &nouveau_fence_ops_uevent) |
78 | return NULL; |
79 | |
80 | return from_fence(fence); |
81 | } |
82 | |
83 | void |
84 | nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error) |
85 | { |
86 | struct nouveau_fence *fence; |
87 | unsigned long flags; |
88 | |
89 | spin_lock_irqsave(&fctx->lock, flags); |
90 | while (!list_empty(head: &fctx->pending)) { |
91 | fence = list_entry(fctx->pending.next, typeof(*fence), head); |
92 | |
93 | if (error) |
94 | dma_fence_set_error(fence: &fence->base, error); |
95 | |
96 | if (nouveau_fence_signal(fence)) |
97 | nvif_event_block(&fctx->event); |
98 | } |
99 | fctx->killed = 1; |
100 | spin_unlock_irqrestore(lock: &fctx->lock, flags); |
101 | } |
102 | |
103 | void |
104 | nouveau_fence_context_del(struct nouveau_fence_chan *fctx) |
105 | { |
106 | nouveau_fence_context_kill(fctx, error: 0); |
107 | nvif_event_dtor(&fctx->event); |
108 | fctx->dead = 1; |
109 | |
110 | /* |
111 | * Ensure that all accesses to fence->channel complete before freeing |
112 | * the channel. |
113 | */ |
114 | synchronize_rcu(); |
115 | } |
116 | |
117 | static void |
118 | nouveau_fence_context_put(struct kref *fence_ref) |
119 | { |
120 | kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref)); |
121 | } |
122 | |
123 | void |
124 | nouveau_fence_context_free(struct nouveau_fence_chan *fctx) |
125 | { |
126 | kref_put(kref: &fctx->fence_ref, release: nouveau_fence_context_put); |
127 | } |
128 | |
129 | static int |
130 | nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) |
131 | { |
132 | struct nouveau_fence *fence; |
133 | int drop = 0; |
134 | u32 seq = fctx->read(chan); |
135 | |
136 | while (!list_empty(head: &fctx->pending)) { |
137 | fence = list_entry(fctx->pending.next, typeof(*fence), head); |
138 | |
139 | if ((int)(seq - fence->base.seqno) < 0) |
140 | break; |
141 | |
142 | drop |= nouveau_fence_signal(fence); |
143 | } |
144 | |
145 | return drop; |
146 | } |
147 | |
148 | static int |
149 | nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc) |
150 | { |
151 | struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event); |
152 | unsigned long flags; |
153 | int ret = NVIF_EVENT_KEEP; |
154 | |
155 | spin_lock_irqsave(&fctx->lock, flags); |
156 | if (!list_empty(head: &fctx->pending)) { |
157 | struct nouveau_fence *fence; |
158 | struct nouveau_channel *chan; |
159 | |
160 | fence = list_entry(fctx->pending.next, typeof(*fence), head); |
161 | chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); |
162 | if (nouveau_fence_update(chan, fctx)) |
163 | ret = NVIF_EVENT_DROP; |
164 | } |
165 | spin_unlock_irqrestore(lock: &fctx->lock, flags); |
166 | |
167 | return ret; |
168 | } |
169 | |
170 | void |
171 | nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) |
172 | { |
173 | struct nouveau_fence_priv *priv = (void*)chan->drm->fence; |
174 | struct nouveau_cli *cli = (void *)chan->user.client; |
175 | struct { |
176 | struct nvif_event_v0 base; |
177 | struct nvif_chan_event_v0 host; |
178 | } args; |
179 | int ret; |
180 | |
181 | INIT_LIST_HEAD(list: &fctx->flip); |
182 | INIT_LIST_HEAD(list: &fctx->pending); |
183 | spin_lock_init(&fctx->lock); |
184 | fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid; |
185 | |
186 | if (chan == chan->drm->cechan) |
187 | strcpy(p: fctx->name, q: "copy engine channel" ); |
188 | else if (chan == chan->drm->channel) |
189 | strcpy(p: fctx->name, q: "generic kernel channel" ); |
190 | else |
191 | strcpy(fctx->name, nvxx_client(&cli->base)->name); |
192 | |
193 | kref_init(kref: &fctx->fence_ref); |
194 | if (!priv->uevent) |
195 | return; |
196 | |
197 | args.host.version = 0; |
198 | args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR; |
199 | |
200 | ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr" , (chan->runlist << 16) | chan->chid, |
201 | nouveau_fence_wait_uevent_handler, false, |
202 | &args.base, sizeof(args), &fctx->event); |
203 | |
204 | WARN_ON(ret); |
205 | } |
206 | |
207 | int |
208 | nouveau_fence_emit(struct nouveau_fence *fence) |
209 | { |
210 | struct nouveau_channel *chan = unrcu_pointer(fence->channel); |
211 | struct nouveau_fence_chan *fctx = chan->fence; |
212 | struct nouveau_fence_priv *priv = (void*)chan->drm->fence; |
213 | int ret; |
214 | |
215 | fence->timeout = jiffies + (15 * HZ); |
216 | |
217 | if (priv->uevent) |
218 | dma_fence_init(fence: &fence->base, ops: &nouveau_fence_ops_uevent, |
219 | lock: &fctx->lock, context: fctx->context, seqno: ++fctx->sequence); |
220 | else |
221 | dma_fence_init(fence: &fence->base, ops: &nouveau_fence_ops_legacy, |
222 | lock: &fctx->lock, context: fctx->context, seqno: ++fctx->sequence); |
223 | kref_get(kref: &fctx->fence_ref); |
224 | |
225 | ret = fctx->emit(fence); |
226 | if (!ret) { |
227 | dma_fence_get(fence: &fence->base); |
228 | spin_lock_irq(lock: &fctx->lock); |
229 | |
230 | if (unlikely(fctx->killed)) { |
231 | spin_unlock_irq(lock: &fctx->lock); |
232 | dma_fence_put(fence: &fence->base); |
233 | return -ENODEV; |
234 | } |
235 | |
236 | if (nouveau_fence_update(chan, fctx)) |
237 | nvif_event_block(&fctx->event); |
238 | |
239 | list_add_tail(new: &fence->head, head: &fctx->pending); |
240 | spin_unlock_irq(lock: &fctx->lock); |
241 | } |
242 | |
243 | return ret; |
244 | } |
245 | |
246 | bool |
247 | nouveau_fence_done(struct nouveau_fence *fence) |
248 | { |
249 | if (fence->base.ops == &nouveau_fence_ops_legacy || |
250 | fence->base.ops == &nouveau_fence_ops_uevent) { |
251 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
252 | struct nouveau_channel *chan; |
253 | unsigned long flags; |
254 | |
255 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) |
256 | return true; |
257 | |
258 | spin_lock_irqsave(&fctx->lock, flags); |
259 | chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); |
260 | if (chan && nouveau_fence_update(chan, fctx)) |
261 | nvif_event_block(&fctx->event); |
262 | spin_unlock_irqrestore(lock: &fctx->lock, flags); |
263 | } |
264 | return dma_fence_is_signaled(fence: &fence->base); |
265 | } |
266 | |
267 | static long |
268 | nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait) |
269 | { |
270 | struct nouveau_fence *fence = from_fence(fence: f); |
271 | unsigned long sleep_time = NSEC_PER_MSEC / 1000; |
272 | unsigned long t = jiffies, timeout = t + wait; |
273 | |
274 | while (!nouveau_fence_done(fence)) { |
275 | ktime_t kt; |
276 | |
277 | t = jiffies; |
278 | |
279 | if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) { |
280 | __set_current_state(TASK_RUNNING); |
281 | return 0; |
282 | } |
283 | |
284 | __set_current_state(intr ? TASK_INTERRUPTIBLE : |
285 | TASK_UNINTERRUPTIBLE); |
286 | |
287 | kt = sleep_time; |
288 | schedule_hrtimeout(expires: &kt, mode: HRTIMER_MODE_REL); |
289 | sleep_time *= 2; |
290 | if (sleep_time > NSEC_PER_MSEC) |
291 | sleep_time = NSEC_PER_MSEC; |
292 | |
293 | if (intr && signal_pending(current)) |
294 | return -ERESTARTSYS; |
295 | } |
296 | |
297 | __set_current_state(TASK_RUNNING); |
298 | |
299 | return timeout - t; |
300 | } |
301 | |
302 | static int |
303 | nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr) |
304 | { |
305 | int ret = 0; |
306 | |
307 | while (!nouveau_fence_done(fence)) { |
308 | if (time_after_eq(jiffies, fence->timeout)) { |
309 | ret = -EBUSY; |
310 | break; |
311 | } |
312 | |
313 | __set_current_state(intr ? |
314 | TASK_INTERRUPTIBLE : |
315 | TASK_UNINTERRUPTIBLE); |
316 | |
317 | if (intr && signal_pending(current)) { |
318 | ret = -ERESTARTSYS; |
319 | break; |
320 | } |
321 | } |
322 | |
323 | __set_current_state(TASK_RUNNING); |
324 | return ret; |
325 | } |
326 | |
327 | int |
328 | nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) |
329 | { |
330 | long ret; |
331 | |
332 | if (!lazy) |
333 | return nouveau_fence_wait_busy(fence, intr); |
334 | |
335 | ret = dma_fence_wait_timeout(&fence->base, intr, timeout: 15 * HZ); |
336 | if (ret < 0) |
337 | return ret; |
338 | else if (!ret) |
339 | return -EBUSY; |
340 | else |
341 | return 0; |
342 | } |
343 | |
344 | int |
345 | nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, |
346 | bool exclusive, bool intr) |
347 | { |
348 | struct nouveau_fence_chan *fctx = chan->fence; |
349 | struct dma_resv *resv = nvbo->bo.base.resv; |
350 | int i, ret; |
351 | |
352 | ret = dma_resv_reserve_fences(obj: resv, num_fences: 1); |
353 | if (ret) |
354 | return ret; |
355 | |
356 | /* Waiting for the writes first causes performance regressions |
357 | * under some circumstances. So manually wait for the reads first. |
358 | */ |
359 | for (i = 0; i < 2; ++i) { |
360 | struct dma_resv_iter cursor; |
361 | struct dma_fence *fence; |
362 | |
363 | dma_resv_for_each_fence(&cursor, resv, |
364 | dma_resv_usage_rw(exclusive), |
365 | fence) { |
366 | enum dma_resv_usage usage; |
367 | struct nouveau_fence *f; |
368 | |
369 | usage = dma_resv_iter_usage(cursor: &cursor); |
370 | if (i == 0 && usage == DMA_RESV_USAGE_WRITE) |
371 | continue; |
372 | |
373 | f = nouveau_local_fence(fence, drm: chan->drm); |
374 | if (f) { |
375 | struct nouveau_channel *prev; |
376 | bool must_wait = true; |
377 | |
378 | rcu_read_lock(); |
379 | prev = rcu_dereference(f->channel); |
380 | if (prev && (prev == chan || |
381 | fctx->sync(f, prev, chan) == 0)) |
382 | must_wait = false; |
383 | rcu_read_unlock(); |
384 | if (!must_wait) |
385 | continue; |
386 | } |
387 | |
388 | ret = dma_fence_wait(fence, intr); |
389 | if (ret) |
390 | return ret; |
391 | } |
392 | } |
393 | |
394 | return 0; |
395 | } |
396 | |
397 | void |
398 | nouveau_fence_unref(struct nouveau_fence **pfence) |
399 | { |
400 | if (*pfence) |
401 | dma_fence_put(fence: &(*pfence)->base); |
402 | *pfence = NULL; |
403 | } |
404 | |
405 | int |
406 | nouveau_fence_create(struct nouveau_fence **pfence, |
407 | struct nouveau_channel *chan) |
408 | { |
409 | struct nouveau_fence *fence; |
410 | |
411 | if (unlikely(!chan->fence)) |
412 | return -ENODEV; |
413 | |
414 | fence = kzalloc(size: sizeof(*fence), GFP_KERNEL); |
415 | if (!fence) |
416 | return -ENOMEM; |
417 | |
418 | fence->channel = chan; |
419 | |
420 | *pfence = fence; |
421 | return 0; |
422 | } |
423 | |
424 | int |
425 | nouveau_fence_new(struct nouveau_fence **pfence, |
426 | struct nouveau_channel *chan) |
427 | { |
428 | int ret = 0; |
429 | |
430 | ret = nouveau_fence_create(pfence, chan); |
431 | if (ret) |
432 | return ret; |
433 | |
434 | ret = nouveau_fence_emit(fence: *pfence); |
435 | if (ret) |
436 | nouveau_fence_unref(pfence); |
437 | |
438 | return ret; |
439 | } |
440 | |
441 | static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence) |
442 | { |
443 | return "nouveau" ; |
444 | } |
445 | |
446 | static const char *nouveau_fence_get_timeline_name(struct dma_fence *f) |
447 | { |
448 | struct nouveau_fence *fence = from_fence(fence: f); |
449 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
450 | |
451 | return !fctx->dead ? fctx->name : "dead channel" ; |
452 | } |
453 | |
454 | /* |
455 | * In an ideal world, read would not assume the channel context is still alive. |
456 | * This function may be called from another device, running into free memory as a |
457 | * result. The drm node should still be there, so we can derive the index from |
458 | * the fence context. |
459 | */ |
460 | static bool nouveau_fence_is_signaled(struct dma_fence *f) |
461 | { |
462 | struct nouveau_fence *fence = from_fence(fence: f); |
463 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
464 | struct nouveau_channel *chan; |
465 | bool ret = false; |
466 | |
467 | rcu_read_lock(); |
468 | chan = rcu_dereference(fence->channel); |
469 | if (chan) |
470 | ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0; |
471 | rcu_read_unlock(); |
472 | |
473 | return ret; |
474 | } |
475 | |
476 | static bool nouveau_fence_no_signaling(struct dma_fence *f) |
477 | { |
478 | struct nouveau_fence *fence = from_fence(fence: f); |
479 | |
480 | /* |
481 | * caller should have a reference on the fence, |
482 | * else fence could get freed here |
483 | */ |
484 | WARN_ON(kref_read(&fence->base.refcount) <= 1); |
485 | |
486 | /* |
487 | * This needs uevents to work correctly, but dma_fence_add_callback relies on |
488 | * being able to enable signaling. It will still get signaled eventually, |
489 | * just not right away. |
490 | */ |
491 | if (nouveau_fence_is_signaled(f)) { |
492 | list_del(entry: &fence->head); |
493 | |
494 | dma_fence_put(fence: &fence->base); |
495 | return false; |
496 | } |
497 | |
498 | return true; |
499 | } |
500 | |
501 | static void nouveau_fence_release(struct dma_fence *f) |
502 | { |
503 | struct nouveau_fence *fence = from_fence(fence: f); |
504 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
505 | |
506 | kref_put(kref: &fctx->fence_ref, release: nouveau_fence_context_put); |
507 | dma_fence_free(fence: &fence->base); |
508 | } |
509 | |
510 | static const struct dma_fence_ops nouveau_fence_ops_legacy = { |
511 | .get_driver_name = nouveau_fence_get_get_driver_name, |
512 | .get_timeline_name = nouveau_fence_get_timeline_name, |
513 | .enable_signaling = nouveau_fence_no_signaling, |
514 | .signaled = nouveau_fence_is_signaled, |
515 | .wait = nouveau_fence_wait_legacy, |
516 | .release = nouveau_fence_release |
517 | }; |
518 | |
519 | static bool nouveau_fence_enable_signaling(struct dma_fence *f) |
520 | { |
521 | struct nouveau_fence *fence = from_fence(fence: f); |
522 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); |
523 | bool ret; |
524 | |
525 | if (!fctx->notify_ref++) |
526 | nvif_event_allow(&fctx->event); |
527 | |
528 | ret = nouveau_fence_no_signaling(f); |
529 | if (ret) |
530 | set_bit(nr: DMA_FENCE_FLAG_USER_BITS, addr: &fence->base.flags); |
531 | else if (!--fctx->notify_ref) |
532 | nvif_event_block(&fctx->event); |
533 | |
534 | return ret; |
535 | } |
536 | |
537 | static const struct dma_fence_ops nouveau_fence_ops_uevent = { |
538 | .get_driver_name = nouveau_fence_get_get_driver_name, |
539 | .get_timeline_name = nouveau_fence_get_timeline_name, |
540 | .enable_signaling = nouveau_fence_enable_signaling, |
541 | .signaled = nouveau_fence_is_signaled, |
542 | .release = nouveau_fence_release |
543 | }; |
544 | |