1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
2 | /************************************************************************** |
3 | * |
4 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
5 | * All Rights Reserved. |
6 | * |
7 | * Permission is hereby granted, free of charge, to any person obtaining a |
8 | * copy of this software and associated documentation files (the |
9 | * "Software"), to deal in the Software without restriction, including |
10 | * without limitation the rights to use, copy, modify, merge, publish, |
11 | * distribute, sub license, and/or sell copies of the Software, and to |
12 | * permit persons to whom the Software is furnished to do so, subject to |
13 | * the following conditions: |
14 | * |
15 | * The above copyright notice and this permission notice (including the |
16 | * next paragraph) shall be included in all copies or substantial portions |
17 | * of the Software. |
18 | * |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
26 | * |
27 | **************************************************************************/ |
28 | /* |
29 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
30 | */ |
31 | |
32 | #define pr_fmt(fmt) "[TTM] " fmt |
33 | |
34 | #include <drm/ttm/ttm_bo.h> |
35 | #include <drm/ttm/ttm_placement.h> |
36 | #include <drm/ttm/ttm_tt.h> |
37 | |
38 | #include <linux/jiffies.h> |
39 | #include <linux/slab.h> |
40 | #include <linux/sched.h> |
41 | #include <linux/mm.h> |
42 | #include <linux/file.h> |
43 | #include <linux/module.h> |
44 | #include <linux/atomic.h> |
45 | #include <linux/dma-resv.h> |
46 | |
47 | #include "ttm_module.h" |
48 | |
49 | static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, |
50 | struct ttm_placement *placement) |
51 | { |
52 | struct drm_printer p = drm_dbg_printer(NULL, category: DRM_UT_CORE, TTM_PFX); |
53 | struct ttm_resource_manager *man; |
54 | int i, mem_type; |
55 | |
56 | for (i = 0; i < placement->num_placement; i++) { |
57 | mem_type = placement->placement[i].mem_type; |
58 | drm_printf(p: &p, f: " placement[%d]=0x%08X (%d)\n" , |
59 | i, placement->placement[i].flags, mem_type); |
60 | man = ttm_manager_type(bdev: bo->bdev, mem_type); |
61 | ttm_resource_manager_debug(man, p: &p); |
62 | } |
63 | } |
64 | |
65 | /** |
66 | * ttm_bo_move_to_lru_tail |
67 | * |
68 | * @bo: The buffer object. |
69 | * |
70 | * Move this BO to the tail of all lru lists used to lookup and reserve an |
71 | * object. This function must be called with struct ttm_global::lru_lock |
72 | * held, and is used to make a BO less likely to be considered for eviction. |
73 | */ |
74 | void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) |
75 | { |
76 | dma_resv_assert_held(bo->base.resv); |
77 | |
78 | if (bo->resource) |
79 | ttm_resource_move_to_lru_tail(res: bo->resource); |
80 | } |
81 | EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); |
82 | |
83 | /** |
84 | * ttm_bo_set_bulk_move - update BOs bulk move object |
85 | * |
86 | * @bo: The buffer object. |
87 | * @bulk: bulk move structure |
88 | * |
89 | * Update the BOs bulk move object, making sure that resources are added/removed |
90 | * as well. A bulk move allows to move many resource on the LRU at once, |
91 | * resulting in much less overhead of maintaining the LRU. |
92 | * The only requirement is that the resources stay together on the LRU and are |
93 | * never separated. This is enforces by setting the bulk_move structure on a BO. |
94 | * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of |
95 | * their LRU list. |
96 | */ |
97 | void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, |
98 | struct ttm_lru_bulk_move *bulk) |
99 | { |
100 | dma_resv_assert_held(bo->base.resv); |
101 | |
102 | if (bo->bulk_move == bulk) |
103 | return; |
104 | |
105 | spin_lock(lock: &bo->bdev->lru_lock); |
106 | if (bo->resource) |
107 | ttm_resource_del_bulk_move(res: bo->resource, bo); |
108 | bo->bulk_move = bulk; |
109 | if (bo->resource) |
110 | ttm_resource_add_bulk_move(res: bo->resource, bo); |
111 | spin_unlock(lock: &bo->bdev->lru_lock); |
112 | } |
113 | EXPORT_SYMBOL(ttm_bo_set_bulk_move); |
114 | |
115 | static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, |
116 | struct ttm_resource *mem, bool evict, |
117 | struct ttm_operation_ctx *ctx, |
118 | struct ttm_place *hop) |
119 | { |
120 | struct ttm_device *bdev = bo->bdev; |
121 | bool old_use_tt, new_use_tt; |
122 | int ret; |
123 | |
124 | old_use_tt = !bo->resource || ttm_manager_type(bdev, mem_type: bo->resource->mem_type)->use_tt; |
125 | new_use_tt = ttm_manager_type(bdev, mem_type: mem->mem_type)->use_tt; |
126 | |
127 | ttm_bo_unmap_virtual(bo); |
128 | |
129 | /* |
130 | * Create and bind a ttm if required. |
131 | */ |
132 | |
133 | if (new_use_tt) { |
134 | /* Zero init the new TTM structure if the old location should |
135 | * have used one as well. |
136 | */ |
137 | ret = ttm_tt_create(bo, zero_alloc: old_use_tt); |
138 | if (ret) |
139 | goto out_err; |
140 | |
141 | if (mem->mem_type != TTM_PL_SYSTEM) { |
142 | ret = ttm_tt_populate(bdev: bo->bdev, ttm: bo->ttm, ctx); |
143 | if (ret) |
144 | goto out_err; |
145 | } |
146 | } |
147 | |
148 | ret = dma_resv_reserve_fences(obj: bo->base.resv, num_fences: 1); |
149 | if (ret) |
150 | goto out_err; |
151 | |
152 | ret = bdev->funcs->move(bo, evict, ctx, mem, hop); |
153 | if (ret) { |
154 | if (ret == -EMULTIHOP) |
155 | return ret; |
156 | goto out_err; |
157 | } |
158 | |
159 | ctx->bytes_moved += bo->base.size; |
160 | return 0; |
161 | |
162 | out_err: |
163 | if (!old_use_tt) |
164 | ttm_bo_tt_destroy(bo); |
165 | |
166 | return ret; |
167 | } |
168 | |
169 | /* |
170 | * Call bo::reserved. |
171 | * Will release GPU memory type usage on destruction. |
172 | * This is the place to put in driver specific hooks to release |
173 | * driver private resources. |
174 | * Will release the bo::reserved lock. |
175 | */ |
176 | |
177 | static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) |
178 | { |
179 | if (bo->bdev->funcs->delete_mem_notify) |
180 | bo->bdev->funcs->delete_mem_notify(bo); |
181 | |
182 | ttm_bo_tt_destroy(bo); |
183 | ttm_resource_free(bo, res: &bo->resource); |
184 | } |
185 | |
186 | static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) |
187 | { |
188 | int r; |
189 | |
190 | if (bo->base.resv == &bo->base._resv) |
191 | return 0; |
192 | |
193 | BUG_ON(!dma_resv_trylock(&bo->base._resv)); |
194 | |
195 | r = dma_resv_copy_fences(dst: &bo->base._resv, src: bo->base.resv); |
196 | dma_resv_unlock(obj: &bo->base._resv); |
197 | if (r) |
198 | return r; |
199 | |
200 | if (bo->type != ttm_bo_type_sg) { |
201 | /* This works because the BO is about to be destroyed and nobody |
202 | * reference it any more. The only tricky case is the trylock on |
203 | * the resv object while holding the lru_lock. |
204 | */ |
205 | spin_lock(lock: &bo->bdev->lru_lock); |
206 | bo->base.resv = &bo->base._resv; |
207 | spin_unlock(lock: &bo->bdev->lru_lock); |
208 | } |
209 | |
210 | return r; |
211 | } |
212 | |
213 | static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) |
214 | { |
215 | struct dma_resv *resv = &bo->base._resv; |
216 | struct dma_resv_iter cursor; |
217 | struct dma_fence *fence; |
218 | |
219 | dma_resv_iter_begin(cursor: &cursor, obj: resv, usage: DMA_RESV_USAGE_BOOKKEEP); |
220 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
221 | if (!fence->ops->signaled) |
222 | dma_fence_enable_sw_signaling(fence); |
223 | } |
224 | dma_resv_iter_end(cursor: &cursor); |
225 | } |
226 | |
227 | /** |
228 | * ttm_bo_cleanup_refs |
229 | * If bo idle, remove from lru lists, and unref. |
230 | * If not idle, block if possible. |
231 | * |
232 | * Must be called with lru_lock and reservation held, this function |
233 | * will drop the lru lock and optionally the reservation lock before returning. |
234 | * |
235 | * @bo: The buffer object to clean-up |
236 | * @interruptible: Any sleeps should occur interruptibly. |
237 | * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. |
238 | * @unlock_resv: Unlock the reservation lock as well. |
239 | */ |
240 | |
241 | static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, |
242 | bool interruptible, bool no_wait_gpu, |
243 | bool unlock_resv) |
244 | { |
245 | struct dma_resv *resv = &bo->base._resv; |
246 | int ret; |
247 | |
248 | if (dma_resv_test_signaled(obj: resv, usage: DMA_RESV_USAGE_BOOKKEEP)) |
249 | ret = 0; |
250 | else |
251 | ret = -EBUSY; |
252 | |
253 | if (ret && !no_wait_gpu) { |
254 | long lret; |
255 | |
256 | if (unlock_resv) |
257 | dma_resv_unlock(obj: bo->base.resv); |
258 | spin_unlock(lock: &bo->bdev->lru_lock); |
259 | |
260 | lret = dma_resv_wait_timeout(obj: resv, usage: DMA_RESV_USAGE_BOOKKEEP, |
261 | intr: interruptible, |
262 | timeout: 30 * HZ); |
263 | |
264 | if (lret < 0) |
265 | return lret; |
266 | else if (lret == 0) |
267 | return -EBUSY; |
268 | |
269 | spin_lock(lock: &bo->bdev->lru_lock); |
270 | if (unlock_resv && !dma_resv_trylock(obj: bo->base.resv)) { |
271 | /* |
272 | * We raced, and lost, someone else holds the reservation now, |
273 | * and is probably busy in ttm_bo_cleanup_memtype_use. |
274 | * |
275 | * Even if it's not the case, because we finished waiting any |
276 | * delayed destruction would succeed, so just return success |
277 | * here. |
278 | */ |
279 | spin_unlock(lock: &bo->bdev->lru_lock); |
280 | return 0; |
281 | } |
282 | ret = 0; |
283 | } |
284 | |
285 | if (ret) { |
286 | if (unlock_resv) |
287 | dma_resv_unlock(obj: bo->base.resv); |
288 | spin_unlock(lock: &bo->bdev->lru_lock); |
289 | return ret; |
290 | } |
291 | |
292 | spin_unlock(lock: &bo->bdev->lru_lock); |
293 | ttm_bo_cleanup_memtype_use(bo); |
294 | |
295 | if (unlock_resv) |
296 | dma_resv_unlock(obj: bo->base.resv); |
297 | |
298 | return 0; |
299 | } |
300 | |
301 | /* |
302 | * Block for the dma_resv object to become idle, lock the buffer and clean up |
303 | * the resource and tt object. |
304 | */ |
305 | static void ttm_bo_delayed_delete(struct work_struct *work) |
306 | { |
307 | struct ttm_buffer_object *bo; |
308 | |
309 | bo = container_of(work, typeof(*bo), delayed_delete); |
310 | |
311 | dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP, intr: false, |
312 | MAX_SCHEDULE_TIMEOUT); |
313 | dma_resv_lock(obj: bo->base.resv, NULL); |
314 | ttm_bo_cleanup_memtype_use(bo); |
315 | dma_resv_unlock(obj: bo->base.resv); |
316 | ttm_bo_put(bo); |
317 | } |
318 | |
319 | static void ttm_bo_release(struct kref *kref) |
320 | { |
321 | struct ttm_buffer_object *bo = |
322 | container_of(kref, struct ttm_buffer_object, kref); |
323 | struct ttm_device *bdev = bo->bdev; |
324 | int ret; |
325 | |
326 | WARN_ON_ONCE(bo->pin_count); |
327 | WARN_ON_ONCE(bo->bulk_move); |
328 | |
329 | if (!bo->deleted) { |
330 | ret = ttm_bo_individualize_resv(bo); |
331 | if (ret) { |
332 | /* Last resort, if we fail to allocate memory for the |
333 | * fences block for the BO to become idle |
334 | */ |
335 | dma_resv_wait_timeout(obj: bo->base.resv, |
336 | usage: DMA_RESV_USAGE_BOOKKEEP, intr: false, |
337 | timeout: 30 * HZ); |
338 | } |
339 | |
340 | if (bo->bdev->funcs->release_notify) |
341 | bo->bdev->funcs->release_notify(bo); |
342 | |
343 | drm_vma_offset_remove(mgr: bdev->vma_manager, node: &bo->base.vma_node); |
344 | ttm_mem_io_free(bdev, mem: bo->resource); |
345 | |
346 | if (!dma_resv_test_signaled(obj: bo->base.resv, |
347 | usage: DMA_RESV_USAGE_BOOKKEEP) || |
348 | (want_init_on_free() && (bo->ttm != NULL)) || |
349 | !dma_resv_trylock(obj: bo->base.resv)) { |
350 | /* The BO is not idle, resurrect it for delayed destroy */ |
351 | ttm_bo_flush_all_fences(bo); |
352 | bo->deleted = true; |
353 | |
354 | spin_lock(lock: &bo->bdev->lru_lock); |
355 | |
356 | /* |
357 | * Make pinned bos immediately available to |
358 | * shrinkers, now that they are queued for |
359 | * destruction. |
360 | * |
361 | * FIXME: QXL is triggering this. Can be removed when the |
362 | * driver is fixed. |
363 | */ |
364 | if (bo->pin_count) { |
365 | bo->pin_count = 0; |
366 | ttm_resource_move_to_lru_tail(res: bo->resource); |
367 | } |
368 | |
369 | kref_init(kref: &bo->kref); |
370 | spin_unlock(lock: &bo->bdev->lru_lock); |
371 | |
372 | INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); |
373 | |
374 | /* Schedule the worker on the closest NUMA node. This |
375 | * improves performance since system memory might be |
376 | * cleared on free and that is best done on a CPU core |
377 | * close to it. |
378 | */ |
379 | queue_work_node(node: bdev->pool.nid, wq: bdev->wq, work: &bo->delayed_delete); |
380 | return; |
381 | } |
382 | |
383 | ttm_bo_cleanup_memtype_use(bo); |
384 | dma_resv_unlock(obj: bo->base.resv); |
385 | } |
386 | |
387 | atomic_dec(v: &ttm_glob.bo_count); |
388 | bo->destroy(bo); |
389 | } |
390 | |
391 | /** |
392 | * ttm_bo_put |
393 | * |
394 | * @bo: The buffer object. |
395 | * |
396 | * Unreference a buffer object. |
397 | */ |
398 | void ttm_bo_put(struct ttm_buffer_object *bo) |
399 | { |
400 | kref_put(kref: &bo->kref, release: ttm_bo_release); |
401 | } |
402 | EXPORT_SYMBOL(ttm_bo_put); |
403 | |
404 | static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, |
405 | struct ttm_resource **mem, |
406 | struct ttm_operation_ctx *ctx, |
407 | struct ttm_place *hop) |
408 | { |
409 | struct ttm_placement hop_placement; |
410 | struct ttm_resource *hop_mem; |
411 | int ret; |
412 | |
413 | hop_placement.num_placement = 1; |
414 | hop_placement.placement = hop; |
415 | |
416 | /* find space in the bounce domain */ |
417 | ret = ttm_bo_mem_space(bo, placement: &hop_placement, mem: &hop_mem, ctx); |
418 | if (ret) |
419 | return ret; |
420 | /* move to the bounce domain */ |
421 | ret = ttm_bo_handle_move_mem(bo, mem: hop_mem, evict: false, ctx, NULL); |
422 | if (ret) { |
423 | ttm_resource_free(bo, res: &hop_mem); |
424 | return ret; |
425 | } |
426 | return 0; |
427 | } |
428 | |
429 | static int ttm_bo_evict(struct ttm_buffer_object *bo, |
430 | struct ttm_operation_ctx *ctx) |
431 | { |
432 | struct ttm_device *bdev = bo->bdev; |
433 | struct ttm_resource *evict_mem; |
434 | struct ttm_placement placement; |
435 | struct ttm_place hop; |
436 | int ret = 0; |
437 | |
438 | memset(&hop, 0, sizeof(hop)); |
439 | |
440 | dma_resv_assert_held(bo->base.resv); |
441 | |
442 | placement.num_placement = 0; |
443 | bdev->funcs->evict_flags(bo, &placement); |
444 | |
445 | if (!placement.num_placement) { |
446 | ret = ttm_bo_wait_ctx(bo, ctx); |
447 | if (ret) |
448 | return ret; |
449 | |
450 | /* |
451 | * Since we've already synced, this frees backing store |
452 | * immediately. |
453 | */ |
454 | return ttm_bo_pipeline_gutting(bo); |
455 | } |
456 | |
457 | ret = ttm_bo_mem_space(bo, placement: &placement, mem: &evict_mem, ctx); |
458 | if (ret) { |
459 | if (ret != -ERESTARTSYS) { |
460 | pr_err("Failed to find memory space for buffer 0x%p eviction\n" , |
461 | bo); |
462 | ttm_bo_mem_space_debug(bo, placement: &placement); |
463 | } |
464 | goto out; |
465 | } |
466 | |
467 | do { |
468 | ret = ttm_bo_handle_move_mem(bo, mem: evict_mem, evict: true, ctx, hop: &hop); |
469 | if (ret != -EMULTIHOP) |
470 | break; |
471 | |
472 | ret = ttm_bo_bounce_temp_buffer(bo, mem: &evict_mem, ctx, hop: &hop); |
473 | } while (!ret); |
474 | |
475 | if (ret) { |
476 | ttm_resource_free(bo, res: &evict_mem); |
477 | if (ret != -ERESTARTSYS && ret != -EINTR) |
478 | pr_err("Buffer eviction failed\n" ); |
479 | } |
480 | out: |
481 | return ret; |
482 | } |
483 | |
484 | /** |
485 | * ttm_bo_eviction_valuable |
486 | * |
487 | * @bo: The buffer object to evict |
488 | * @place: the placement we need to make room for |
489 | * |
490 | * Check if it is valuable to evict the BO to make room for the given placement. |
491 | */ |
492 | bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, |
493 | const struct ttm_place *place) |
494 | { |
495 | struct ttm_resource *res = bo->resource; |
496 | struct ttm_device *bdev = bo->bdev; |
497 | |
498 | dma_resv_assert_held(bo->base.resv); |
499 | if (bo->resource->mem_type == TTM_PL_SYSTEM) |
500 | return true; |
501 | |
502 | /* Don't evict this BO if it's outside of the |
503 | * requested placement range |
504 | */ |
505 | return ttm_resource_intersects(bdev, res, place, size: bo->base.size); |
506 | } |
507 | EXPORT_SYMBOL(ttm_bo_eviction_valuable); |
508 | |
509 | /* |
510 | * Check the target bo is allowable to be evicted or swapout, including cases: |
511 | * |
512 | * a. if share same reservation object with ctx->resv, have assumption |
513 | * reservation objects should already be locked, so not lock again and |
514 | * return true directly when either the opreation allow_reserved_eviction |
515 | * or the target bo already is in delayed free list; |
516 | * |
517 | * b. Otherwise, trylock it. |
518 | */ |
519 | static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, |
520 | struct ttm_operation_ctx *ctx, |
521 | const struct ttm_place *place, |
522 | bool *locked, bool *busy) |
523 | { |
524 | bool ret = false; |
525 | |
526 | if (bo->pin_count) { |
527 | *locked = false; |
528 | if (busy) |
529 | *busy = false; |
530 | return false; |
531 | } |
532 | |
533 | if (bo->base.resv == ctx->resv) { |
534 | dma_resv_assert_held(bo->base.resv); |
535 | if (ctx->allow_res_evict) |
536 | ret = true; |
537 | *locked = false; |
538 | if (busy) |
539 | *busy = false; |
540 | } else { |
541 | ret = dma_resv_trylock(obj: bo->base.resv); |
542 | *locked = ret; |
543 | if (busy) |
544 | *busy = !ret; |
545 | } |
546 | |
547 | if (ret && place && (bo->resource->mem_type != place->mem_type || |
548 | !bo->bdev->funcs->eviction_valuable(bo, place))) { |
549 | ret = false; |
550 | if (*locked) { |
551 | dma_resv_unlock(obj: bo->base.resv); |
552 | *locked = false; |
553 | } |
554 | } |
555 | |
556 | return ret; |
557 | } |
558 | |
559 | /** |
560 | * ttm_mem_evict_wait_busy - wait for a busy BO to become available |
561 | * |
562 | * @busy_bo: BO which couldn't be locked with trylock |
563 | * @ctx: operation context |
564 | * @ticket: acquire ticket |
565 | * |
566 | * Try to lock a busy buffer object to avoid failing eviction. |
567 | */ |
568 | static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, |
569 | struct ttm_operation_ctx *ctx, |
570 | struct ww_acquire_ctx *ticket) |
571 | { |
572 | int r; |
573 | |
574 | if (!busy_bo || !ticket) |
575 | return -EBUSY; |
576 | |
577 | if (ctx->interruptible) |
578 | r = dma_resv_lock_interruptible(obj: busy_bo->base.resv, |
579 | ctx: ticket); |
580 | else |
581 | r = dma_resv_lock(obj: busy_bo->base.resv, ctx: ticket); |
582 | |
583 | /* |
584 | * TODO: It would be better to keep the BO locked until allocation is at |
585 | * least tried one more time, but that would mean a much larger rework |
586 | * of TTM. |
587 | */ |
588 | if (!r) |
589 | dma_resv_unlock(obj: busy_bo->base.resv); |
590 | |
591 | return r == -EDEADLK ? -EBUSY : r; |
592 | } |
593 | |
594 | int ttm_mem_evict_first(struct ttm_device *bdev, |
595 | struct ttm_resource_manager *man, |
596 | const struct ttm_place *place, |
597 | struct ttm_operation_ctx *ctx, |
598 | struct ww_acquire_ctx *ticket) |
599 | { |
600 | struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; |
601 | struct ttm_resource_cursor cursor; |
602 | struct ttm_resource *res; |
603 | bool locked = false; |
604 | int ret; |
605 | |
606 | spin_lock(lock: &bdev->lru_lock); |
607 | ttm_resource_manager_for_each_res(man, &cursor, res) { |
608 | bool busy; |
609 | |
610 | if (!ttm_bo_evict_swapout_allowable(bo: res->bo, ctx, place, |
611 | locked: &locked, busy: &busy)) { |
612 | if (busy && !busy_bo && ticket != |
613 | dma_resv_locking_ctx(obj: res->bo->base.resv)) |
614 | busy_bo = res->bo; |
615 | continue; |
616 | } |
617 | |
618 | if (ttm_bo_get_unless_zero(bo: res->bo)) { |
619 | bo = res->bo; |
620 | break; |
621 | } |
622 | if (locked) |
623 | dma_resv_unlock(obj: res->bo->base.resv); |
624 | } |
625 | |
626 | if (!bo) { |
627 | if (busy_bo && !ttm_bo_get_unless_zero(bo: busy_bo)) |
628 | busy_bo = NULL; |
629 | spin_unlock(lock: &bdev->lru_lock); |
630 | ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); |
631 | if (busy_bo) |
632 | ttm_bo_put(busy_bo); |
633 | return ret; |
634 | } |
635 | |
636 | if (bo->deleted) { |
637 | ret = ttm_bo_cleanup_refs(bo, interruptible: ctx->interruptible, |
638 | no_wait_gpu: ctx->no_wait_gpu, unlock_resv: locked); |
639 | ttm_bo_put(bo); |
640 | return ret; |
641 | } |
642 | |
643 | spin_unlock(lock: &bdev->lru_lock); |
644 | |
645 | ret = ttm_bo_evict(bo, ctx); |
646 | if (locked) |
647 | ttm_bo_unreserve(bo); |
648 | else |
649 | ttm_bo_move_to_lru_tail_unlocked(bo); |
650 | |
651 | ttm_bo_put(bo); |
652 | return ret; |
653 | } |
654 | |
655 | /** |
656 | * ttm_bo_pin - Pin the buffer object. |
657 | * @bo: The buffer object to pin |
658 | * |
659 | * Make sure the buffer is not evicted any more during memory pressure. |
660 | * @bo must be unpinned again by calling ttm_bo_unpin(). |
661 | */ |
662 | void ttm_bo_pin(struct ttm_buffer_object *bo) |
663 | { |
664 | dma_resv_assert_held(bo->base.resv); |
665 | WARN_ON_ONCE(!kref_read(&bo->kref)); |
666 | spin_lock(lock: &bo->bdev->lru_lock); |
667 | if (bo->resource) |
668 | ttm_resource_del_bulk_move(res: bo->resource, bo); |
669 | ++bo->pin_count; |
670 | spin_unlock(lock: &bo->bdev->lru_lock); |
671 | } |
672 | EXPORT_SYMBOL(ttm_bo_pin); |
673 | |
674 | /** |
675 | * ttm_bo_unpin - Unpin the buffer object. |
676 | * @bo: The buffer object to unpin |
677 | * |
678 | * Allows the buffer object to be evicted again during memory pressure. |
679 | */ |
680 | void ttm_bo_unpin(struct ttm_buffer_object *bo) |
681 | { |
682 | dma_resv_assert_held(bo->base.resv); |
683 | WARN_ON_ONCE(!kref_read(&bo->kref)); |
684 | if (WARN_ON_ONCE(!bo->pin_count)) |
685 | return; |
686 | |
687 | spin_lock(lock: &bo->bdev->lru_lock); |
688 | --bo->pin_count; |
689 | if (bo->resource) |
690 | ttm_resource_add_bulk_move(res: bo->resource, bo); |
691 | spin_unlock(lock: &bo->bdev->lru_lock); |
692 | } |
693 | EXPORT_SYMBOL(ttm_bo_unpin); |
694 | |
695 | /* |
696 | * Add the last move fence to the BO as kernel dependency and reserve a new |
697 | * fence slot. |
698 | */ |
699 | static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, |
700 | struct ttm_resource_manager *man, |
701 | struct ttm_resource *mem, |
702 | bool no_wait_gpu) |
703 | { |
704 | struct dma_fence *fence; |
705 | int ret; |
706 | |
707 | spin_lock(lock: &man->move_lock); |
708 | fence = dma_fence_get(fence: man->move); |
709 | spin_unlock(lock: &man->move_lock); |
710 | |
711 | if (!fence) |
712 | return 0; |
713 | |
714 | if (no_wait_gpu) { |
715 | ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; |
716 | dma_fence_put(fence); |
717 | return ret; |
718 | } |
719 | |
720 | dma_resv_add_fence(obj: bo->base.resv, fence, usage: DMA_RESV_USAGE_KERNEL); |
721 | |
722 | ret = dma_resv_reserve_fences(obj: bo->base.resv, num_fences: 1); |
723 | dma_fence_put(fence); |
724 | return ret; |
725 | } |
726 | |
727 | /* |
728 | * Repeatedly evict memory from the LRU for @mem_type until we create enough |
729 | * space, or we've evicted everything and there isn't enough space. |
730 | */ |
731 | static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, |
732 | const struct ttm_place *place, |
733 | struct ttm_resource **mem, |
734 | struct ttm_operation_ctx *ctx) |
735 | { |
736 | struct ttm_device *bdev = bo->bdev; |
737 | struct ttm_resource_manager *man; |
738 | struct ww_acquire_ctx *ticket; |
739 | int ret; |
740 | |
741 | man = ttm_manager_type(bdev, mem_type: place->mem_type); |
742 | ticket = dma_resv_locking_ctx(obj: bo->base.resv); |
743 | do { |
744 | ret = ttm_resource_alloc(bo, place, res: mem); |
745 | if (likely(!ret)) |
746 | break; |
747 | if (unlikely(ret != -ENOSPC)) |
748 | return ret; |
749 | ret = ttm_mem_evict_first(bdev, man, place, ctx, |
750 | ticket); |
751 | if (unlikely(ret != 0)) |
752 | return ret; |
753 | } while (1); |
754 | |
755 | return ttm_bo_add_move_fence(bo, man, mem: *mem, no_wait_gpu: ctx->no_wait_gpu); |
756 | } |
757 | |
758 | /** |
759 | * ttm_bo_mem_space |
760 | * |
761 | * @bo: Pointer to a struct ttm_buffer_object. the data of which |
762 | * we want to allocate space for. |
763 | * @placement: Proposed new placement for the buffer object. |
764 | * @mem: A struct ttm_resource. |
765 | * @ctx: if and how to sleep, lock buffers and alloc memory |
766 | * |
767 | * Allocate memory space for the buffer object pointed to by @bo, using |
768 | * the placement flags in @placement, potentially evicting other idle buffer objects. |
769 | * This function may sleep while waiting for space to become available. |
770 | * Returns: |
771 | * -EBUSY: No space available (only if no_wait == 1). |
772 | * -ENOSPC: Could not allocate space for the buffer object, either due to |
773 | * fragmentation or concurrent allocators. |
774 | * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. |
775 | */ |
776 | int ttm_bo_mem_space(struct ttm_buffer_object *bo, |
777 | struct ttm_placement *placement, |
778 | struct ttm_resource **mem, |
779 | struct ttm_operation_ctx *ctx) |
780 | { |
781 | struct ttm_device *bdev = bo->bdev; |
782 | bool type_found = false; |
783 | int i, ret; |
784 | |
785 | ret = dma_resv_reserve_fences(obj: bo->base.resv, num_fences: 1); |
786 | if (unlikely(ret)) |
787 | return ret; |
788 | |
789 | for (i = 0; i < placement->num_placement; ++i) { |
790 | const struct ttm_place *place = &placement->placement[i]; |
791 | struct ttm_resource_manager *man; |
792 | |
793 | if (place->flags & TTM_PL_FLAG_FALLBACK) |
794 | continue; |
795 | |
796 | man = ttm_manager_type(bdev, mem_type: place->mem_type); |
797 | if (!man || !ttm_resource_manager_used(man)) |
798 | continue; |
799 | |
800 | type_found = true; |
801 | ret = ttm_resource_alloc(bo, place, res: mem); |
802 | if (ret == -ENOSPC) |
803 | continue; |
804 | if (unlikely(ret)) |
805 | goto error; |
806 | |
807 | ret = ttm_bo_add_move_fence(bo, man, mem: *mem, no_wait_gpu: ctx->no_wait_gpu); |
808 | if (unlikely(ret)) { |
809 | ttm_resource_free(bo, res: mem); |
810 | if (ret == -EBUSY) |
811 | continue; |
812 | |
813 | goto error; |
814 | } |
815 | return 0; |
816 | } |
817 | |
818 | for (i = 0; i < placement->num_placement; ++i) { |
819 | const struct ttm_place *place = &placement->placement[i]; |
820 | struct ttm_resource_manager *man; |
821 | |
822 | if (place->flags & TTM_PL_FLAG_DESIRED) |
823 | continue; |
824 | |
825 | man = ttm_manager_type(bdev, mem_type: place->mem_type); |
826 | if (!man || !ttm_resource_manager_used(man)) |
827 | continue; |
828 | |
829 | type_found = true; |
830 | ret = ttm_bo_mem_force_space(bo, place, mem, ctx); |
831 | if (likely(!ret)) |
832 | return 0; |
833 | |
834 | if (ret && ret != -EBUSY) |
835 | goto error; |
836 | } |
837 | |
838 | ret = -ENOSPC; |
839 | if (!type_found) { |
840 | pr_err(TTM_PFX "No compatible memory type found\n" ); |
841 | ret = -EINVAL; |
842 | } |
843 | |
844 | error: |
845 | return ret; |
846 | } |
847 | EXPORT_SYMBOL(ttm_bo_mem_space); |
848 | |
849 | static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, |
850 | struct ttm_placement *placement, |
851 | struct ttm_operation_ctx *ctx) |
852 | { |
853 | struct ttm_resource *mem; |
854 | struct ttm_place hop; |
855 | int ret; |
856 | |
857 | dma_resv_assert_held(bo->base.resv); |
858 | |
859 | /* |
860 | * Determine where to move the buffer. |
861 | * |
862 | * If driver determines move is going to need |
863 | * an extra step then it will return -EMULTIHOP |
864 | * and the buffer will be moved to the temporary |
865 | * stop and the driver will be called to make |
866 | * the second hop. |
867 | */ |
868 | ret = ttm_bo_mem_space(bo, placement, &mem, ctx); |
869 | if (ret) |
870 | return ret; |
871 | bounce: |
872 | ret = ttm_bo_handle_move_mem(bo, mem, evict: false, ctx, hop: &hop); |
873 | if (ret == -EMULTIHOP) { |
874 | ret = ttm_bo_bounce_temp_buffer(bo, mem: &mem, ctx, hop: &hop); |
875 | if (ret) |
876 | goto out; |
877 | /* try and move to final place now. */ |
878 | goto bounce; |
879 | } |
880 | out: |
881 | if (ret) |
882 | ttm_resource_free(bo, res: &mem); |
883 | return ret; |
884 | } |
885 | |
886 | /** |
887 | * ttm_bo_validate |
888 | * |
889 | * @bo: The buffer object. |
890 | * @placement: Proposed placement for the buffer object. |
891 | * @ctx: validation parameters. |
892 | * |
893 | * Changes placement and caching policy of the buffer object |
894 | * according proposed placement. |
895 | * Returns |
896 | * -EINVAL on invalid proposed placement. |
897 | * -ENOMEM on out-of-memory condition. |
898 | * -EBUSY if no_wait is true and buffer busy. |
899 | * -ERESTARTSYS if interrupted by a signal. |
900 | */ |
901 | int ttm_bo_validate(struct ttm_buffer_object *bo, |
902 | struct ttm_placement *placement, |
903 | struct ttm_operation_ctx *ctx) |
904 | { |
905 | int ret; |
906 | |
907 | dma_resv_assert_held(bo->base.resv); |
908 | |
909 | /* |
910 | * Remove the backing store if no placement is given. |
911 | */ |
912 | if (!placement->num_placement) |
913 | return ttm_bo_pipeline_gutting(bo); |
914 | |
915 | /* Check whether we need to move buffer. */ |
916 | if (bo->resource && ttm_resource_compatible(res: bo->resource, placement)) |
917 | return 0; |
918 | |
919 | /* Moving of pinned BOs is forbidden */ |
920 | if (bo->pin_count) |
921 | return -EINVAL; |
922 | |
923 | ret = ttm_bo_move_buffer(bo, placement, ctx); |
924 | /* For backward compatibility with userspace */ |
925 | if (ret == -ENOSPC) |
926 | return -ENOMEM; |
927 | if (ret) |
928 | return ret; |
929 | |
930 | /* |
931 | * We might need to add a TTM. |
932 | */ |
933 | if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { |
934 | ret = ttm_tt_create(bo, zero_alloc: true); |
935 | if (ret) |
936 | return ret; |
937 | } |
938 | return 0; |
939 | } |
940 | EXPORT_SYMBOL(ttm_bo_validate); |
941 | |
942 | /** |
943 | * ttm_bo_init_reserved |
944 | * |
945 | * @bdev: Pointer to a ttm_device struct. |
946 | * @bo: Pointer to a ttm_buffer_object to be initialized. |
947 | * @type: Requested type of buffer object. |
948 | * @placement: Initial placement for buffer object. |
949 | * @alignment: Data alignment in pages. |
950 | * @ctx: TTM operation context for memory allocation. |
951 | * @sg: Scatter-gather table. |
952 | * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. |
953 | * @destroy: Destroy function. Use NULL for kfree(). |
954 | * |
955 | * This function initializes a pre-allocated struct ttm_buffer_object. |
956 | * As this object may be part of a larger structure, this function, |
957 | * together with the @destroy function, enables driver-specific objects |
958 | * derived from a ttm_buffer_object. |
959 | * |
960 | * On successful return, the caller owns an object kref to @bo. The kref and |
961 | * list_kref are usually set to 1, but note that in some situations, other |
962 | * tasks may already be holding references to @bo as well. |
963 | * Furthermore, if resv == NULL, the buffer's reservation lock will be held, |
964 | * and it is the caller's responsibility to call ttm_bo_unreserve. |
965 | * |
966 | * If a failure occurs, the function will call the @destroy function. Thus, |
967 | * after a failure, dereferencing @bo is illegal and will likely cause memory |
968 | * corruption. |
969 | * |
970 | * Returns |
971 | * -ENOMEM: Out of memory. |
972 | * -EINVAL: Invalid placement flags. |
973 | * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. |
974 | */ |
975 | int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, |
976 | enum ttm_bo_type type, struct ttm_placement *placement, |
977 | uint32_t alignment, struct ttm_operation_ctx *ctx, |
978 | struct sg_table *sg, struct dma_resv *resv, |
979 | void (*destroy) (struct ttm_buffer_object *)) |
980 | { |
981 | int ret; |
982 | |
983 | kref_init(kref: &bo->kref); |
984 | bo->bdev = bdev; |
985 | bo->type = type; |
986 | bo->page_alignment = alignment; |
987 | bo->destroy = destroy; |
988 | bo->pin_count = 0; |
989 | bo->sg = sg; |
990 | bo->bulk_move = NULL; |
991 | if (resv) |
992 | bo->base.resv = resv; |
993 | else |
994 | bo->base.resv = &bo->base._resv; |
995 | atomic_inc(v: &ttm_glob.bo_count); |
996 | |
997 | /* |
998 | * For ttm_bo_type_device buffers, allocate |
999 | * address space from the device. |
1000 | */ |
1001 | if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { |
1002 | ret = drm_vma_offset_add(mgr: bdev->vma_manager, node: &bo->base.vma_node, |
1003 | PFN_UP(bo->base.size)); |
1004 | if (ret) |
1005 | goto err_put; |
1006 | } |
1007 | |
1008 | /* passed reservation objects should already be locked, |
1009 | * since otherwise lockdep will be angered in radeon. |
1010 | */ |
1011 | if (!resv) |
1012 | WARN_ON(!dma_resv_trylock(bo->base.resv)); |
1013 | else |
1014 | dma_resv_assert_held(resv); |
1015 | |
1016 | ret = ttm_bo_validate(bo, placement, ctx); |
1017 | if (unlikely(ret)) |
1018 | goto err_unlock; |
1019 | |
1020 | return 0; |
1021 | |
1022 | err_unlock: |
1023 | if (!resv) |
1024 | dma_resv_unlock(obj: bo->base.resv); |
1025 | |
1026 | err_put: |
1027 | ttm_bo_put(bo); |
1028 | return ret; |
1029 | } |
1030 | EXPORT_SYMBOL(ttm_bo_init_reserved); |
1031 | |
1032 | /** |
1033 | * ttm_bo_init_validate |
1034 | * |
1035 | * @bdev: Pointer to a ttm_device struct. |
1036 | * @bo: Pointer to a ttm_buffer_object to be initialized. |
1037 | * @type: Requested type of buffer object. |
1038 | * @placement: Initial placement for buffer object. |
1039 | * @alignment: Data alignment in pages. |
1040 | * @interruptible: If needing to sleep to wait for GPU resources, |
1041 | * sleep interruptible. |
1042 | * pinned in physical memory. If this behaviour is not desired, this member |
1043 | * holds a pointer to a persistent shmem object. Typically, this would |
1044 | * point to the shmem object backing a GEM object if TTM is used to back a |
1045 | * GEM user interface. |
1046 | * @sg: Scatter-gather table. |
1047 | * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. |
1048 | * @destroy: Destroy function. Use NULL for kfree(). |
1049 | * |
1050 | * This function initializes a pre-allocated struct ttm_buffer_object. |
1051 | * As this object may be part of a larger structure, this function, |
1052 | * together with the @destroy function, |
1053 | * enables driver-specific objects derived from a ttm_buffer_object. |
1054 | * |
1055 | * On successful return, the caller owns an object kref to @bo. The kref and |
1056 | * list_kref are usually set to 1, but note that in some situations, other |
1057 | * tasks may already be holding references to @bo as well. |
1058 | * |
1059 | * If a failure occurs, the function will call the @destroy function, Thus, |
1060 | * after a failure, dereferencing @bo is illegal and will likely cause memory |
1061 | * corruption. |
1062 | * |
1063 | * Returns |
1064 | * -ENOMEM: Out of memory. |
1065 | * -EINVAL: Invalid placement flags. |
1066 | * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. |
1067 | */ |
1068 | int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, |
1069 | enum ttm_bo_type type, struct ttm_placement *placement, |
1070 | uint32_t alignment, bool interruptible, |
1071 | struct sg_table *sg, struct dma_resv *resv, |
1072 | void (*destroy) (struct ttm_buffer_object *)) |
1073 | { |
1074 | struct ttm_operation_ctx ctx = { interruptible, false }; |
1075 | int ret; |
1076 | |
1077 | ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, |
1078 | sg, resv, destroy); |
1079 | if (ret) |
1080 | return ret; |
1081 | |
1082 | if (!resv) |
1083 | ttm_bo_unreserve(bo); |
1084 | |
1085 | return 0; |
1086 | } |
1087 | EXPORT_SYMBOL(ttm_bo_init_validate); |
1088 | |
1089 | /* |
1090 | * buffer object vm functions. |
1091 | */ |
1092 | |
1093 | /** |
1094 | * ttm_bo_unmap_virtual |
1095 | * |
1096 | * @bo: tear down the virtual mappings for this BO |
1097 | */ |
1098 | void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) |
1099 | { |
1100 | struct ttm_device *bdev = bo->bdev; |
1101 | |
1102 | drm_vma_node_unmap(node: &bo->base.vma_node, file_mapping: bdev->dev_mapping); |
1103 | ttm_mem_io_free(bdev, mem: bo->resource); |
1104 | } |
1105 | EXPORT_SYMBOL(ttm_bo_unmap_virtual); |
1106 | |
1107 | /** |
1108 | * ttm_bo_wait_ctx - wait for buffer idle. |
1109 | * |
1110 | * @bo: The buffer object. |
1111 | * @ctx: defines how to wait |
1112 | * |
1113 | * Waits for the buffer to be idle. Used timeout depends on the context. |
1114 | * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or |
1115 | * zero on success. |
1116 | */ |
1117 | int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) |
1118 | { |
1119 | long ret; |
1120 | |
1121 | if (ctx->no_wait_gpu) { |
1122 | if (dma_resv_test_signaled(obj: bo->base.resv, |
1123 | usage: DMA_RESV_USAGE_BOOKKEEP)) |
1124 | return 0; |
1125 | else |
1126 | return -EBUSY; |
1127 | } |
1128 | |
1129 | ret = dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP, |
1130 | intr: ctx->interruptible, timeout: 15 * HZ); |
1131 | if (unlikely(ret < 0)) |
1132 | return ret; |
1133 | if (unlikely(ret == 0)) |
1134 | return -EBUSY; |
1135 | return 0; |
1136 | } |
1137 | EXPORT_SYMBOL(ttm_bo_wait_ctx); |
1138 | |
1139 | int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, |
1140 | gfp_t gfp_flags) |
1141 | { |
1142 | struct ttm_place place; |
1143 | bool locked; |
1144 | long ret; |
1145 | |
1146 | /* |
1147 | * While the bo may already reside in SYSTEM placement, set |
1148 | * SYSTEM as new placement to cover also the move further below. |
1149 | * The driver may use the fact that we're moving from SYSTEM |
1150 | * as an indication that we're about to swap out. |
1151 | */ |
1152 | memset(&place, 0, sizeof(place)); |
1153 | place.mem_type = bo->resource->mem_type; |
1154 | if (!ttm_bo_evict_swapout_allowable(bo, ctx, place: &place, locked: &locked, NULL)) |
1155 | return -EBUSY; |
1156 | |
1157 | if (!bo->ttm || !ttm_tt_is_populated(tt: bo->ttm) || |
1158 | bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || |
1159 | bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED || |
1160 | !ttm_bo_get_unless_zero(bo)) { |
1161 | if (locked) |
1162 | dma_resv_unlock(obj: bo->base.resv); |
1163 | return -EBUSY; |
1164 | } |
1165 | |
1166 | if (bo->deleted) { |
1167 | ret = ttm_bo_cleanup_refs(bo, interruptible: false, no_wait_gpu: false, unlock_resv: locked); |
1168 | ttm_bo_put(bo); |
1169 | return ret == -EBUSY ? -ENOSPC : ret; |
1170 | } |
1171 | |
1172 | /* TODO: Cleanup the locking */ |
1173 | spin_unlock(lock: &bo->bdev->lru_lock); |
1174 | |
1175 | /* |
1176 | * Move to system cached |
1177 | */ |
1178 | if (bo->resource->mem_type != TTM_PL_SYSTEM) { |
1179 | struct ttm_resource *evict_mem; |
1180 | struct ttm_place hop; |
1181 | |
1182 | memset(&hop, 0, sizeof(hop)); |
1183 | place.mem_type = TTM_PL_SYSTEM; |
1184 | ret = ttm_resource_alloc(bo, place: &place, res: &evict_mem); |
1185 | if (unlikely(ret)) |
1186 | goto out; |
1187 | |
1188 | ret = ttm_bo_handle_move_mem(bo, mem: evict_mem, evict: true, ctx, hop: &hop); |
1189 | if (unlikely(ret != 0)) { |
1190 | WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n" ); |
1191 | ttm_resource_free(bo, res: &evict_mem); |
1192 | goto out; |
1193 | } |
1194 | } |
1195 | |
1196 | /* |
1197 | * Make sure BO is idle. |
1198 | */ |
1199 | ret = ttm_bo_wait_ctx(bo, ctx); |
1200 | if (unlikely(ret != 0)) |
1201 | goto out; |
1202 | |
1203 | ttm_bo_unmap_virtual(bo); |
1204 | |
1205 | /* |
1206 | * Swap out. Buffer will be swapped in again as soon as |
1207 | * anyone tries to access a ttm page. |
1208 | */ |
1209 | if (bo->bdev->funcs->swap_notify) |
1210 | bo->bdev->funcs->swap_notify(bo); |
1211 | |
1212 | if (ttm_tt_is_populated(tt: bo->ttm)) |
1213 | ret = ttm_tt_swapout(bdev: bo->bdev, ttm: bo->ttm, gfp_flags); |
1214 | out: |
1215 | |
1216 | /* |
1217 | * Unreserve without putting on LRU to avoid swapping out an |
1218 | * already swapped buffer. |
1219 | */ |
1220 | if (locked) |
1221 | dma_resv_unlock(obj: bo->base.resv); |
1222 | ttm_bo_put(bo); |
1223 | return ret == -EBUSY ? -ENOSPC : ret; |
1224 | } |
1225 | |
1226 | void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) |
1227 | { |
1228 | if (bo->ttm == NULL) |
1229 | return; |
1230 | |
1231 | ttm_tt_unpopulate(bdev: bo->bdev, ttm: bo->ttm); |
1232 | ttm_tt_destroy(bdev: bo->bdev, ttm: bo->ttm); |
1233 | bo->ttm = NULL; |
1234 | } |
1235 | |