1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
2 | /************************************************************************** |
3 | * |
4 | * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA |
5 | * All Rights Reserved. |
6 | * |
7 | * Permission is hereby granted, free of charge, to any person obtaining a |
8 | * copy of this software and associated documentation files (the |
9 | * "Software"), to deal in the Software without restriction, including |
10 | * without limitation the rights to use, copy, modify, merge, publish, |
11 | * distribute, sub license, and/or sell copies of the Software, and to |
12 | * permit persons to whom the Software is furnished to do so, subject to |
13 | * the following conditions: |
14 | * |
15 | * The above copyright notice and this permission notice (including the |
16 | * next paragraph) shall be included in all copies or substantial portions |
17 | * of the Software. |
18 | * |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
26 | * |
27 | **************************************************************************/ |
28 | /* |
29 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
30 | */ |
31 | |
32 | #include <linux/vmalloc.h> |
33 | |
34 | #include <drm/ttm/ttm_bo.h> |
35 | #include <drm/ttm/ttm_placement.h> |
36 | #include <drm/ttm/ttm_tt.h> |
37 | |
38 | #include <drm/drm_cache.h> |
39 | |
40 | struct ttm_transfer_obj { |
41 | struct ttm_buffer_object base; |
42 | struct ttm_buffer_object *bo; |
43 | }; |
44 | |
45 | int ttm_mem_io_reserve(struct ttm_device *bdev, |
46 | struct ttm_resource *mem) |
47 | { |
48 | if (mem->bus.offset || mem->bus.addr) |
49 | return 0; |
50 | |
51 | mem->bus.is_iomem = false; |
52 | if (!bdev->funcs->io_mem_reserve) |
53 | return 0; |
54 | |
55 | return bdev->funcs->io_mem_reserve(bdev, mem); |
56 | } |
57 | |
58 | void ttm_mem_io_free(struct ttm_device *bdev, |
59 | struct ttm_resource *mem) |
60 | { |
61 | if (!mem) |
62 | return; |
63 | |
64 | if (!mem->bus.offset && !mem->bus.addr) |
65 | return; |
66 | |
67 | if (bdev->funcs->io_mem_free) |
68 | bdev->funcs->io_mem_free(bdev, mem); |
69 | |
70 | mem->bus.offset = 0; |
71 | mem->bus.addr = NULL; |
72 | } |
73 | |
74 | /** |
75 | * ttm_move_memcpy - Helper to perform a memcpy ttm move operation. |
76 | * @clear: Whether to clear rather than copy. |
77 | * @num_pages: Number of pages of the operation. |
78 | * @dst_iter: A struct ttm_kmap_iter representing the destination resource. |
79 | * @src_iter: A struct ttm_kmap_iter representing the source resource. |
80 | * |
81 | * This function is intended to be able to move out async under a |
82 | * dma-fence if desired. |
83 | */ |
84 | void ttm_move_memcpy(bool clear, |
85 | u32 num_pages, |
86 | struct ttm_kmap_iter *dst_iter, |
87 | struct ttm_kmap_iter *src_iter) |
88 | { |
89 | const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops; |
90 | const struct ttm_kmap_iter_ops *src_ops = src_iter->ops; |
91 | struct iosys_map src_map, dst_map; |
92 | pgoff_t i; |
93 | |
94 | /* Single TTM move. NOP */ |
95 | if (dst_ops->maps_tt && src_ops->maps_tt) |
96 | return; |
97 | |
98 | /* Don't move nonexistent data. Clear destination instead. */ |
99 | if (clear) { |
100 | for (i = 0; i < num_pages; ++i) { |
101 | dst_ops->map_local(dst_iter, &dst_map, i); |
102 | if (dst_map.is_iomem) |
103 | memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE); |
104 | else |
105 | memset(dst_map.vaddr, 0, PAGE_SIZE); |
106 | if (dst_ops->unmap_local) |
107 | dst_ops->unmap_local(dst_iter, &dst_map); |
108 | } |
109 | return; |
110 | } |
111 | |
112 | for (i = 0; i < num_pages; ++i) { |
113 | dst_ops->map_local(dst_iter, &dst_map, i); |
114 | src_ops->map_local(src_iter, &src_map, i); |
115 | |
116 | drm_memcpy_from_wc(dst: &dst_map, src: &src_map, PAGE_SIZE); |
117 | |
118 | if (src_ops->unmap_local) |
119 | src_ops->unmap_local(src_iter, &src_map); |
120 | if (dst_ops->unmap_local) |
121 | dst_ops->unmap_local(dst_iter, &dst_map); |
122 | } |
123 | } |
124 | EXPORT_SYMBOL(ttm_move_memcpy); |
125 | |
126 | /** |
127 | * ttm_bo_move_memcpy |
128 | * |
129 | * @bo: A pointer to a struct ttm_buffer_object. |
130 | * @ctx: operation context |
131 | * @dst_mem: struct ttm_resource indicating where to move. |
132 | * |
133 | * Fallback move function for a mappable buffer object in mappable memory. |
134 | * The function will, if successful, |
135 | * free any old aperture space, and set (@new_mem)->mm_node to NULL, |
136 | * and update the (@bo)->mem placement flags. If unsuccessful, the old |
137 | * data remains untouched, and it's up to the caller to free the |
138 | * memory space indicated by @new_mem. |
139 | * Returns: |
140 | * !0: Failure. |
141 | */ |
142 | int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, |
143 | struct ttm_operation_ctx *ctx, |
144 | struct ttm_resource *dst_mem) |
145 | { |
146 | struct ttm_device *bdev = bo->bdev; |
147 | struct ttm_resource_manager *dst_man = |
148 | ttm_manager_type(bdev: bo->bdev, mem_type: dst_mem->mem_type); |
149 | struct ttm_tt *ttm = bo->ttm; |
150 | struct ttm_resource *src_mem = bo->resource; |
151 | struct ttm_resource_manager *src_man; |
152 | union { |
153 | struct ttm_kmap_iter_tt tt; |
154 | struct ttm_kmap_iter_linear_io io; |
155 | } _dst_iter, _src_iter; |
156 | struct ttm_kmap_iter *dst_iter, *src_iter; |
157 | bool clear; |
158 | int ret = 0; |
159 | |
160 | if (WARN_ON(!src_mem)) |
161 | return -EINVAL; |
162 | |
163 | src_man = ttm_manager_type(bdev, mem_type: src_mem->mem_type); |
164 | if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) || |
165 | dst_man->use_tt)) { |
166 | ret = ttm_tt_populate(bdev, ttm, ctx); |
167 | if (ret) |
168 | return ret; |
169 | } |
170 | |
171 | dst_iter = ttm_kmap_iter_linear_io_init(iter_io: &_dst_iter.io, bdev, mem: dst_mem); |
172 | if (PTR_ERR(ptr: dst_iter) == -EINVAL && dst_man->use_tt) |
173 | dst_iter = ttm_kmap_iter_tt_init(iter_tt: &_dst_iter.tt, tt: bo->ttm); |
174 | if (IS_ERR(ptr: dst_iter)) |
175 | return PTR_ERR(ptr: dst_iter); |
176 | |
177 | src_iter = ttm_kmap_iter_linear_io_init(iter_io: &_src_iter.io, bdev, mem: src_mem); |
178 | if (PTR_ERR(ptr: src_iter) == -EINVAL && src_man->use_tt) |
179 | src_iter = ttm_kmap_iter_tt_init(iter_tt: &_src_iter.tt, tt: bo->ttm); |
180 | if (IS_ERR(ptr: src_iter)) { |
181 | ret = PTR_ERR(ptr: src_iter); |
182 | goto out_src_iter; |
183 | } |
184 | |
185 | clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(tt: ttm)); |
186 | if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) |
187 | ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter); |
188 | |
189 | if (!src_iter->ops->maps_tt) |
190 | ttm_kmap_iter_linear_io_fini(iter_io: &_src_iter.io, bdev, mem: src_mem); |
191 | ttm_bo_move_sync_cleanup(bo, new_mem: dst_mem); |
192 | |
193 | out_src_iter: |
194 | if (!dst_iter->ops->maps_tt) |
195 | ttm_kmap_iter_linear_io_fini(iter_io: &_dst_iter.io, bdev, mem: dst_mem); |
196 | |
197 | return ret; |
198 | } |
199 | EXPORT_SYMBOL(ttm_bo_move_memcpy); |
200 | |
201 | static void ttm_transfered_destroy(struct ttm_buffer_object *bo) |
202 | { |
203 | struct ttm_transfer_obj *fbo; |
204 | |
205 | fbo = container_of(bo, struct ttm_transfer_obj, base); |
206 | dma_resv_fini(obj: &fbo->base.base._resv); |
207 | ttm_bo_put(bo: fbo->bo); |
208 | kfree(objp: fbo); |
209 | } |
210 | |
211 | /** |
212 | * ttm_buffer_object_transfer |
213 | * |
214 | * @bo: A pointer to a struct ttm_buffer_object. |
215 | * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, |
216 | * holding the data of @bo with the old placement. |
217 | * |
218 | * This is a utility function that may be called after an accelerated move |
219 | * has been scheduled. A new buffer object is created as a placeholder for |
220 | * the old data while it's being copied. When that buffer object is idle, |
221 | * it can be destroyed, releasing the space of the old placement. |
222 | * Returns: |
223 | * !0: Failure. |
224 | */ |
225 | |
226 | static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, |
227 | struct ttm_buffer_object **new_obj) |
228 | { |
229 | struct ttm_transfer_obj *fbo; |
230 | int ret; |
231 | |
232 | fbo = kmalloc(size: sizeof(*fbo), GFP_KERNEL); |
233 | if (!fbo) |
234 | return -ENOMEM; |
235 | |
236 | fbo->base = *bo; |
237 | |
238 | /** |
239 | * Fix up members that we shouldn't copy directly: |
240 | * TODO: Explicit member copy would probably be better here. |
241 | */ |
242 | |
243 | atomic_inc(v: &ttm_glob.bo_count); |
244 | drm_vma_node_reset(node: &fbo->base.base.vma_node); |
245 | |
246 | kref_init(kref: &fbo->base.kref); |
247 | fbo->base.destroy = &ttm_transfered_destroy; |
248 | fbo->base.pin_count = 0; |
249 | if (bo->type != ttm_bo_type_sg) |
250 | fbo->base.base.resv = &fbo->base.base._resv; |
251 | |
252 | dma_resv_init(obj: &fbo->base.base._resv); |
253 | fbo->base.base.dev = NULL; |
254 | ret = dma_resv_trylock(obj: &fbo->base.base._resv); |
255 | WARN_ON(!ret); |
256 | |
257 | if (fbo->base.resource) { |
258 | ttm_resource_set_bo(res: fbo->base.resource, bo: &fbo->base); |
259 | bo->resource = NULL; |
260 | ttm_bo_set_bulk_move(bo: &fbo->base, NULL); |
261 | } else { |
262 | fbo->base.bulk_move = NULL; |
263 | } |
264 | |
265 | ret = dma_resv_reserve_fences(obj: &fbo->base.base._resv, num_fences: 1); |
266 | if (ret) { |
267 | kfree(objp: fbo); |
268 | return ret; |
269 | } |
270 | |
271 | ttm_bo_get(bo); |
272 | fbo->bo = bo; |
273 | |
274 | ttm_bo_move_to_lru_tail_unlocked(bo: &fbo->base); |
275 | |
276 | *new_obj = &fbo->base; |
277 | return 0; |
278 | } |
279 | |
280 | /** |
281 | * ttm_io_prot |
282 | * |
283 | * @bo: ttm buffer object |
284 | * @res: ttm resource object |
285 | * @tmp: Page protection flag for a normal, cached mapping. |
286 | * |
287 | * Utility function that returns the pgprot_t that should be used for |
288 | * setting up a PTE with the caching model indicated by @c_state. |
289 | */ |
290 | pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, |
291 | pgprot_t tmp) |
292 | { |
293 | struct ttm_resource_manager *man; |
294 | enum ttm_caching caching; |
295 | |
296 | man = ttm_manager_type(bdev: bo->bdev, mem_type: res->mem_type); |
297 | if (man->use_tt) { |
298 | caching = bo->ttm->caching; |
299 | if (bo->ttm->page_flags & TTM_TT_FLAG_DECRYPTED) |
300 | tmp = pgprot_decrypted(tmp); |
301 | } else { |
302 | caching = res->bus.caching; |
303 | } |
304 | |
305 | return ttm_prot_from_caching(caching, tmp); |
306 | } |
307 | EXPORT_SYMBOL(ttm_io_prot); |
308 | |
309 | static int ttm_bo_ioremap(struct ttm_buffer_object *bo, |
310 | unsigned long offset, |
311 | unsigned long size, |
312 | struct ttm_bo_kmap_obj *map) |
313 | { |
314 | struct ttm_resource *mem = bo->resource; |
315 | |
316 | if (bo->resource->bus.addr) { |
317 | map->bo_kmap_type = ttm_bo_map_premapped; |
318 | map->virtual = ((u8 *)bo->resource->bus.addr) + offset; |
319 | } else { |
320 | resource_size_t res = bo->resource->bus.offset + offset; |
321 | |
322 | map->bo_kmap_type = ttm_bo_map_iomap; |
323 | if (mem->bus.caching == ttm_write_combined) |
324 | map->virtual = ioremap_wc(offset: res, size); |
325 | #ifdef CONFIG_X86 |
326 | else if (mem->bus.caching == ttm_cached) |
327 | map->virtual = ioremap_cache(offset: res, size); |
328 | #endif |
329 | else |
330 | map->virtual = ioremap(offset: res, size); |
331 | } |
332 | return (!map->virtual) ? -ENOMEM : 0; |
333 | } |
334 | |
335 | static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, |
336 | unsigned long start_page, |
337 | unsigned long num_pages, |
338 | struct ttm_bo_kmap_obj *map) |
339 | { |
340 | struct ttm_resource *mem = bo->resource; |
341 | struct ttm_operation_ctx ctx = { |
342 | .interruptible = false, |
343 | .no_wait_gpu = false |
344 | }; |
345 | struct ttm_tt *ttm = bo->ttm; |
346 | struct ttm_resource_manager *man = |
347 | ttm_manager_type(bdev: bo->bdev, mem_type: bo->resource->mem_type); |
348 | pgprot_t prot; |
349 | int ret; |
350 | |
351 | BUG_ON(!ttm); |
352 | |
353 | ret = ttm_tt_populate(bdev: bo->bdev, ttm, ctx: &ctx); |
354 | if (ret) |
355 | return ret; |
356 | |
357 | if (num_pages == 1 && ttm->caching == ttm_cached && |
358 | !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) { |
359 | /* |
360 | * We're mapping a single page, and the desired |
361 | * page protection is consistent with the bo. |
362 | */ |
363 | |
364 | map->bo_kmap_type = ttm_bo_map_kmap; |
365 | map->page = ttm->pages[start_page]; |
366 | map->virtual = kmap(page: map->page); |
367 | } else { |
368 | /* |
369 | * We need to use vmap to get the desired page protection |
370 | * or to make the buffer object look contiguous. |
371 | */ |
372 | prot = ttm_io_prot(bo, mem, PAGE_KERNEL); |
373 | map->bo_kmap_type = ttm_bo_map_vmap; |
374 | map->virtual = vmap(pages: ttm->pages + start_page, count: num_pages, |
375 | flags: 0, prot); |
376 | } |
377 | return (!map->virtual) ? -ENOMEM : 0; |
378 | } |
379 | |
380 | /** |
381 | * ttm_bo_kmap |
382 | * |
383 | * @bo: The buffer object. |
384 | * @start_page: The first page to map. |
385 | * @num_pages: Number of pages to map. |
386 | * @map: pointer to a struct ttm_bo_kmap_obj representing the map. |
387 | * |
388 | * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the |
389 | * data in the buffer object. The ttm_kmap_obj_virtual function can then be |
390 | * used to obtain a virtual address to the data. |
391 | * |
392 | * Returns |
393 | * -ENOMEM: Out of memory. |
394 | * -EINVAL: Invalid range. |
395 | */ |
396 | int ttm_bo_kmap(struct ttm_buffer_object *bo, |
397 | unsigned long start_page, unsigned long num_pages, |
398 | struct ttm_bo_kmap_obj *map) |
399 | { |
400 | unsigned long offset, size; |
401 | int ret; |
402 | |
403 | map->virtual = NULL; |
404 | map->bo = bo; |
405 | if (num_pages > PFN_UP(bo->resource->size)) |
406 | return -EINVAL; |
407 | if ((start_page + num_pages) > PFN_UP(bo->resource->size)) |
408 | return -EINVAL; |
409 | |
410 | ret = ttm_mem_io_reserve(bdev: bo->bdev, mem: bo->resource); |
411 | if (ret) |
412 | return ret; |
413 | if (!bo->resource->bus.is_iomem) { |
414 | return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); |
415 | } else { |
416 | offset = start_page << PAGE_SHIFT; |
417 | size = num_pages << PAGE_SHIFT; |
418 | return ttm_bo_ioremap(bo, offset, size, map); |
419 | } |
420 | } |
421 | EXPORT_SYMBOL(ttm_bo_kmap); |
422 | |
423 | /** |
424 | * ttm_bo_kunmap |
425 | * |
426 | * @map: Object describing the map to unmap. |
427 | * |
428 | * Unmaps a kernel map set up by ttm_bo_kmap. |
429 | */ |
430 | void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) |
431 | { |
432 | if (!map->virtual) |
433 | return; |
434 | switch (map->bo_kmap_type) { |
435 | case ttm_bo_map_iomap: |
436 | iounmap(addr: map->virtual); |
437 | break; |
438 | case ttm_bo_map_vmap: |
439 | vunmap(addr: map->virtual); |
440 | break; |
441 | case ttm_bo_map_kmap: |
442 | kunmap(page: map->page); |
443 | break; |
444 | case ttm_bo_map_premapped: |
445 | break; |
446 | default: |
447 | BUG(); |
448 | } |
449 | ttm_mem_io_free(bdev: map->bo->bdev, mem: map->bo->resource); |
450 | map->virtual = NULL; |
451 | map->page = NULL; |
452 | } |
453 | EXPORT_SYMBOL(ttm_bo_kunmap); |
454 | |
455 | /** |
456 | * ttm_bo_vmap |
457 | * |
458 | * @bo: The buffer object. |
459 | * @map: pointer to a struct iosys_map representing the map. |
460 | * |
461 | * Sets up a kernel virtual mapping, using ioremap or vmap to the |
462 | * data in the buffer object. The parameter @map returns the virtual |
463 | * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap(). |
464 | * |
465 | * Returns |
466 | * -ENOMEM: Out of memory. |
467 | * -EINVAL: Invalid range. |
468 | */ |
469 | int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) |
470 | { |
471 | struct ttm_resource *mem = bo->resource; |
472 | int ret; |
473 | |
474 | dma_resv_assert_held(bo->base.resv); |
475 | |
476 | ret = ttm_mem_io_reserve(bdev: bo->bdev, mem); |
477 | if (ret) |
478 | return ret; |
479 | |
480 | if (mem->bus.is_iomem) { |
481 | void __iomem *vaddr_iomem; |
482 | |
483 | if (mem->bus.addr) |
484 | vaddr_iomem = (void __iomem *)mem->bus.addr; |
485 | else if (mem->bus.caching == ttm_write_combined) |
486 | vaddr_iomem = ioremap_wc(offset: mem->bus.offset, |
487 | size: bo->base.size); |
488 | #ifdef CONFIG_X86 |
489 | else if (mem->bus.caching == ttm_cached) |
490 | vaddr_iomem = ioremap_cache(offset: mem->bus.offset, |
491 | size: bo->base.size); |
492 | #endif |
493 | else |
494 | vaddr_iomem = ioremap(offset: mem->bus.offset, size: bo->base.size); |
495 | |
496 | if (!vaddr_iomem) |
497 | return -ENOMEM; |
498 | |
499 | iosys_map_set_vaddr_iomem(map, vaddr_iomem); |
500 | |
501 | } else { |
502 | struct ttm_operation_ctx ctx = { |
503 | .interruptible = false, |
504 | .no_wait_gpu = false |
505 | }; |
506 | struct ttm_tt *ttm = bo->ttm; |
507 | pgprot_t prot; |
508 | void *vaddr; |
509 | |
510 | ret = ttm_tt_populate(bdev: bo->bdev, ttm, ctx: &ctx); |
511 | if (ret) |
512 | return ret; |
513 | |
514 | /* |
515 | * We need to use vmap to get the desired page protection |
516 | * or to make the buffer object look contiguous. |
517 | */ |
518 | prot = ttm_io_prot(bo, mem, PAGE_KERNEL); |
519 | vaddr = vmap(pages: ttm->pages, count: ttm->num_pages, flags: 0, prot); |
520 | if (!vaddr) |
521 | return -ENOMEM; |
522 | |
523 | iosys_map_set_vaddr(map, vaddr); |
524 | } |
525 | |
526 | return 0; |
527 | } |
528 | EXPORT_SYMBOL(ttm_bo_vmap); |
529 | |
530 | /** |
531 | * ttm_bo_vunmap |
532 | * |
533 | * @bo: The buffer object. |
534 | * @map: Object describing the map to unmap. |
535 | * |
536 | * Unmaps a kernel map set up by ttm_bo_vmap(). |
537 | */ |
538 | void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map) |
539 | { |
540 | struct ttm_resource *mem = bo->resource; |
541 | |
542 | dma_resv_assert_held(bo->base.resv); |
543 | |
544 | if (iosys_map_is_null(map)) |
545 | return; |
546 | |
547 | if (!map->is_iomem) |
548 | vunmap(addr: map->vaddr); |
549 | else if (!mem->bus.addr) |
550 | iounmap(addr: map->vaddr_iomem); |
551 | iosys_map_clear(map); |
552 | |
553 | ttm_mem_io_free(bdev: bo->bdev, mem: bo->resource); |
554 | } |
555 | EXPORT_SYMBOL(ttm_bo_vunmap); |
556 | |
557 | static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, |
558 | bool dst_use_tt) |
559 | { |
560 | long ret; |
561 | |
562 | ret = dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP, |
563 | intr: false, timeout: 15 * HZ); |
564 | if (ret == 0) |
565 | return -EBUSY; |
566 | if (ret < 0) |
567 | return ret; |
568 | |
569 | if (!dst_use_tt) |
570 | ttm_bo_tt_destroy(bo); |
571 | ttm_resource_free(bo, res: &bo->resource); |
572 | return 0; |
573 | } |
574 | |
575 | static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, |
576 | struct dma_fence *fence, |
577 | bool dst_use_tt) |
578 | { |
579 | struct ttm_buffer_object *ghost_obj; |
580 | int ret; |
581 | |
582 | /** |
583 | * This should help pipeline ordinary buffer moves. |
584 | * |
585 | * Hang old buffer memory on a new buffer object, |
586 | * and leave it to be released when the GPU |
587 | * operation has completed. |
588 | */ |
589 | |
590 | ret = ttm_buffer_object_transfer(bo, new_obj: &ghost_obj); |
591 | if (ret) |
592 | return ret; |
593 | |
594 | dma_resv_add_fence(obj: &ghost_obj->base._resv, fence, |
595 | usage: DMA_RESV_USAGE_KERNEL); |
596 | |
597 | /** |
598 | * If we're not moving to fixed memory, the TTM object |
599 | * needs to stay alive. Otherwhise hang it on the ghost |
600 | * bo to be unbound and destroyed. |
601 | */ |
602 | |
603 | if (dst_use_tt) |
604 | ghost_obj->ttm = NULL; |
605 | else |
606 | bo->ttm = NULL; |
607 | |
608 | dma_resv_unlock(obj: &ghost_obj->base._resv); |
609 | ttm_bo_put(bo: ghost_obj); |
610 | return 0; |
611 | } |
612 | |
613 | static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, |
614 | struct dma_fence *fence) |
615 | { |
616 | struct ttm_device *bdev = bo->bdev; |
617 | struct ttm_resource_manager *from; |
618 | |
619 | from = ttm_manager_type(bdev, mem_type: bo->resource->mem_type); |
620 | |
621 | /** |
622 | * BO doesn't have a TTM we need to bind/unbind. Just remember |
623 | * this eviction and free up the allocation |
624 | */ |
625 | spin_lock(lock: &from->move_lock); |
626 | if (!from->move || dma_fence_is_later(f1: fence, f2: from->move)) { |
627 | dma_fence_put(fence: from->move); |
628 | from->move = dma_fence_get(fence); |
629 | } |
630 | spin_unlock(lock: &from->move_lock); |
631 | |
632 | ttm_resource_free(bo, res: &bo->resource); |
633 | } |
634 | |
635 | /** |
636 | * ttm_bo_move_accel_cleanup - cleanup helper for hw copies |
637 | * |
638 | * @bo: A pointer to a struct ttm_buffer_object. |
639 | * @fence: A fence object that signals when moving is complete. |
640 | * @evict: This is an evict move. Don't return until the buffer is idle. |
641 | * @pipeline: evictions are to be pipelined. |
642 | * @new_mem: struct ttm_resource indicating where to move. |
643 | * |
644 | * Accelerated move function to be called when an accelerated move |
645 | * has been scheduled. The function will create a new temporary buffer object |
646 | * representing the old placement, and put the sync object on both buffer |
647 | * objects. After that the newly created buffer object is unref'd to be |
648 | * destroyed when the move is complete. This will help pipeline |
649 | * buffer moves. |
650 | */ |
651 | int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, |
652 | struct dma_fence *fence, |
653 | bool evict, |
654 | bool pipeline, |
655 | struct ttm_resource *new_mem) |
656 | { |
657 | struct ttm_device *bdev = bo->bdev; |
658 | struct ttm_resource_manager *from = ttm_manager_type(bdev, mem_type: bo->resource->mem_type); |
659 | struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type: new_mem->mem_type); |
660 | int ret = 0; |
661 | |
662 | dma_resv_add_fence(obj: bo->base.resv, fence, usage: DMA_RESV_USAGE_KERNEL); |
663 | if (!evict) |
664 | ret = ttm_bo_move_to_ghost(bo, fence, dst_use_tt: man->use_tt); |
665 | else if (!from->use_tt && pipeline) |
666 | ttm_bo_move_pipeline_evict(bo, fence); |
667 | else |
668 | ret = ttm_bo_wait_free_node(bo, dst_use_tt: man->use_tt); |
669 | |
670 | if (ret) |
671 | return ret; |
672 | |
673 | ttm_bo_assign_mem(bo, new_mem); |
674 | |
675 | return 0; |
676 | } |
677 | EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); |
678 | |
679 | /** |
680 | * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish |
681 | * |
682 | * @bo: A pointer to a struct ttm_buffer_object. |
683 | * @new_mem: struct ttm_resource indicating where to move. |
684 | * |
685 | * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed |
686 | * by the caller to be idle. Typically used after memcpy buffer moves. |
687 | */ |
688 | void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo, |
689 | struct ttm_resource *new_mem) |
690 | { |
691 | struct ttm_device *bdev = bo->bdev; |
692 | struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type: new_mem->mem_type); |
693 | int ret; |
694 | |
695 | ret = ttm_bo_wait_free_node(bo, dst_use_tt: man->use_tt); |
696 | if (WARN_ON(ret)) |
697 | return; |
698 | |
699 | ttm_bo_assign_mem(bo, new_mem); |
700 | } |
701 | EXPORT_SYMBOL(ttm_bo_move_sync_cleanup); |
702 | |
703 | /** |
704 | * ttm_bo_pipeline_gutting - purge the contents of a bo |
705 | * @bo: The buffer object |
706 | * |
707 | * Purge the contents of a bo, async if the bo is not idle. |
708 | * After a successful call, the bo is left unpopulated in |
709 | * system placement. The function may wait uninterruptible |
710 | * for idle on OOM. |
711 | * |
712 | * Return: 0 if successful, negative error code on failure. |
713 | */ |
714 | int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) |
715 | { |
716 | struct ttm_buffer_object *ghost; |
717 | struct ttm_tt *ttm; |
718 | int ret; |
719 | |
720 | /* If already idle, no need for ghost object dance. */ |
721 | if (dma_resv_test_signaled(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP)) { |
722 | if (!bo->ttm) { |
723 | /* See comment below about clearing. */ |
724 | ret = ttm_tt_create(bo, zero_alloc: true); |
725 | if (ret) |
726 | return ret; |
727 | } else { |
728 | ttm_tt_unpopulate(bdev: bo->bdev, ttm: bo->ttm); |
729 | if (bo->type == ttm_bo_type_device) |
730 | ttm_tt_mark_for_clear(ttm: bo->ttm); |
731 | } |
732 | ttm_resource_free(bo, res: &bo->resource); |
733 | return 0; |
734 | } |
735 | |
736 | /* |
737 | * We need an unpopulated ttm_tt after giving our current one, |
738 | * if any, to the ghost object. And we can't afford to fail |
739 | * creating one *after* the operation. If the bo subsequently gets |
740 | * resurrected, make sure it's cleared (if ttm_bo_type_device) |
741 | * to avoid leaking sensitive information to user-space. |
742 | */ |
743 | |
744 | ttm = bo->ttm; |
745 | bo->ttm = NULL; |
746 | ret = ttm_tt_create(bo, zero_alloc: true); |
747 | swap(bo->ttm, ttm); |
748 | if (ret) |
749 | return ret; |
750 | |
751 | ret = ttm_buffer_object_transfer(bo, new_obj: &ghost); |
752 | if (ret) |
753 | goto error_destroy_tt; |
754 | |
755 | ret = dma_resv_copy_fences(dst: &ghost->base._resv, src: bo->base.resv); |
756 | /* Last resort, wait for the BO to be idle when we are OOM */ |
757 | if (ret) { |
758 | dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP, |
759 | intr: false, MAX_SCHEDULE_TIMEOUT); |
760 | } |
761 | |
762 | dma_resv_unlock(obj: &ghost->base._resv); |
763 | ttm_bo_put(bo: ghost); |
764 | bo->ttm = ttm; |
765 | return 0; |
766 | |
767 | error_destroy_tt: |
768 | ttm_tt_destroy(bdev: bo->bdev, ttm); |
769 | return ret; |
770 | } |
771 | |