1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * Copyright (C) 2023 Loongson Technology Corporation Limited |
4 | */ |
5 | |
6 | #include <drm/drm_drv.h> |
7 | #include <drm/drm_file.h> |
8 | #include <drm/drm_gem.h> |
9 | #include <drm/drm_managed.h> |
10 | #include <drm/drm_prime.h> |
11 | |
12 | #include "lsdc_drv.h" |
13 | #include "lsdc_ttm.h" |
14 | |
15 | const char *lsdc_mem_type_to_str(uint32_t mem_type) |
16 | { |
17 | switch (mem_type) { |
18 | case TTM_PL_VRAM: |
19 | return "VRAM" ; |
20 | case TTM_PL_TT: |
21 | return "GTT" ; |
22 | case TTM_PL_SYSTEM: |
23 | return "SYSTEM" ; |
24 | default: |
25 | break; |
26 | } |
27 | |
28 | return "Unknown" ; |
29 | } |
30 | |
31 | const char *lsdc_domain_to_str(u32 domain) |
32 | { |
33 | switch (domain) { |
34 | case LSDC_GEM_DOMAIN_VRAM: |
35 | return "VRAM" ; |
36 | case LSDC_GEM_DOMAIN_GTT: |
37 | return "GTT" ; |
38 | case LSDC_GEM_DOMAIN_SYSTEM: |
39 | return "SYSTEM" ; |
40 | default: |
41 | break; |
42 | } |
43 | |
44 | return "Unknown" ; |
45 | } |
46 | |
47 | static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain) |
48 | { |
49 | u32 c = 0; |
50 | u32 pflags = 0; |
51 | u32 i; |
52 | |
53 | if (lbo->tbo.base.size <= PAGE_SIZE) |
54 | pflags |= TTM_PL_FLAG_TOPDOWN; |
55 | |
56 | lbo->placement.placement = lbo->placements; |
57 | |
58 | if (domain & LSDC_GEM_DOMAIN_VRAM) { |
59 | lbo->placements[c].mem_type = TTM_PL_VRAM; |
60 | lbo->placements[c++].flags = pflags; |
61 | } |
62 | |
63 | if (domain & LSDC_GEM_DOMAIN_GTT) { |
64 | lbo->placements[c].mem_type = TTM_PL_TT; |
65 | lbo->placements[c++].flags = pflags; |
66 | } |
67 | |
68 | if (domain & LSDC_GEM_DOMAIN_SYSTEM) { |
69 | lbo->placements[c].mem_type = TTM_PL_SYSTEM; |
70 | lbo->placements[c++].flags = 0; |
71 | } |
72 | |
73 | if (!c) { |
74 | lbo->placements[c].mem_type = TTM_PL_SYSTEM; |
75 | lbo->placements[c++].flags = 0; |
76 | } |
77 | |
78 | lbo->placement.num_placement = c; |
79 | |
80 | for (i = 0; i < c; ++i) { |
81 | lbo->placements[i].fpfn = 0; |
82 | lbo->placements[i].lpfn = 0; |
83 | } |
84 | } |
85 | |
86 | static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt) |
87 | { |
88 | ttm_tt_fini(ttm: tt); |
89 | kfree(objp: tt); |
90 | } |
91 | |
92 | static struct ttm_tt * |
93 | lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags) |
94 | { |
95 | struct ttm_tt *tt; |
96 | int ret; |
97 | |
98 | tt = kzalloc(size: sizeof(*tt), GFP_KERNEL); |
99 | if (!tt) |
100 | return NULL; |
101 | |
102 | ret = ttm_sg_tt_init(ttm_dma: tt, bo: tbo, page_flags, caching: ttm_cached); |
103 | if (ret < 0) { |
104 | kfree(objp: tt); |
105 | return NULL; |
106 | } |
107 | |
108 | return tt; |
109 | } |
110 | |
111 | static int lsdc_ttm_tt_populate(struct ttm_device *bdev, |
112 | struct ttm_tt *ttm, |
113 | struct ttm_operation_ctx *ctx) |
114 | { |
115 | bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); |
116 | |
117 | if (slave && ttm->sg) { |
118 | drm_prime_sg_to_dma_addr_array(sgt: ttm->sg, |
119 | addrs: ttm->dma_address, |
120 | max_pages: ttm->num_pages); |
121 | |
122 | return 0; |
123 | } |
124 | |
125 | return ttm_pool_alloc(pool: &bdev->pool, tt: ttm, ctx); |
126 | } |
127 | |
128 | static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev, |
129 | struct ttm_tt *ttm) |
130 | { |
131 | bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); |
132 | |
133 | if (slave) |
134 | return; |
135 | |
136 | return ttm_pool_free(pool: &bdev->pool, tt: ttm); |
137 | } |
138 | |
139 | static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo, |
140 | struct ttm_placement *tplacement) |
141 | { |
142 | struct ttm_resource *resource = tbo->resource; |
143 | struct lsdc_bo *lbo = to_lsdc_bo(tbo); |
144 | |
145 | switch (resource->mem_type) { |
146 | case TTM_PL_VRAM: |
147 | lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT); |
148 | break; |
149 | case TTM_PL_TT: |
150 | default: |
151 | lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM); |
152 | break; |
153 | } |
154 | |
155 | *tplacement = lbo->placement; |
156 | } |
157 | |
158 | static int lsdc_bo_move(struct ttm_buffer_object *tbo, |
159 | bool evict, |
160 | struct ttm_operation_ctx *ctx, |
161 | struct ttm_resource *new_mem, |
162 | struct ttm_place *hop) |
163 | { |
164 | struct drm_device *ddev = tbo->base.dev; |
165 | struct ttm_resource *old_mem = tbo->resource; |
166 | struct lsdc_bo *lbo = to_lsdc_bo(tbo); |
167 | int ret; |
168 | |
169 | if (unlikely(tbo->pin_count > 0)) { |
170 | drm_warn(ddev, "Can't move a pinned BO\n" ); |
171 | return -EINVAL; |
172 | } |
173 | |
174 | ret = ttm_bo_wait_ctx(bo: tbo, ctx); |
175 | if (ret) |
176 | return ret; |
177 | |
178 | if (!old_mem) { |
179 | drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n" , |
180 | lbo, lsdc_mem_type_to_str(new_mem->mem_type), |
181 | lsdc_bo_size(lbo)); |
182 | ttm_bo_move_null(bo: tbo, new_mem); |
183 | return 0; |
184 | } |
185 | |
186 | if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) { |
187 | ttm_bo_move_null(bo: tbo, new_mem); |
188 | drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n" , |
189 | lbo, lsdc_bo_size(lbo)); |
190 | return 0; |
191 | } |
192 | |
193 | if (old_mem->mem_type == TTM_PL_SYSTEM && |
194 | new_mem->mem_type == TTM_PL_TT) { |
195 | drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n" , |
196 | lbo, lsdc_bo_size(lbo)); |
197 | ttm_bo_move_null(bo: tbo, new_mem); |
198 | return 0; |
199 | } |
200 | |
201 | if (old_mem->mem_type == TTM_PL_TT && |
202 | new_mem->mem_type == TTM_PL_SYSTEM) { |
203 | drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n" , |
204 | lbo, lsdc_bo_size(lbo)); |
205 | ttm_resource_free(bo: tbo, res: &tbo->resource); |
206 | ttm_bo_assign_mem(bo: tbo, new_mem); |
207 | return 0; |
208 | } |
209 | |
210 | drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n" , |
211 | lbo, |
212 | lsdc_mem_type_to_str(old_mem->mem_type), |
213 | lsdc_mem_type_to_str(new_mem->mem_type), |
214 | lsdc_bo_size(lbo)); |
215 | |
216 | return ttm_bo_move_memcpy(bo: tbo, ctx, new_mem); |
217 | } |
218 | |
219 | static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev, |
220 | struct ttm_resource *mem) |
221 | { |
222 | struct lsdc_device *ldev = tdev_to_ldev(bdev); |
223 | |
224 | switch (mem->mem_type) { |
225 | case TTM_PL_SYSTEM: |
226 | break; |
227 | case TTM_PL_TT: |
228 | break; |
229 | case TTM_PL_VRAM: |
230 | mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base; |
231 | mem->bus.is_iomem = true; |
232 | mem->bus.caching = ttm_write_combined; |
233 | break; |
234 | default: |
235 | return -EINVAL; |
236 | } |
237 | |
238 | return 0; |
239 | } |
240 | |
241 | static struct ttm_device_funcs lsdc_bo_driver = { |
242 | .ttm_tt_create = lsdc_ttm_tt_create, |
243 | .ttm_tt_populate = lsdc_ttm_tt_populate, |
244 | .ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate, |
245 | .ttm_tt_destroy = lsdc_ttm_tt_destroy, |
246 | .eviction_valuable = ttm_bo_eviction_valuable, |
247 | .evict_flags = lsdc_bo_evict_flags, |
248 | .move = lsdc_bo_move, |
249 | .io_mem_reserve = lsdc_bo_reserve_io_mem, |
250 | }; |
251 | |
252 | u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo) |
253 | { |
254 | struct ttm_buffer_object *tbo = &lbo->tbo; |
255 | struct drm_device *ddev = tbo->base.dev; |
256 | struct ttm_resource *resource = tbo->resource; |
257 | |
258 | if (unlikely(!tbo->pin_count)) { |
259 | drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n" ); |
260 | return 0; |
261 | } |
262 | |
263 | if (unlikely(resource->mem_type == TTM_PL_SYSTEM)) |
264 | return 0; |
265 | |
266 | return resource->start << PAGE_SHIFT; |
267 | } |
268 | |
269 | size_t lsdc_bo_size(struct lsdc_bo *lbo) |
270 | { |
271 | struct ttm_buffer_object *tbo = &lbo->tbo; |
272 | |
273 | return tbo->base.size; |
274 | } |
275 | |
276 | int lsdc_bo_reserve(struct lsdc_bo *lbo) |
277 | { |
278 | return ttm_bo_reserve(bo: &lbo->tbo, interruptible: true, no_wait: false, NULL); |
279 | } |
280 | |
281 | void lsdc_bo_unreserve(struct lsdc_bo *lbo) |
282 | { |
283 | return ttm_bo_unreserve(bo: &lbo->tbo); |
284 | } |
285 | |
286 | int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr) |
287 | { |
288 | struct ttm_operation_ctx ctx = { false, false }; |
289 | struct ttm_buffer_object *tbo = &lbo->tbo; |
290 | struct lsdc_device *ldev = tdev_to_ldev(bdev: tbo->bdev); |
291 | int ret; |
292 | |
293 | if (tbo->pin_count) |
294 | goto bo_pinned; |
295 | |
296 | if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM) |
297 | return -EINVAL; |
298 | |
299 | if (domain) |
300 | lsdc_bo_set_placement(lbo, domain); |
301 | |
302 | ret = ttm_bo_validate(bo: tbo, placement: &lbo->placement, ctx: &ctx); |
303 | if (unlikely(ret)) { |
304 | drm_err(&ldev->base, "%p validate failed: %d\n" , lbo, ret); |
305 | return ret; |
306 | } |
307 | |
308 | if (domain == LSDC_GEM_DOMAIN_VRAM) |
309 | ldev->vram_pinned_size += lsdc_bo_size(lbo); |
310 | else if (domain == LSDC_GEM_DOMAIN_GTT) |
311 | ldev->gtt_pinned_size += lsdc_bo_size(lbo); |
312 | |
313 | bo_pinned: |
314 | ttm_bo_pin(bo: tbo); |
315 | |
316 | if (gpu_addr) |
317 | *gpu_addr = lsdc_bo_gpu_offset(lbo); |
318 | |
319 | return 0; |
320 | } |
321 | |
322 | void lsdc_bo_unpin(struct lsdc_bo *lbo) |
323 | { |
324 | struct ttm_buffer_object *tbo = &lbo->tbo; |
325 | struct lsdc_device *ldev = tdev_to_ldev(bdev: tbo->bdev); |
326 | |
327 | if (unlikely(!tbo->pin_count)) { |
328 | drm_dbg(&ldev->base, "%p unpin is not necessary\n" , lbo); |
329 | return; |
330 | } |
331 | |
332 | ttm_bo_unpin(bo: tbo); |
333 | |
334 | if (!tbo->pin_count) { |
335 | if (tbo->resource->mem_type == TTM_PL_VRAM) |
336 | ldev->vram_pinned_size -= lsdc_bo_size(lbo); |
337 | else if (tbo->resource->mem_type == TTM_PL_TT) |
338 | ldev->gtt_pinned_size -= lsdc_bo_size(lbo); |
339 | } |
340 | } |
341 | |
342 | void lsdc_bo_ref(struct lsdc_bo *lbo) |
343 | { |
344 | struct ttm_buffer_object *tbo = &lbo->tbo; |
345 | |
346 | ttm_bo_get(bo: tbo); |
347 | } |
348 | |
349 | void lsdc_bo_unref(struct lsdc_bo *lbo) |
350 | { |
351 | struct ttm_buffer_object *tbo = &lbo->tbo; |
352 | |
353 | ttm_bo_put(bo: tbo); |
354 | } |
355 | |
356 | int lsdc_bo_kmap(struct lsdc_bo *lbo) |
357 | { |
358 | struct ttm_buffer_object *tbo = &lbo->tbo; |
359 | struct drm_gem_object *gem = &tbo->base; |
360 | struct drm_device *ddev = gem->dev; |
361 | long ret; |
362 | int err; |
363 | |
364 | ret = dma_resv_wait_timeout(obj: gem->resv, usage: DMA_RESV_USAGE_KERNEL, intr: false, |
365 | MAX_SCHEDULE_TIMEOUT); |
366 | if (ret < 0) { |
367 | drm_warn(ddev, "wait fence timeout\n" ); |
368 | return ret; |
369 | } |
370 | |
371 | if (lbo->kptr) |
372 | return 0; |
373 | |
374 | err = ttm_bo_kmap(bo: tbo, start_page: 0, PFN_UP(lsdc_bo_size(lbo)), map: &lbo->kmap); |
375 | if (err) { |
376 | drm_err(ddev, "kmap %p failed: %d\n" , lbo, err); |
377 | return err; |
378 | } |
379 | |
380 | lbo->kptr = ttm_kmap_obj_virtual(map: &lbo->kmap, is_iomem: &lbo->is_iomem); |
381 | |
382 | return 0; |
383 | } |
384 | |
385 | void lsdc_bo_kunmap(struct lsdc_bo *lbo) |
386 | { |
387 | if (!lbo->kptr) |
388 | return; |
389 | |
390 | lbo->kptr = NULL; |
391 | ttm_bo_kunmap(map: &lbo->kmap); |
392 | } |
393 | |
394 | void lsdc_bo_clear(struct lsdc_bo *lbo) |
395 | { |
396 | lsdc_bo_kmap(lbo); |
397 | |
398 | if (lbo->is_iomem) |
399 | memset_io((void __iomem *)lbo->kptr, 0, lbo->size); |
400 | else |
401 | memset(lbo->kptr, 0, lbo->size); |
402 | |
403 | lsdc_bo_kunmap(lbo); |
404 | } |
405 | |
406 | int lsdc_bo_evict_vram(struct drm_device *ddev) |
407 | { |
408 | struct lsdc_device *ldev = to_lsdc(ddev); |
409 | struct ttm_device *bdev = &ldev->bdev; |
410 | struct ttm_resource_manager *man; |
411 | |
412 | man = ttm_manager_type(bdev, TTM_PL_VRAM); |
413 | if (unlikely(!man)) |
414 | return 0; |
415 | |
416 | return ttm_resource_manager_evict_all(bdev, man); |
417 | } |
418 | |
419 | static void lsdc_bo_destroy(struct ttm_buffer_object *tbo) |
420 | { |
421 | struct lsdc_device *ldev = tdev_to_ldev(bdev: tbo->bdev); |
422 | struct lsdc_bo *lbo = to_lsdc_bo(tbo); |
423 | |
424 | mutex_lock(&ldev->gem.mutex); |
425 | list_del_init(entry: &lbo->list); |
426 | mutex_unlock(lock: &ldev->gem.mutex); |
427 | |
428 | drm_gem_object_release(obj: &tbo->base); |
429 | |
430 | kfree(objp: lbo); |
431 | } |
432 | |
433 | struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev, |
434 | u32 domain, |
435 | size_t size, |
436 | bool kernel, |
437 | struct sg_table *sg, |
438 | struct dma_resv *resv) |
439 | { |
440 | struct lsdc_device *ldev = to_lsdc(ddev); |
441 | struct ttm_device *bdev = &ldev->bdev; |
442 | struct ttm_buffer_object *tbo; |
443 | struct lsdc_bo *lbo; |
444 | enum ttm_bo_type bo_type; |
445 | int ret; |
446 | |
447 | lbo = kzalloc(size: sizeof(*lbo), GFP_KERNEL); |
448 | if (!lbo) |
449 | return ERR_PTR(error: -ENOMEM); |
450 | |
451 | INIT_LIST_HEAD(list: &lbo->list); |
452 | |
453 | lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM | |
454 | LSDC_GEM_DOMAIN_GTT | |
455 | LSDC_GEM_DOMAIN_SYSTEM); |
456 | |
457 | tbo = &lbo->tbo; |
458 | |
459 | size = ALIGN(size, PAGE_SIZE); |
460 | |
461 | ret = drm_gem_object_init(dev: ddev, obj: &tbo->base, size); |
462 | if (ret) { |
463 | kfree(objp: lbo); |
464 | return ERR_PTR(error: ret); |
465 | } |
466 | |
467 | tbo->bdev = bdev; |
468 | |
469 | if (kernel) |
470 | bo_type = ttm_bo_type_kernel; |
471 | else if (sg) |
472 | bo_type = ttm_bo_type_sg; |
473 | else |
474 | bo_type = ttm_bo_type_device; |
475 | |
476 | lsdc_bo_set_placement(lbo, domain); |
477 | lbo->size = size; |
478 | |
479 | ret = ttm_bo_init_validate(bdev, bo: tbo, type: bo_type, placement: &lbo->placement, alignment: 0, |
480 | interruptible: false, sg, resv, destroy: lsdc_bo_destroy); |
481 | if (ret) { |
482 | kfree(objp: lbo); |
483 | return ERR_PTR(error: ret); |
484 | } |
485 | |
486 | return lbo; |
487 | } |
488 | |
489 | struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev, |
490 | u32 domain, |
491 | size_t size) |
492 | { |
493 | struct lsdc_bo *lbo; |
494 | int ret; |
495 | |
496 | lbo = lsdc_bo_create(ddev, domain, size, kernel: true, NULL, NULL); |
497 | if (IS_ERR(ptr: lbo)) |
498 | return ERR_CAST(ptr: lbo); |
499 | |
500 | ret = lsdc_bo_reserve(lbo); |
501 | if (unlikely(ret)) { |
502 | lsdc_bo_unref(lbo); |
503 | return ERR_PTR(error: ret); |
504 | } |
505 | |
506 | ret = lsdc_bo_pin(lbo, domain, NULL); |
507 | lsdc_bo_unreserve(lbo); |
508 | if (unlikely(ret)) { |
509 | lsdc_bo_unref(lbo); |
510 | return ERR_PTR(error: ret); |
511 | } |
512 | |
513 | return lbo; |
514 | } |
515 | |
516 | void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo) |
517 | { |
518 | int ret; |
519 | |
520 | ret = lsdc_bo_reserve(lbo); |
521 | if (unlikely(ret)) |
522 | return; |
523 | |
524 | lsdc_bo_unpin(lbo); |
525 | lsdc_bo_unreserve(lbo); |
526 | |
527 | lsdc_bo_unref(lbo); |
528 | } |
529 | |
530 | static void lsdc_ttm_fini(struct drm_device *ddev, void *data) |
531 | { |
532 | struct lsdc_device *ldev = (struct lsdc_device *)data; |
533 | |
534 | ttm_range_man_fini(bdev: &ldev->bdev, TTM_PL_VRAM); |
535 | ttm_range_man_fini(bdev: &ldev->bdev, TTM_PL_TT); |
536 | |
537 | ttm_device_fini(bdev: &ldev->bdev); |
538 | |
539 | drm_dbg(ddev, "ttm finished\n" ); |
540 | } |
541 | |
542 | int lsdc_ttm_init(struct lsdc_device *ldev) |
543 | { |
544 | struct drm_device *ddev = &ldev->base; |
545 | unsigned long num_vram_pages; |
546 | unsigned long num_gtt_pages; |
547 | int ret; |
548 | |
549 | ret = ttm_device_init(bdev: &ldev->bdev, funcs: &lsdc_bo_driver, dev: ddev->dev, |
550 | mapping: ddev->anon_inode->i_mapping, |
551 | vma_manager: ddev->vma_offset_manager, use_dma_alloc: false, use_dma32: true); |
552 | if (ret) |
553 | return ret; |
554 | |
555 | num_vram_pages = ldev->vram_size >> PAGE_SHIFT; |
556 | |
557 | ret = ttm_range_man_init(bdev: &ldev->bdev, TTM_PL_VRAM, use_tt: false, p_size: num_vram_pages); |
558 | if (unlikely(ret)) |
559 | return ret; |
560 | |
561 | drm_info(ddev, "VRAM: %lu pages ready\n" , num_vram_pages); |
562 | |
563 | /* 512M is far enough for us now */ |
564 | ldev->gtt_size = 512 << 20; |
565 | |
566 | num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT; |
567 | |
568 | ret = ttm_range_man_init(bdev: &ldev->bdev, TTM_PL_TT, use_tt: true, p_size: num_gtt_pages); |
569 | if (unlikely(ret)) |
570 | return ret; |
571 | |
572 | drm_info(ddev, "GTT: %lu pages ready\n" , num_gtt_pages); |
573 | |
574 | return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev); |
575 | } |
576 | |
577 | void lsdc_ttm_debugfs_init(struct lsdc_device *ldev) |
578 | { |
579 | struct ttm_device *bdev = &ldev->bdev; |
580 | struct drm_device *ddev = &ldev->base; |
581 | struct drm_minor *minor = ddev->primary; |
582 | struct dentry *root = minor->debugfs_root; |
583 | struct ttm_resource_manager *vram_man; |
584 | struct ttm_resource_manager *gtt_man; |
585 | |
586 | vram_man = ttm_manager_type(bdev, TTM_PL_VRAM); |
587 | gtt_man = ttm_manager_type(bdev, TTM_PL_TT); |
588 | |
589 | ttm_resource_manager_create_debugfs(man: vram_man, parent: root, name: "vram_mm" ); |
590 | ttm_resource_manager_create_debugfs(man: gtt_man, parent: root, name: "gtt_mm" ); |
591 | } |
592 | |