1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Copyright (C) 2023 Loongson Technology Corporation Limited
4 */
5
6#include <drm/drm_drv.h>
7#include <drm/drm_file.h>
8#include <drm/drm_gem.h>
9#include <drm/drm_managed.h>
10#include <drm/drm_prime.h>
11
12#include "lsdc_drv.h"
13#include "lsdc_ttm.h"
14
15const char *lsdc_mem_type_to_str(uint32_t mem_type)
16{
17 switch (mem_type) {
18 case TTM_PL_VRAM:
19 return "VRAM";
20 case TTM_PL_TT:
21 return "GTT";
22 case TTM_PL_SYSTEM:
23 return "SYSTEM";
24 default:
25 break;
26 }
27
28 return "Unknown";
29}
30
31const char *lsdc_domain_to_str(u32 domain)
32{
33 switch (domain) {
34 case LSDC_GEM_DOMAIN_VRAM:
35 return "VRAM";
36 case LSDC_GEM_DOMAIN_GTT:
37 return "GTT";
38 case LSDC_GEM_DOMAIN_SYSTEM:
39 return "SYSTEM";
40 default:
41 break;
42 }
43
44 return "Unknown";
45}
46
47static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
48{
49 u32 c = 0;
50 u32 pflags = 0;
51 u32 i;
52
53 if (lbo->tbo.base.size <= PAGE_SIZE)
54 pflags |= TTM_PL_FLAG_TOPDOWN;
55
56 lbo->placement.placement = lbo->placements;
57
58 if (domain & LSDC_GEM_DOMAIN_VRAM) {
59 lbo->placements[c].mem_type = TTM_PL_VRAM;
60 lbo->placements[c++].flags = pflags;
61 }
62
63 if (domain & LSDC_GEM_DOMAIN_GTT) {
64 lbo->placements[c].mem_type = TTM_PL_TT;
65 lbo->placements[c++].flags = pflags;
66 }
67
68 if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
69 lbo->placements[c].mem_type = TTM_PL_SYSTEM;
70 lbo->placements[c++].flags = 0;
71 }
72
73 if (!c) {
74 lbo->placements[c].mem_type = TTM_PL_SYSTEM;
75 lbo->placements[c++].flags = 0;
76 }
77
78 lbo->placement.num_placement = c;
79
80 for (i = 0; i < c; ++i) {
81 lbo->placements[i].fpfn = 0;
82 lbo->placements[i].lpfn = 0;
83 }
84}
85
86static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
87{
88 ttm_tt_fini(ttm: tt);
89 kfree(objp: tt);
90}
91
92static struct ttm_tt *
93lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
94{
95 struct ttm_tt *tt;
96 int ret;
97
98 tt = kzalloc(size: sizeof(*tt), GFP_KERNEL);
99 if (!tt)
100 return NULL;
101
102 ret = ttm_sg_tt_init(ttm_dma: tt, bo: tbo, page_flags, caching: ttm_cached);
103 if (ret < 0) {
104 kfree(objp: tt);
105 return NULL;
106 }
107
108 return tt;
109}
110
111static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
112 struct ttm_tt *ttm,
113 struct ttm_operation_ctx *ctx)
114{
115 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
116
117 if (slave && ttm->sg) {
118 drm_prime_sg_to_dma_addr_array(sgt: ttm->sg,
119 addrs: ttm->dma_address,
120 max_pages: ttm->num_pages);
121
122 return 0;
123 }
124
125 return ttm_pool_alloc(pool: &bdev->pool, tt: ttm, ctx);
126}
127
128static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
129 struct ttm_tt *ttm)
130{
131 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
132
133 if (slave)
134 return;
135
136 return ttm_pool_free(pool: &bdev->pool, tt: ttm);
137}
138
139static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
140 struct ttm_placement *tplacement)
141{
142 struct ttm_resource *resource = tbo->resource;
143 struct lsdc_bo *lbo = to_lsdc_bo(tbo);
144
145 switch (resource->mem_type) {
146 case TTM_PL_VRAM:
147 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
148 break;
149 case TTM_PL_TT:
150 default:
151 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
152 break;
153 }
154
155 *tplacement = lbo->placement;
156}
157
158static int lsdc_bo_move(struct ttm_buffer_object *tbo,
159 bool evict,
160 struct ttm_operation_ctx *ctx,
161 struct ttm_resource *new_mem,
162 struct ttm_place *hop)
163{
164 struct drm_device *ddev = tbo->base.dev;
165 struct ttm_resource *old_mem = tbo->resource;
166 struct lsdc_bo *lbo = to_lsdc_bo(tbo);
167 int ret;
168
169 if (unlikely(tbo->pin_count > 0)) {
170 drm_warn(ddev, "Can't move a pinned BO\n");
171 return -EINVAL;
172 }
173
174 ret = ttm_bo_wait_ctx(bo: tbo, ctx);
175 if (ret)
176 return ret;
177
178 if (!old_mem) {
179 drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
180 lbo, lsdc_mem_type_to_str(new_mem->mem_type),
181 lsdc_bo_size(lbo));
182 ttm_bo_move_null(bo: tbo, new_mem);
183 return 0;
184 }
185
186 if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
187 ttm_bo_move_null(bo: tbo, new_mem);
188 drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
189 lbo, lsdc_bo_size(lbo));
190 return 0;
191 }
192
193 if (old_mem->mem_type == TTM_PL_SYSTEM &&
194 new_mem->mem_type == TTM_PL_TT) {
195 drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
196 lbo, lsdc_bo_size(lbo));
197 ttm_bo_move_null(bo: tbo, new_mem);
198 return 0;
199 }
200
201 if (old_mem->mem_type == TTM_PL_TT &&
202 new_mem->mem_type == TTM_PL_SYSTEM) {
203 drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
204 lbo, lsdc_bo_size(lbo));
205 ttm_resource_free(bo: tbo, res: &tbo->resource);
206 ttm_bo_assign_mem(bo: tbo, new_mem);
207 return 0;
208 }
209
210 drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
211 lbo,
212 lsdc_mem_type_to_str(old_mem->mem_type),
213 lsdc_mem_type_to_str(new_mem->mem_type),
214 lsdc_bo_size(lbo));
215
216 return ttm_bo_move_memcpy(bo: tbo, ctx, new_mem);
217}
218
219static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
220 struct ttm_resource *mem)
221{
222 struct lsdc_device *ldev = tdev_to_ldev(bdev);
223
224 switch (mem->mem_type) {
225 case TTM_PL_SYSTEM:
226 break;
227 case TTM_PL_TT:
228 break;
229 case TTM_PL_VRAM:
230 mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
231 mem->bus.is_iomem = true;
232 mem->bus.caching = ttm_write_combined;
233 break;
234 default:
235 return -EINVAL;
236 }
237
238 return 0;
239}
240
241static struct ttm_device_funcs lsdc_bo_driver = {
242 .ttm_tt_create = lsdc_ttm_tt_create,
243 .ttm_tt_populate = lsdc_ttm_tt_populate,
244 .ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
245 .ttm_tt_destroy = lsdc_ttm_tt_destroy,
246 .eviction_valuable = ttm_bo_eviction_valuable,
247 .evict_flags = lsdc_bo_evict_flags,
248 .move = lsdc_bo_move,
249 .io_mem_reserve = lsdc_bo_reserve_io_mem,
250};
251
252u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
253{
254 struct ttm_buffer_object *tbo = &lbo->tbo;
255 struct drm_device *ddev = tbo->base.dev;
256 struct ttm_resource *resource = tbo->resource;
257
258 if (unlikely(!tbo->pin_count)) {
259 drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
260 return 0;
261 }
262
263 if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
264 return 0;
265
266 return resource->start << PAGE_SHIFT;
267}
268
269size_t lsdc_bo_size(struct lsdc_bo *lbo)
270{
271 struct ttm_buffer_object *tbo = &lbo->tbo;
272
273 return tbo->base.size;
274}
275
276int lsdc_bo_reserve(struct lsdc_bo *lbo)
277{
278 return ttm_bo_reserve(bo: &lbo->tbo, interruptible: true, no_wait: false, NULL);
279}
280
281void lsdc_bo_unreserve(struct lsdc_bo *lbo)
282{
283 return ttm_bo_unreserve(bo: &lbo->tbo);
284}
285
286int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
287{
288 struct ttm_operation_ctx ctx = { false, false };
289 struct ttm_buffer_object *tbo = &lbo->tbo;
290 struct lsdc_device *ldev = tdev_to_ldev(bdev: tbo->bdev);
291 int ret;
292
293 if (tbo->pin_count)
294 goto bo_pinned;
295
296 if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
297 return -EINVAL;
298
299 if (domain)
300 lsdc_bo_set_placement(lbo, domain);
301
302 ret = ttm_bo_validate(bo: tbo, placement: &lbo->placement, ctx: &ctx);
303 if (unlikely(ret)) {
304 drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
305 return ret;
306 }
307
308 if (domain == LSDC_GEM_DOMAIN_VRAM)
309 ldev->vram_pinned_size += lsdc_bo_size(lbo);
310 else if (domain == LSDC_GEM_DOMAIN_GTT)
311 ldev->gtt_pinned_size += lsdc_bo_size(lbo);
312
313bo_pinned:
314 ttm_bo_pin(bo: tbo);
315
316 if (gpu_addr)
317 *gpu_addr = lsdc_bo_gpu_offset(lbo);
318
319 return 0;
320}
321
322void lsdc_bo_unpin(struct lsdc_bo *lbo)
323{
324 struct ttm_buffer_object *tbo = &lbo->tbo;
325 struct lsdc_device *ldev = tdev_to_ldev(bdev: tbo->bdev);
326
327 if (unlikely(!tbo->pin_count)) {
328 drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
329 return;
330 }
331
332 ttm_bo_unpin(bo: tbo);
333
334 if (!tbo->pin_count) {
335 if (tbo->resource->mem_type == TTM_PL_VRAM)
336 ldev->vram_pinned_size -= lsdc_bo_size(lbo);
337 else if (tbo->resource->mem_type == TTM_PL_TT)
338 ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
339 }
340}
341
342void lsdc_bo_ref(struct lsdc_bo *lbo)
343{
344 struct ttm_buffer_object *tbo = &lbo->tbo;
345
346 ttm_bo_get(bo: tbo);
347}
348
349void lsdc_bo_unref(struct lsdc_bo *lbo)
350{
351 struct ttm_buffer_object *tbo = &lbo->tbo;
352
353 ttm_bo_put(bo: tbo);
354}
355
356int lsdc_bo_kmap(struct lsdc_bo *lbo)
357{
358 struct ttm_buffer_object *tbo = &lbo->tbo;
359 struct drm_gem_object *gem = &tbo->base;
360 struct drm_device *ddev = gem->dev;
361 long ret;
362 int err;
363
364 ret = dma_resv_wait_timeout(obj: gem->resv, usage: DMA_RESV_USAGE_KERNEL, intr: false,
365 MAX_SCHEDULE_TIMEOUT);
366 if (ret < 0) {
367 drm_warn(ddev, "wait fence timeout\n");
368 return ret;
369 }
370
371 if (lbo->kptr)
372 return 0;
373
374 err = ttm_bo_kmap(bo: tbo, start_page: 0, PFN_UP(lsdc_bo_size(lbo)), map: &lbo->kmap);
375 if (err) {
376 drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
377 return err;
378 }
379
380 lbo->kptr = ttm_kmap_obj_virtual(map: &lbo->kmap, is_iomem: &lbo->is_iomem);
381
382 return 0;
383}
384
385void lsdc_bo_kunmap(struct lsdc_bo *lbo)
386{
387 if (!lbo->kptr)
388 return;
389
390 lbo->kptr = NULL;
391 ttm_bo_kunmap(map: &lbo->kmap);
392}
393
394void lsdc_bo_clear(struct lsdc_bo *lbo)
395{
396 lsdc_bo_kmap(lbo);
397
398 if (lbo->is_iomem)
399 memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
400 else
401 memset(lbo->kptr, 0, lbo->size);
402
403 lsdc_bo_kunmap(lbo);
404}
405
406int lsdc_bo_evict_vram(struct drm_device *ddev)
407{
408 struct lsdc_device *ldev = to_lsdc(ddev);
409 struct ttm_device *bdev = &ldev->bdev;
410 struct ttm_resource_manager *man;
411
412 man = ttm_manager_type(bdev, TTM_PL_VRAM);
413 if (unlikely(!man))
414 return 0;
415
416 return ttm_resource_manager_evict_all(bdev, man);
417}
418
419static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
420{
421 struct lsdc_device *ldev = tdev_to_ldev(bdev: tbo->bdev);
422 struct lsdc_bo *lbo = to_lsdc_bo(tbo);
423
424 mutex_lock(&ldev->gem.mutex);
425 list_del_init(entry: &lbo->list);
426 mutex_unlock(lock: &ldev->gem.mutex);
427
428 drm_gem_object_release(obj: &tbo->base);
429
430 kfree(objp: lbo);
431}
432
433struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
434 u32 domain,
435 size_t size,
436 bool kernel,
437 struct sg_table *sg,
438 struct dma_resv *resv)
439{
440 struct lsdc_device *ldev = to_lsdc(ddev);
441 struct ttm_device *bdev = &ldev->bdev;
442 struct ttm_buffer_object *tbo;
443 struct lsdc_bo *lbo;
444 enum ttm_bo_type bo_type;
445 int ret;
446
447 lbo = kzalloc(size: sizeof(*lbo), GFP_KERNEL);
448 if (!lbo)
449 return ERR_PTR(error: -ENOMEM);
450
451 INIT_LIST_HEAD(list: &lbo->list);
452
453 lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
454 LSDC_GEM_DOMAIN_GTT |
455 LSDC_GEM_DOMAIN_SYSTEM);
456
457 tbo = &lbo->tbo;
458
459 size = ALIGN(size, PAGE_SIZE);
460
461 ret = drm_gem_object_init(dev: ddev, obj: &tbo->base, size);
462 if (ret) {
463 kfree(objp: lbo);
464 return ERR_PTR(error: ret);
465 }
466
467 tbo->bdev = bdev;
468
469 if (kernel)
470 bo_type = ttm_bo_type_kernel;
471 else if (sg)
472 bo_type = ttm_bo_type_sg;
473 else
474 bo_type = ttm_bo_type_device;
475
476 lsdc_bo_set_placement(lbo, domain);
477 lbo->size = size;
478
479 ret = ttm_bo_init_validate(bdev, bo: tbo, type: bo_type, placement: &lbo->placement, alignment: 0,
480 interruptible: false, sg, resv, destroy: lsdc_bo_destroy);
481 if (ret) {
482 kfree(objp: lbo);
483 return ERR_PTR(error: ret);
484 }
485
486 return lbo;
487}
488
489struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
490 u32 domain,
491 size_t size)
492{
493 struct lsdc_bo *lbo;
494 int ret;
495
496 lbo = lsdc_bo_create(ddev, domain, size, kernel: true, NULL, NULL);
497 if (IS_ERR(ptr: lbo))
498 return ERR_CAST(ptr: lbo);
499
500 ret = lsdc_bo_reserve(lbo);
501 if (unlikely(ret)) {
502 lsdc_bo_unref(lbo);
503 return ERR_PTR(error: ret);
504 }
505
506 ret = lsdc_bo_pin(lbo, domain, NULL);
507 lsdc_bo_unreserve(lbo);
508 if (unlikely(ret)) {
509 lsdc_bo_unref(lbo);
510 return ERR_PTR(error: ret);
511 }
512
513 return lbo;
514}
515
516void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
517{
518 int ret;
519
520 ret = lsdc_bo_reserve(lbo);
521 if (unlikely(ret))
522 return;
523
524 lsdc_bo_unpin(lbo);
525 lsdc_bo_unreserve(lbo);
526
527 lsdc_bo_unref(lbo);
528}
529
530static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
531{
532 struct lsdc_device *ldev = (struct lsdc_device *)data;
533
534 ttm_range_man_fini(bdev: &ldev->bdev, TTM_PL_VRAM);
535 ttm_range_man_fini(bdev: &ldev->bdev, TTM_PL_TT);
536
537 ttm_device_fini(bdev: &ldev->bdev);
538
539 drm_dbg(ddev, "ttm finished\n");
540}
541
542int lsdc_ttm_init(struct lsdc_device *ldev)
543{
544 struct drm_device *ddev = &ldev->base;
545 unsigned long num_vram_pages;
546 unsigned long num_gtt_pages;
547 int ret;
548
549 ret = ttm_device_init(bdev: &ldev->bdev, funcs: &lsdc_bo_driver, dev: ddev->dev,
550 mapping: ddev->anon_inode->i_mapping,
551 vma_manager: ddev->vma_offset_manager, use_dma_alloc: false, use_dma32: true);
552 if (ret)
553 return ret;
554
555 num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
556
557 ret = ttm_range_man_init(bdev: &ldev->bdev, TTM_PL_VRAM, use_tt: false, p_size: num_vram_pages);
558 if (unlikely(ret))
559 return ret;
560
561 drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
562
563 /* 512M is far enough for us now */
564 ldev->gtt_size = 512 << 20;
565
566 num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
567
568 ret = ttm_range_man_init(bdev: &ldev->bdev, TTM_PL_TT, use_tt: true, p_size: num_gtt_pages);
569 if (unlikely(ret))
570 return ret;
571
572 drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
573
574 return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
575}
576
577void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
578{
579 struct ttm_device *bdev = &ldev->bdev;
580 struct drm_device *ddev = &ldev->base;
581 struct drm_minor *minor = ddev->primary;
582 struct dentry *root = minor->debugfs_root;
583 struct ttm_resource_manager *vram_man;
584 struct ttm_resource_manager *gtt_man;
585
586 vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
587 gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
588
589 ttm_resource_manager_create_debugfs(man: vram_man, parent: root, name: "vram_mm");
590 ttm_resource_manager_create_debugfs(man: gtt_man, parent: root, name: "gtt_mm");
591}
592

source code of linux/drivers/gpu/drm/loongson/lsdc_ttm.c