1 | /* |
2 | * Copyright 2020 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | * Authors: Christian König |
23 | */ |
24 | |
25 | #include <linux/iosys-map.h> |
26 | #include <linux/io-mapping.h> |
27 | #include <linux/scatterlist.h> |
28 | |
29 | #include <drm/ttm/ttm_bo.h> |
30 | #include <drm/ttm/ttm_placement.h> |
31 | #include <drm/ttm/ttm_resource.h> |
32 | |
33 | #include <drm/drm_util.h> |
34 | |
35 | /** |
36 | * ttm_lru_bulk_move_init - initialize a bulk move structure |
37 | * @bulk: the structure to init |
38 | * |
39 | * For now just memset the structure to zero. |
40 | */ |
41 | void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk) |
42 | { |
43 | memset(bulk, 0, sizeof(*bulk)); |
44 | } |
45 | EXPORT_SYMBOL(ttm_lru_bulk_move_init); |
46 | |
47 | /** |
48 | * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail. |
49 | * |
50 | * @bulk: bulk move structure |
51 | * |
52 | * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that |
53 | * resource order never changes. Should be called with &ttm_device.lru_lock held. |
54 | */ |
55 | void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk) |
56 | { |
57 | unsigned i, j; |
58 | |
59 | for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) { |
60 | for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) { |
61 | struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j]; |
62 | struct ttm_resource_manager *man; |
63 | |
64 | if (!pos->first) |
65 | continue; |
66 | |
67 | lockdep_assert_held(&pos->first->bo->bdev->lru_lock); |
68 | dma_resv_assert_held(pos->first->bo->base.resv); |
69 | dma_resv_assert_held(pos->last->bo->base.resv); |
70 | |
71 | man = ttm_manager_type(bdev: pos->first->bo->bdev, mem_type: i); |
72 | list_bulk_move_tail(head: &man->lru[j], first: &pos->first->lru, |
73 | last: &pos->last->lru); |
74 | } |
75 | } |
76 | } |
77 | EXPORT_SYMBOL(ttm_lru_bulk_move_tail); |
78 | |
79 | /* Return the bulk move pos object for this resource */ |
80 | static struct ttm_lru_bulk_move_pos * |
81 | ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res) |
82 | { |
83 | return &bulk->pos[res->mem_type][res->bo->priority]; |
84 | } |
85 | |
86 | /* Move the resource to the tail of the bulk move range */ |
87 | static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos, |
88 | struct ttm_resource *res) |
89 | { |
90 | if (pos->last != res) { |
91 | if (pos->first == res) |
92 | pos->first = list_next_entry(res, lru); |
93 | list_move(list: &res->lru, head: &pos->last->lru); |
94 | pos->last = res; |
95 | } |
96 | } |
97 | |
98 | /* Add the resource to a bulk_move cursor */ |
99 | static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk, |
100 | struct ttm_resource *res) |
101 | { |
102 | struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); |
103 | |
104 | if (!pos->first) { |
105 | pos->first = res; |
106 | pos->last = res; |
107 | } else { |
108 | ttm_lru_bulk_move_pos_tail(pos, res); |
109 | } |
110 | } |
111 | |
112 | /* Remove the resource from a bulk_move range */ |
113 | static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk, |
114 | struct ttm_resource *res) |
115 | { |
116 | struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); |
117 | |
118 | if (unlikely(WARN_ON(!pos->first || !pos->last) || |
119 | (pos->first == res && pos->last == res))) { |
120 | pos->first = NULL; |
121 | pos->last = NULL; |
122 | } else if (pos->first == res) { |
123 | pos->first = list_next_entry(res, lru); |
124 | } else if (pos->last == res) { |
125 | pos->last = list_prev_entry(res, lru); |
126 | } else { |
127 | list_move(list: &res->lru, head: &pos->last->lru); |
128 | } |
129 | } |
130 | |
131 | /* Add the resource to a bulk move if the BO is configured for it */ |
132 | void ttm_resource_add_bulk_move(struct ttm_resource *res, |
133 | struct ttm_buffer_object *bo) |
134 | { |
135 | if (bo->bulk_move && !bo->pin_count) |
136 | ttm_lru_bulk_move_add(bulk: bo->bulk_move, res); |
137 | } |
138 | |
139 | /* Remove the resource from a bulk move if the BO is configured for it */ |
140 | void ttm_resource_del_bulk_move(struct ttm_resource *res, |
141 | struct ttm_buffer_object *bo) |
142 | { |
143 | if (bo->bulk_move && !bo->pin_count) |
144 | ttm_lru_bulk_move_del(bulk: bo->bulk_move, res); |
145 | } |
146 | |
147 | /* Move a resource to the LRU or bulk tail */ |
148 | void ttm_resource_move_to_lru_tail(struct ttm_resource *res) |
149 | { |
150 | struct ttm_buffer_object *bo = res->bo; |
151 | struct ttm_device *bdev = bo->bdev; |
152 | |
153 | lockdep_assert_held(&bo->bdev->lru_lock); |
154 | |
155 | if (bo->pin_count) { |
156 | list_move_tail(list: &res->lru, head: &bdev->pinned); |
157 | |
158 | } else if (bo->bulk_move) { |
159 | struct ttm_lru_bulk_move_pos *pos = |
160 | ttm_lru_bulk_move_pos(bulk: bo->bulk_move, res); |
161 | |
162 | ttm_lru_bulk_move_pos_tail(pos, res); |
163 | } else { |
164 | struct ttm_resource_manager *man; |
165 | |
166 | man = ttm_manager_type(bdev, mem_type: res->mem_type); |
167 | list_move_tail(list: &res->lru, head: &man->lru[bo->priority]); |
168 | } |
169 | } |
170 | |
171 | /** |
172 | * ttm_resource_init - resource object constructure |
173 | * @bo: buffer object this resources is allocated for |
174 | * @place: placement of the resource |
175 | * @res: the resource object to inistilize |
176 | * |
177 | * Initialize a new resource object. Counterpart of ttm_resource_fini(). |
178 | */ |
179 | void ttm_resource_init(struct ttm_buffer_object *bo, |
180 | const struct ttm_place *place, |
181 | struct ttm_resource *res) |
182 | { |
183 | struct ttm_resource_manager *man; |
184 | |
185 | res->start = 0; |
186 | res->size = bo->base.size; |
187 | res->mem_type = place->mem_type; |
188 | res->placement = place->flags; |
189 | res->bus.addr = NULL; |
190 | res->bus.offset = 0; |
191 | res->bus.is_iomem = false; |
192 | res->bus.caching = ttm_cached; |
193 | res->bo = bo; |
194 | |
195 | man = ttm_manager_type(bdev: bo->bdev, mem_type: place->mem_type); |
196 | spin_lock(lock: &bo->bdev->lru_lock); |
197 | if (bo->pin_count) |
198 | list_add_tail(new: &res->lru, head: &bo->bdev->pinned); |
199 | else |
200 | list_add_tail(new: &res->lru, head: &man->lru[bo->priority]); |
201 | man->usage += res->size; |
202 | spin_unlock(lock: &bo->bdev->lru_lock); |
203 | } |
204 | EXPORT_SYMBOL(ttm_resource_init); |
205 | |
206 | /** |
207 | * ttm_resource_fini - resource destructor |
208 | * @man: the resource manager this resource belongs to |
209 | * @res: the resource to clean up |
210 | * |
211 | * Should be used by resource manager backends to clean up the TTM resource |
212 | * objects before freeing the underlying structure. Makes sure the resource is |
213 | * removed from the LRU before destruction. |
214 | * Counterpart of ttm_resource_init(). |
215 | */ |
216 | void ttm_resource_fini(struct ttm_resource_manager *man, |
217 | struct ttm_resource *res) |
218 | { |
219 | struct ttm_device *bdev = man->bdev; |
220 | |
221 | spin_lock(lock: &bdev->lru_lock); |
222 | list_del_init(entry: &res->lru); |
223 | man->usage -= res->size; |
224 | spin_unlock(lock: &bdev->lru_lock); |
225 | } |
226 | EXPORT_SYMBOL(ttm_resource_fini); |
227 | |
228 | int ttm_resource_alloc(struct ttm_buffer_object *bo, |
229 | const struct ttm_place *place, |
230 | struct ttm_resource **res_ptr) |
231 | { |
232 | struct ttm_resource_manager *man = |
233 | ttm_manager_type(bdev: bo->bdev, mem_type: place->mem_type); |
234 | int ret; |
235 | |
236 | ret = man->func->alloc(man, bo, place, res_ptr); |
237 | if (ret) |
238 | return ret; |
239 | |
240 | spin_lock(lock: &bo->bdev->lru_lock); |
241 | ttm_resource_add_bulk_move(res: *res_ptr, bo); |
242 | spin_unlock(lock: &bo->bdev->lru_lock); |
243 | return 0; |
244 | } |
245 | EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc); |
246 | |
247 | void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) |
248 | { |
249 | struct ttm_resource_manager *man; |
250 | |
251 | if (!*res) |
252 | return; |
253 | |
254 | spin_lock(lock: &bo->bdev->lru_lock); |
255 | ttm_resource_del_bulk_move(res: *res, bo); |
256 | spin_unlock(lock: &bo->bdev->lru_lock); |
257 | man = ttm_manager_type(bdev: bo->bdev, mem_type: (*res)->mem_type); |
258 | man->func->free(man, *res); |
259 | *res = NULL; |
260 | } |
261 | EXPORT_SYMBOL(ttm_resource_free); |
262 | |
263 | /** |
264 | * ttm_resource_intersects - test for intersection |
265 | * |
266 | * @bdev: TTM device structure |
267 | * @res: The resource to test |
268 | * @place: The placement to test |
269 | * @size: How many bytes the new allocation needs. |
270 | * |
271 | * Test if @res intersects with @place and @size. Used for testing if evictions |
272 | * are valueable or not. |
273 | * |
274 | * Returns true if the res placement intersects with @place and @size. |
275 | */ |
276 | bool ttm_resource_intersects(struct ttm_device *bdev, |
277 | struct ttm_resource *res, |
278 | const struct ttm_place *place, |
279 | size_t size) |
280 | { |
281 | struct ttm_resource_manager *man; |
282 | |
283 | if (!res) |
284 | return false; |
285 | |
286 | man = ttm_manager_type(bdev, mem_type: res->mem_type); |
287 | if (!place || !man->func->intersects) |
288 | return true; |
289 | |
290 | return man->func->intersects(man, res, place, size); |
291 | } |
292 | |
293 | /** |
294 | * ttm_resource_compatible - check if resource is compatible with placement |
295 | * |
296 | * @res: the resource to check |
297 | * @placement: the placement to check against |
298 | * |
299 | * Returns true if the placement is compatible. |
300 | */ |
301 | bool ttm_resource_compatible(struct ttm_resource *res, |
302 | struct ttm_placement *placement) |
303 | { |
304 | struct ttm_buffer_object *bo = res->bo; |
305 | struct ttm_device *bdev = bo->bdev; |
306 | unsigned i; |
307 | |
308 | if (res->placement & TTM_PL_FLAG_TEMPORARY) |
309 | return false; |
310 | |
311 | for (i = 0; i < placement->num_placement; i++) { |
312 | const struct ttm_place *place = &placement->placement[i]; |
313 | struct ttm_resource_manager *man; |
314 | |
315 | if (res->mem_type != place->mem_type) |
316 | continue; |
317 | |
318 | man = ttm_manager_type(bdev, mem_type: res->mem_type); |
319 | if (man->func->compatible && |
320 | !man->func->compatible(man, res, place, bo->base.size)) |
321 | continue; |
322 | |
323 | if ((!(place->flags & TTM_PL_FLAG_CONTIGUOUS) || |
324 | (res->placement & TTM_PL_FLAG_CONTIGUOUS))) |
325 | return true; |
326 | } |
327 | return false; |
328 | } |
329 | |
330 | void ttm_resource_set_bo(struct ttm_resource *res, |
331 | struct ttm_buffer_object *bo) |
332 | { |
333 | spin_lock(lock: &bo->bdev->lru_lock); |
334 | res->bo = bo; |
335 | spin_unlock(lock: &bo->bdev->lru_lock); |
336 | } |
337 | |
338 | /** |
339 | * ttm_resource_manager_init |
340 | * |
341 | * @man: memory manager object to init |
342 | * @bdev: ttm device this manager belongs to |
343 | * @size: size of managed resources in arbitrary units |
344 | * |
345 | * Initialise core parts of a manager object. |
346 | */ |
347 | void ttm_resource_manager_init(struct ttm_resource_manager *man, |
348 | struct ttm_device *bdev, |
349 | uint64_t size) |
350 | { |
351 | unsigned i; |
352 | |
353 | spin_lock_init(&man->move_lock); |
354 | man->bdev = bdev; |
355 | man->size = size; |
356 | man->usage = 0; |
357 | |
358 | for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) |
359 | INIT_LIST_HEAD(list: &man->lru[i]); |
360 | man->move = NULL; |
361 | } |
362 | EXPORT_SYMBOL(ttm_resource_manager_init); |
363 | |
364 | /* |
365 | * ttm_resource_manager_evict_all |
366 | * |
367 | * @bdev - device to use |
368 | * @man - manager to use |
369 | * |
370 | * Evict all the objects out of a memory manager until it is empty. |
371 | * Part of memory manager cleanup sequence. |
372 | */ |
373 | int ttm_resource_manager_evict_all(struct ttm_device *bdev, |
374 | struct ttm_resource_manager *man) |
375 | { |
376 | struct ttm_operation_ctx ctx = { |
377 | .interruptible = false, |
378 | .no_wait_gpu = false, |
379 | .force_alloc = true |
380 | }; |
381 | struct dma_fence *fence; |
382 | int ret; |
383 | unsigned i; |
384 | |
385 | /* |
386 | * Can't use standard list traversal since we're unlocking. |
387 | */ |
388 | |
389 | spin_lock(lock: &bdev->lru_lock); |
390 | for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { |
391 | while (!list_empty(head: &man->lru[i])) { |
392 | spin_unlock(lock: &bdev->lru_lock); |
393 | ret = ttm_mem_evict_first(bdev, man, NULL, ctx: &ctx, |
394 | NULL); |
395 | if (ret) |
396 | return ret; |
397 | spin_lock(lock: &bdev->lru_lock); |
398 | } |
399 | } |
400 | spin_unlock(lock: &bdev->lru_lock); |
401 | |
402 | spin_lock(lock: &man->move_lock); |
403 | fence = dma_fence_get(fence: man->move); |
404 | spin_unlock(lock: &man->move_lock); |
405 | |
406 | if (fence) { |
407 | ret = dma_fence_wait(fence, intr: false); |
408 | dma_fence_put(fence); |
409 | if (ret) |
410 | return ret; |
411 | } |
412 | |
413 | return 0; |
414 | } |
415 | EXPORT_SYMBOL(ttm_resource_manager_evict_all); |
416 | |
417 | /** |
418 | * ttm_resource_manager_usage |
419 | * |
420 | * @man: A memory manager object. |
421 | * |
422 | * Return how many resources are currently used. |
423 | */ |
424 | uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man) |
425 | { |
426 | uint64_t usage; |
427 | |
428 | spin_lock(lock: &man->bdev->lru_lock); |
429 | usage = man->usage; |
430 | spin_unlock(lock: &man->bdev->lru_lock); |
431 | return usage; |
432 | } |
433 | EXPORT_SYMBOL(ttm_resource_manager_usage); |
434 | |
435 | /** |
436 | * ttm_resource_manager_debug |
437 | * |
438 | * @man: manager type to dump. |
439 | * @p: printer to use for debug. |
440 | */ |
441 | void ttm_resource_manager_debug(struct ttm_resource_manager *man, |
442 | struct drm_printer *p) |
443 | { |
444 | drm_printf(p, f: " use_type: %d\n" , man->use_type); |
445 | drm_printf(p, f: " use_tt: %d\n" , man->use_tt); |
446 | drm_printf(p, f: " size: %llu\n" , man->size); |
447 | drm_printf(p, f: " usage: %llu\n" , ttm_resource_manager_usage(man)); |
448 | if (man->func->debug) |
449 | man->func->debug(man, p); |
450 | } |
451 | EXPORT_SYMBOL(ttm_resource_manager_debug); |
452 | |
453 | /** |
454 | * ttm_resource_manager_first |
455 | * |
456 | * @man: resource manager to iterate over |
457 | * @cursor: cursor to record the position |
458 | * |
459 | * Returns the first resource from the resource manager. |
460 | */ |
461 | struct ttm_resource * |
462 | ttm_resource_manager_first(struct ttm_resource_manager *man, |
463 | struct ttm_resource_cursor *cursor) |
464 | { |
465 | struct ttm_resource *res; |
466 | |
467 | lockdep_assert_held(&man->bdev->lru_lock); |
468 | |
469 | for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY; |
470 | ++cursor->priority) |
471 | list_for_each_entry(res, &man->lru[cursor->priority], lru) |
472 | return res; |
473 | |
474 | return NULL; |
475 | } |
476 | |
477 | /** |
478 | * ttm_resource_manager_next |
479 | * |
480 | * @man: resource manager to iterate over |
481 | * @cursor: cursor to record the position |
482 | * @res: the current resource pointer |
483 | * |
484 | * Returns the next resource from the resource manager. |
485 | */ |
486 | struct ttm_resource * |
487 | ttm_resource_manager_next(struct ttm_resource_manager *man, |
488 | struct ttm_resource_cursor *cursor, |
489 | struct ttm_resource *res) |
490 | { |
491 | lockdep_assert_held(&man->bdev->lru_lock); |
492 | |
493 | list_for_each_entry_continue(res, &man->lru[cursor->priority], lru) |
494 | return res; |
495 | |
496 | for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY; |
497 | ++cursor->priority) |
498 | list_for_each_entry(res, &man->lru[cursor->priority], lru) |
499 | return res; |
500 | |
501 | return NULL; |
502 | } |
503 | |
504 | static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter, |
505 | struct iosys_map *dmap, |
506 | pgoff_t i) |
507 | { |
508 | struct ttm_kmap_iter_iomap *iter_io = |
509 | container_of(iter, typeof(*iter_io), base); |
510 | void __iomem *addr; |
511 | |
512 | retry: |
513 | while (i >= iter_io->cache.end) { |
514 | iter_io->cache.sg = iter_io->cache.sg ? |
515 | sg_next(iter_io->cache.sg) : iter_io->st->sgl; |
516 | iter_io->cache.i = iter_io->cache.end; |
517 | iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >> |
518 | PAGE_SHIFT; |
519 | iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) - |
520 | iter_io->start; |
521 | } |
522 | |
523 | if (i < iter_io->cache.i) { |
524 | iter_io->cache.end = 0; |
525 | iter_io->cache.sg = NULL; |
526 | goto retry; |
527 | } |
528 | |
529 | addr = io_mapping_map_local_wc(mapping: iter_io->iomap, offset: iter_io->cache.offs + |
530 | (((resource_size_t)i - iter_io->cache.i) |
531 | << PAGE_SHIFT)); |
532 | iosys_map_set_vaddr_iomem(map: dmap, vaddr_iomem: addr); |
533 | } |
534 | |
535 | static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter, |
536 | struct iosys_map *map) |
537 | { |
538 | io_mapping_unmap_local(vaddr: map->vaddr_iomem); |
539 | } |
540 | |
541 | static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = { |
542 | .map_local = ttm_kmap_iter_iomap_map_local, |
543 | .unmap_local = ttm_kmap_iter_iomap_unmap_local, |
544 | .maps_tt = false, |
545 | }; |
546 | |
547 | /** |
548 | * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap |
549 | * @iter_io: The struct ttm_kmap_iter_iomap to initialize. |
550 | * @iomap: The struct io_mapping representing the underlying linear io_memory. |
551 | * @st: sg_table into @iomap, representing the memory of the struct |
552 | * ttm_resource. |
553 | * @start: Offset that needs to be subtracted from @st to make |
554 | * sg_dma_address(st->sgl) - @start == 0 for @iomap start. |
555 | * |
556 | * Return: Pointer to the embedded struct ttm_kmap_iter. |
557 | */ |
558 | struct ttm_kmap_iter * |
559 | ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, |
560 | struct io_mapping *iomap, |
561 | struct sg_table *st, |
562 | resource_size_t start) |
563 | { |
564 | iter_io->base.ops = &ttm_kmap_iter_io_ops; |
565 | iter_io->iomap = iomap; |
566 | iter_io->st = st; |
567 | iter_io->start = start; |
568 | memset(&iter_io->cache, 0, sizeof(iter_io->cache)); |
569 | |
570 | return &iter_io->base; |
571 | } |
572 | EXPORT_SYMBOL(ttm_kmap_iter_iomap_init); |
573 | |
574 | /** |
575 | * DOC: Linear io iterator |
576 | * |
577 | * This code should die in the not too near future. Best would be if we could |
578 | * make io-mapping use memremap for all io memory, and have memremap |
579 | * implement a kmap_local functionality. We could then strip a huge amount of |
580 | * code. These linear io iterators are implemented to mimic old functionality, |
581 | * and they don't use kmap_local semantics at all internally. Rather ioremap or |
582 | * friends, and at least on 32-bit they add global TLB flushes and points |
583 | * of failure. |
584 | */ |
585 | |
586 | static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter, |
587 | struct iosys_map *dmap, |
588 | pgoff_t i) |
589 | { |
590 | struct ttm_kmap_iter_linear_io *iter_io = |
591 | container_of(iter, typeof(*iter_io), base); |
592 | |
593 | *dmap = iter_io->dmap; |
594 | iosys_map_incr(map: dmap, incr: i * PAGE_SIZE); |
595 | } |
596 | |
597 | static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = { |
598 | .map_local = ttm_kmap_iter_linear_io_map_local, |
599 | .maps_tt = false, |
600 | }; |
601 | |
602 | /** |
603 | * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory |
604 | * @iter_io: The iterator to initialize |
605 | * @bdev: The TTM device |
606 | * @mem: The ttm resource representing the iomap. |
607 | * |
608 | * This function is for internal TTM use only. It sets up a memcpy kmap iterator |
609 | * pointing at a linear chunk of io memory. |
610 | * |
611 | * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on |
612 | * failure. |
613 | */ |
614 | struct ttm_kmap_iter * |
615 | ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, |
616 | struct ttm_device *bdev, |
617 | struct ttm_resource *mem) |
618 | { |
619 | int ret; |
620 | |
621 | ret = ttm_mem_io_reserve(bdev, mem); |
622 | if (ret) |
623 | goto out_err; |
624 | if (!mem->bus.is_iomem) { |
625 | ret = -EINVAL; |
626 | goto out_io_free; |
627 | } |
628 | |
629 | if (mem->bus.addr) { |
630 | iosys_map_set_vaddr(map: &iter_io->dmap, vaddr: mem->bus.addr); |
631 | iter_io->needs_unmap = false; |
632 | } else { |
633 | iter_io->needs_unmap = true; |
634 | memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); |
635 | if (mem->bus.caching == ttm_write_combined) |
636 | iosys_map_set_vaddr_iomem(map: &iter_io->dmap, |
637 | ioremap_wc(offset: mem->bus.offset, |
638 | size: mem->size)); |
639 | else if (mem->bus.caching == ttm_cached) |
640 | iosys_map_set_vaddr(map: &iter_io->dmap, |
641 | vaddr: memremap(offset: mem->bus.offset, size: mem->size, |
642 | flags: MEMREMAP_WB | |
643 | MEMREMAP_WT | |
644 | MEMREMAP_WC)); |
645 | |
646 | /* If uncached requested or if mapping cached or wc failed */ |
647 | if (iosys_map_is_null(map: &iter_io->dmap)) |
648 | iosys_map_set_vaddr_iomem(map: &iter_io->dmap, |
649 | ioremap(offset: mem->bus.offset, |
650 | size: mem->size)); |
651 | |
652 | if (iosys_map_is_null(map: &iter_io->dmap)) { |
653 | ret = -ENOMEM; |
654 | goto out_io_free; |
655 | } |
656 | } |
657 | |
658 | iter_io->base.ops = &ttm_kmap_iter_linear_io_ops; |
659 | return &iter_io->base; |
660 | |
661 | out_io_free: |
662 | ttm_mem_io_free(bdev, mem); |
663 | out_err: |
664 | return ERR_PTR(error: ret); |
665 | } |
666 | |
667 | /** |
668 | * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory |
669 | * @iter_io: The iterator to initialize |
670 | * @bdev: The TTM device |
671 | * @mem: The ttm resource representing the iomap. |
672 | * |
673 | * This function is for internal TTM use only. It cleans up a memcpy kmap |
674 | * iterator initialized by ttm_kmap_iter_linear_io_init. |
675 | */ |
676 | void |
677 | ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, |
678 | struct ttm_device *bdev, |
679 | struct ttm_resource *mem) |
680 | { |
681 | if (iter_io->needs_unmap && iosys_map_is_set(map: &iter_io->dmap)) { |
682 | if (iter_io->dmap.is_iomem) |
683 | iounmap(addr: iter_io->dmap.vaddr_iomem); |
684 | else |
685 | memunmap(addr: iter_io->dmap.vaddr); |
686 | } |
687 | |
688 | ttm_mem_io_free(bdev, mem); |
689 | } |
690 | |
691 | #if defined(CONFIG_DEBUG_FS) |
692 | |
693 | static int ttm_resource_manager_show(struct seq_file *m, void *unused) |
694 | { |
695 | struct ttm_resource_manager *man = |
696 | (struct ttm_resource_manager *)m->private; |
697 | struct drm_printer p = drm_seq_file_printer(f: m); |
698 | ttm_resource_manager_debug(man, &p); |
699 | return 0; |
700 | } |
701 | DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager); |
702 | |
703 | #endif |
704 | |
705 | /** |
706 | * ttm_resource_manager_create_debugfs - Create debugfs entry for specified |
707 | * resource manager. |
708 | * @man: The TTM resource manager for which the debugfs stats file be creates |
709 | * @parent: debugfs directory in which the file will reside |
710 | * @name: The filename to create. |
711 | * |
712 | * This function setups up a debugfs file that can be used to look |
713 | * at debug statistics of the specified ttm_resource_manager. |
714 | */ |
715 | void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, |
716 | struct dentry * parent, |
717 | const char *name) |
718 | { |
719 | #if defined(CONFIG_DEBUG_FS) |
720 | debugfs_create_file(name, mode: 0444, parent, data: man, fops: &ttm_resource_manager_fops); |
721 | #endif |
722 | } |
723 | EXPORT_SYMBOL(ttm_resource_manager_create_debugfs); |
724 | |