1 | /* |
2 | * Copyright 2020 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | * |
22 | * Authors: Christian König |
23 | */ |
24 | |
25 | #ifndef _TTM_RESOURCE_H_ |
26 | #define _TTM_RESOURCE_H_ |
27 | |
28 | #include <linux/types.h> |
29 | #include <linux/list.h> |
30 | #include <linux/mutex.h> |
31 | #include <linux/iosys-map.h> |
32 | #include <linux/dma-fence.h> |
33 | |
34 | #include <drm/drm_print.h> |
35 | #include <drm/ttm/ttm_caching.h> |
36 | #include <drm/ttm/ttm_kmap_iter.h> |
37 | |
38 | #define TTM_MAX_BO_PRIORITY 4U |
39 | #define TTM_NUM_MEM_TYPES 8 |
40 | |
41 | struct ttm_device; |
42 | struct ttm_resource_manager; |
43 | struct ttm_resource; |
44 | struct ttm_place; |
45 | struct ttm_buffer_object; |
46 | struct ttm_placement; |
47 | struct iosys_map; |
48 | struct io_mapping; |
49 | struct sg_table; |
50 | struct scatterlist; |
51 | |
52 | struct ttm_resource_manager_func { |
53 | /** |
54 | * struct ttm_resource_manager_func member alloc |
55 | * |
56 | * @man: Pointer to a memory type manager. |
57 | * @bo: Pointer to the buffer object we're allocating space for. |
58 | * @place: Placement details. |
59 | * @res: Resulting pointer to the ttm_resource. |
60 | * |
61 | * This function should allocate space in the memory type managed |
62 | * by @man. Placement details if applicable are given by @place. If |
63 | * successful, a filled in ttm_resource object should be returned in |
64 | * @res. @res::start should be set to a value identifying the beginning |
65 | * of the range allocated, and the function should return zero. |
66 | * If the manager can't fulfill the request -ENOSPC should be returned. |
67 | * If a system error occurred, preventing the request to be fulfilled, |
68 | * the function should return a negative error code. |
69 | * |
70 | * This function may not be called from within atomic context and needs |
71 | * to take care of its own locking to protect any data structures |
72 | * managing the space. |
73 | */ |
74 | int (*alloc)(struct ttm_resource_manager *man, |
75 | struct ttm_buffer_object *bo, |
76 | const struct ttm_place *place, |
77 | struct ttm_resource **res); |
78 | |
79 | /** |
80 | * struct ttm_resource_manager_func member free |
81 | * |
82 | * @man: Pointer to a memory type manager. |
83 | * @res: Pointer to a struct ttm_resource to be freed. |
84 | * |
85 | * This function frees memory type resources previously allocated. |
86 | * May not be called from within atomic context. |
87 | */ |
88 | void (*free)(struct ttm_resource_manager *man, |
89 | struct ttm_resource *res); |
90 | |
91 | /** |
92 | * struct ttm_resource_manager_func member intersects |
93 | * |
94 | * @man: Pointer to a memory type manager. |
95 | * @res: Pointer to a struct ttm_resource to be checked. |
96 | * @place: Placement to check against. |
97 | * @size: Size of the check. |
98 | * |
99 | * Test if @res intersects with @place + @size. Used to judge if |
100 | * evictions are valueable or not. |
101 | */ |
102 | bool (*intersects)(struct ttm_resource_manager *man, |
103 | struct ttm_resource *res, |
104 | const struct ttm_place *place, |
105 | size_t size); |
106 | |
107 | /** |
108 | * struct ttm_resource_manager_func member compatible |
109 | * |
110 | * @man: Pointer to a memory type manager. |
111 | * @res: Pointer to a struct ttm_resource to be checked. |
112 | * @place: Placement to check against. |
113 | * @size: Size of the check. |
114 | * |
115 | * Test if @res compatible with @place + @size. Used to check of |
116 | * the need to move the backing store or not. |
117 | */ |
118 | bool (*compatible)(struct ttm_resource_manager *man, |
119 | struct ttm_resource *res, |
120 | const struct ttm_place *place, |
121 | size_t size); |
122 | |
123 | /** |
124 | * struct ttm_resource_manager_func member debug |
125 | * |
126 | * @man: Pointer to a memory type manager. |
127 | * @printer: Prefix to be used in printout to identify the caller. |
128 | * |
129 | * This function is called to print out the state of the memory |
130 | * type manager to aid debugging of out-of-memory conditions. |
131 | * It may not be called from within atomic context. |
132 | */ |
133 | void (*debug)(struct ttm_resource_manager *man, |
134 | struct drm_printer *printer); |
135 | }; |
136 | |
137 | /** |
138 | * struct ttm_resource_manager |
139 | * |
140 | * @use_type: The memory type is enabled. |
141 | * @use_tt: If a TT object should be used for the backing store. |
142 | * @size: Size of the managed region. |
143 | * @bdev: ttm device this manager belongs to |
144 | * @func: structure pointer implementing the range manager. See above |
145 | * @move_lock: lock for move fence |
146 | * @move: The fence of the last pipelined move operation. |
147 | * @lru: The lru list for this memory type. |
148 | * |
149 | * This structure is used to identify and manage memory types for a device. |
150 | */ |
151 | struct ttm_resource_manager { |
152 | /* |
153 | * No protection. Constant from start. |
154 | */ |
155 | bool use_type; |
156 | bool use_tt; |
157 | struct ttm_device *bdev; |
158 | uint64_t size; |
159 | const struct ttm_resource_manager_func *func; |
160 | spinlock_t move_lock; |
161 | |
162 | /* |
163 | * Protected by @move_lock. |
164 | */ |
165 | struct dma_fence *move; |
166 | |
167 | /* |
168 | * Protected by the bdev->lru_lock. |
169 | */ |
170 | struct list_head lru[TTM_MAX_BO_PRIORITY]; |
171 | |
172 | /** |
173 | * @usage: How much of the resources are used, protected by the |
174 | * bdev->lru_lock. |
175 | */ |
176 | uint64_t usage; |
177 | }; |
178 | |
179 | /** |
180 | * struct ttm_bus_placement |
181 | * |
182 | * @addr: mapped virtual address |
183 | * @offset: physical addr |
184 | * @is_iomem: is this io memory ? |
185 | * @caching: See enum ttm_caching |
186 | * |
187 | * Structure indicating the bus placement of an object. |
188 | */ |
189 | struct ttm_bus_placement { |
190 | void *addr; |
191 | phys_addr_t offset; |
192 | bool is_iomem; |
193 | enum ttm_caching caching; |
194 | }; |
195 | |
196 | /** |
197 | * struct ttm_resource |
198 | * |
199 | * @start: Start of the allocation. |
200 | * @size: Actual size of resource in bytes. |
201 | * @mem_type: Resource type of the allocation. |
202 | * @placement: Placement flags. |
203 | * @bus: Placement on io bus accessible to the CPU |
204 | * @bo: weak reference to the BO, protected by ttm_device::lru_lock |
205 | * |
206 | * Structure indicating the placement and space resources used by a |
207 | * buffer object. |
208 | */ |
209 | struct ttm_resource { |
210 | unsigned long start; |
211 | size_t size; |
212 | uint32_t mem_type; |
213 | uint32_t placement; |
214 | struct ttm_bus_placement bus; |
215 | struct ttm_buffer_object *bo; |
216 | |
217 | /** |
218 | * @lru: Least recently used list, see &ttm_resource_manager.lru |
219 | */ |
220 | struct list_head lru; |
221 | }; |
222 | |
223 | /** |
224 | * struct ttm_resource_cursor |
225 | * |
226 | * @priority: the current priority |
227 | * |
228 | * Cursor to iterate over the resources in a manager. |
229 | */ |
230 | struct ttm_resource_cursor { |
231 | unsigned int priority; |
232 | }; |
233 | |
234 | /** |
235 | * struct ttm_lru_bulk_move_pos |
236 | * |
237 | * @first: first res in the bulk move range |
238 | * @last: last res in the bulk move range |
239 | * |
240 | * Range of resources for a lru bulk move. |
241 | */ |
242 | struct ttm_lru_bulk_move_pos { |
243 | struct ttm_resource *first; |
244 | struct ttm_resource *last; |
245 | }; |
246 | |
247 | /** |
248 | * struct ttm_lru_bulk_move |
249 | * |
250 | * @pos: first/last lru entry for resources in the each domain/priority |
251 | * |
252 | * Container for the current bulk move state. Should be used with |
253 | * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). |
254 | */ |
255 | struct ttm_lru_bulk_move { |
256 | struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; |
257 | }; |
258 | |
259 | /** |
260 | * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + |
261 | * struct sg_table backed struct ttm_resource. |
262 | * @base: Embedded struct ttm_kmap_iter providing the usage interface. |
263 | * @iomap: struct io_mapping representing the underlying linear io_memory. |
264 | * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. |
265 | * @start: Offset that needs to be subtracted from @st to make |
266 | * sg_dma_address(st->sgl) - @start == 0 for @iomap start. |
267 | * @cache: Scatterlist traversal cache for fast lookups. |
268 | * @cache.sg: Pointer to the currently cached scatterlist segment. |
269 | * @cache.i: First index of @sg. PAGE_SIZE granularity. |
270 | * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. |
271 | * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. |
272 | */ |
273 | struct ttm_kmap_iter_iomap { |
274 | struct ttm_kmap_iter base; |
275 | struct io_mapping *iomap; |
276 | struct sg_table *st; |
277 | resource_size_t start; |
278 | struct { |
279 | struct scatterlist *sg; |
280 | pgoff_t i; |
281 | pgoff_t end; |
282 | pgoff_t offs; |
283 | } cache; |
284 | }; |
285 | |
286 | /** |
287 | * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io |
288 | * @base: The base iterator |
289 | * @dmap: Points to the starting address of the region |
290 | * @needs_unmap: Whether we need to unmap on fini |
291 | */ |
292 | struct ttm_kmap_iter_linear_io { |
293 | struct ttm_kmap_iter base; |
294 | struct iosys_map dmap; |
295 | bool needs_unmap; |
296 | }; |
297 | |
298 | /** |
299 | * ttm_resource_manager_set_used |
300 | * |
301 | * @man: A memory manager object. |
302 | * @used: usage state to set. |
303 | * |
304 | * Set the manager in use flag. If disabled the manager is no longer |
305 | * used for object placement. |
306 | */ |
307 | static inline void |
308 | ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) |
309 | { |
310 | int i; |
311 | |
312 | for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) |
313 | WARN_ON(!list_empty(&man->lru[i])); |
314 | man->use_type = used; |
315 | } |
316 | |
317 | /** |
318 | * ttm_resource_manager_used |
319 | * |
320 | * @man: Manager to get used state for |
321 | * |
322 | * Get the in use flag for a manager. |
323 | * Returns: |
324 | * true is used, false if not. |
325 | */ |
326 | static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) |
327 | { |
328 | return man->use_type; |
329 | } |
330 | |
331 | /** |
332 | * ttm_resource_manager_cleanup |
333 | * |
334 | * @man: A memory manager object. |
335 | * |
336 | * Cleanup the move fences from the memory manager object. |
337 | */ |
338 | static inline void |
339 | ttm_resource_manager_cleanup(struct ttm_resource_manager *man) |
340 | { |
341 | dma_fence_put(fence: man->move); |
342 | man->move = NULL; |
343 | } |
344 | |
345 | void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); |
346 | void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); |
347 | |
348 | void ttm_resource_add_bulk_move(struct ttm_resource *res, |
349 | struct ttm_buffer_object *bo); |
350 | void ttm_resource_del_bulk_move(struct ttm_resource *res, |
351 | struct ttm_buffer_object *bo); |
352 | void ttm_resource_move_to_lru_tail(struct ttm_resource *res); |
353 | |
354 | void ttm_resource_init(struct ttm_buffer_object *bo, |
355 | const struct ttm_place *place, |
356 | struct ttm_resource *res); |
357 | void ttm_resource_fini(struct ttm_resource_manager *man, |
358 | struct ttm_resource *res); |
359 | |
360 | int ttm_resource_alloc(struct ttm_buffer_object *bo, |
361 | const struct ttm_place *place, |
362 | struct ttm_resource **res); |
363 | void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); |
364 | bool ttm_resource_intersects(struct ttm_device *bdev, |
365 | struct ttm_resource *res, |
366 | const struct ttm_place *place, |
367 | size_t size); |
368 | bool ttm_resource_compatible(struct ttm_device *bdev, |
369 | struct ttm_resource *res, |
370 | const struct ttm_place *place, |
371 | size_t size); |
372 | bool ttm_resource_compat(struct ttm_resource *res, |
373 | struct ttm_placement *placement); |
374 | void ttm_resource_set_bo(struct ttm_resource *res, |
375 | struct ttm_buffer_object *bo); |
376 | |
377 | void ttm_resource_manager_init(struct ttm_resource_manager *man, |
378 | struct ttm_device *bdev, |
379 | uint64_t size); |
380 | |
381 | int ttm_resource_manager_evict_all(struct ttm_device *bdev, |
382 | struct ttm_resource_manager *man); |
383 | |
384 | uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); |
385 | void ttm_resource_manager_debug(struct ttm_resource_manager *man, |
386 | struct drm_printer *p); |
387 | |
388 | struct ttm_resource * |
389 | ttm_resource_manager_first(struct ttm_resource_manager *man, |
390 | struct ttm_resource_cursor *cursor); |
391 | struct ttm_resource * |
392 | ttm_resource_manager_next(struct ttm_resource_manager *man, |
393 | struct ttm_resource_cursor *cursor, |
394 | struct ttm_resource *res); |
395 | |
396 | /** |
397 | * ttm_resource_manager_for_each_res - iterate over all resources |
398 | * @man: the resource manager |
399 | * @cursor: struct ttm_resource_cursor for the current position |
400 | * @res: the current resource |
401 | * |
402 | * Iterate over all the evictable resources in a resource manager. |
403 | */ |
404 | #define ttm_resource_manager_for_each_res(man, cursor, res) \ |
405 | for (res = ttm_resource_manager_first(man, cursor); res; \ |
406 | res = ttm_resource_manager_next(man, cursor, res)) |
407 | |
408 | struct ttm_kmap_iter * |
409 | ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, |
410 | struct io_mapping *iomap, |
411 | struct sg_table *st, |
412 | resource_size_t start); |
413 | |
414 | struct ttm_kmap_iter_linear_io; |
415 | |
416 | struct ttm_kmap_iter * |
417 | ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, |
418 | struct ttm_device *bdev, |
419 | struct ttm_resource *mem); |
420 | |
421 | void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, |
422 | struct ttm_device *bdev, |
423 | struct ttm_resource *mem); |
424 | |
425 | void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, |
426 | struct dentry * parent, |
427 | const char *name); |
428 | #endif |
429 | |