1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2020 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | * |
23 | * Authors: Christian König |
24 | */ |
25 | |
26 | /* Pooling of allocated pages is necessary because changing the caching |
27 | * attributes on x86 of the linear mapping requires a costly cross CPU TLB |
28 | * invalidate for those addresses. |
29 | * |
30 | * Additional to that allocations from the DMA coherent API are pooled as well |
31 | * cause they are rather slow compared to alloc_pages+map. |
32 | */ |
33 | |
34 | #include <linux/module.h> |
35 | #include <linux/dma-mapping.h> |
36 | #include <linux/debugfs.h> |
37 | #include <linux/highmem.h> |
38 | #include <linux/sched/mm.h> |
39 | |
40 | #ifdef CONFIG_X86 |
41 | #include <asm/set_memory.h> |
42 | #endif |
43 | |
44 | #include <drm/ttm/ttm_pool.h> |
45 | #include <drm/ttm/ttm_tt.h> |
46 | #include <drm/ttm/ttm_bo.h> |
47 | |
48 | #include "ttm_module.h" |
49 | |
50 | /** |
51 | * struct ttm_pool_dma - Helper object for coherent DMA mappings |
52 | * |
53 | * @addr: original DMA address returned for the mapping |
54 | * @vaddr: original vaddr return for the mapping and order in the lower bits |
55 | */ |
56 | struct ttm_pool_dma { |
57 | dma_addr_t addr; |
58 | unsigned long vaddr; |
59 | }; |
60 | |
61 | static unsigned long page_pool_size; |
62 | |
63 | MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool" ); |
64 | module_param(page_pool_size, ulong, 0644); |
65 | |
66 | static atomic_long_t allocated_pages; |
67 | |
68 | static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS]; |
69 | static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS]; |
70 | |
71 | static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS]; |
72 | static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS]; |
73 | |
74 | static spinlock_t shrinker_lock; |
75 | static struct list_head shrinker_list; |
76 | static struct shrinker *mm_shrinker; |
77 | static DECLARE_RWSEM(pool_shrink_rwsem); |
78 | |
79 | /* Allocate pages of size 1 << order with the given gfp_flags */ |
80 | static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags, |
81 | unsigned int order) |
82 | { |
83 | unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; |
84 | struct ttm_pool_dma *dma; |
85 | struct page *p; |
86 | void *vaddr; |
87 | |
88 | /* Don't set the __GFP_COMP flag for higher order allocations. |
89 | * Mapping pages directly into an userspace process and calling |
90 | * put_page() on a TTM allocated page is illegal. |
91 | */ |
92 | if (order) |
93 | gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | |
94 | __GFP_KSWAPD_RECLAIM; |
95 | |
96 | if (!pool->use_dma_alloc) { |
97 | p = alloc_pages_node(nid: pool->nid, gfp_mask: gfp_flags, order); |
98 | if (p) |
99 | p->private = order; |
100 | return p; |
101 | } |
102 | |
103 | dma = kmalloc(size: sizeof(*dma), GFP_KERNEL); |
104 | if (!dma) |
105 | return NULL; |
106 | |
107 | if (order) |
108 | attr |= DMA_ATTR_NO_WARN; |
109 | |
110 | vaddr = dma_alloc_attrs(dev: pool->dev, size: (1ULL << order) * PAGE_SIZE, |
111 | dma_handle: &dma->addr, flag: gfp_flags, attrs: attr); |
112 | if (!vaddr) |
113 | goto error_free; |
114 | |
115 | /* TODO: This is an illegal abuse of the DMA API, but we need to rework |
116 | * TTM page fault handling and extend the DMA API to clean this up. |
117 | */ |
118 | if (is_vmalloc_addr(x: vaddr)) |
119 | p = vmalloc_to_page(addr: vaddr); |
120 | else |
121 | p = virt_to_page(vaddr); |
122 | |
123 | dma->vaddr = (unsigned long)vaddr | order; |
124 | p->private = (unsigned long)dma; |
125 | return p; |
126 | |
127 | error_free: |
128 | kfree(objp: dma); |
129 | return NULL; |
130 | } |
131 | |
132 | /* Reset the caching and pages of size 1 << order */ |
133 | static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, |
134 | unsigned int order, struct page *p) |
135 | { |
136 | unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; |
137 | struct ttm_pool_dma *dma; |
138 | void *vaddr; |
139 | |
140 | #ifdef CONFIG_X86 |
141 | /* We don't care that set_pages_wb is inefficient here. This is only |
142 | * used when we have to shrink and CPU overhead is irrelevant then. |
143 | */ |
144 | if (caching != ttm_cached && !PageHighMem(page: p)) |
145 | set_pages_wb(page: p, numpages: 1 << order); |
146 | #endif |
147 | |
148 | if (!pool || !pool->use_dma_alloc) { |
149 | __free_pages(page: p, order); |
150 | return; |
151 | } |
152 | |
153 | if (order) |
154 | attr |= DMA_ATTR_NO_WARN; |
155 | |
156 | dma = (void *)p->private; |
157 | vaddr = (void *)(dma->vaddr & PAGE_MASK); |
158 | dma_free_attrs(dev: pool->dev, size: (1UL << order) * PAGE_SIZE, cpu_addr: vaddr, dma_handle: dma->addr, |
159 | attrs: attr); |
160 | kfree(objp: dma); |
161 | } |
162 | |
163 | /* Apply a new caching to an array of pages */ |
164 | static int ttm_pool_apply_caching(struct page **first, struct page **last, |
165 | enum ttm_caching caching) |
166 | { |
167 | #ifdef CONFIG_X86 |
168 | unsigned int num_pages = last - first; |
169 | |
170 | if (!num_pages) |
171 | return 0; |
172 | |
173 | switch (caching) { |
174 | case ttm_cached: |
175 | break; |
176 | case ttm_write_combined: |
177 | return set_pages_array_wc(pages: first, addrinarray: num_pages); |
178 | case ttm_uncached: |
179 | return set_pages_array_uc(pages: first, addrinarray: num_pages); |
180 | } |
181 | #endif |
182 | return 0; |
183 | } |
184 | |
185 | /* Map pages of 1 << order size and fill the DMA address array */ |
186 | static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, |
187 | struct page *p, dma_addr_t **dma_addr) |
188 | { |
189 | dma_addr_t addr; |
190 | unsigned int i; |
191 | |
192 | if (pool->use_dma_alloc) { |
193 | struct ttm_pool_dma *dma = (void *)p->private; |
194 | |
195 | addr = dma->addr; |
196 | } else { |
197 | size_t size = (1ULL << order) * PAGE_SIZE; |
198 | |
199 | addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL); |
200 | if (dma_mapping_error(dev: pool->dev, dma_addr: addr)) |
201 | return -EFAULT; |
202 | } |
203 | |
204 | for (i = 1 << order; i ; --i) { |
205 | *(*dma_addr)++ = addr; |
206 | addr += PAGE_SIZE; |
207 | } |
208 | |
209 | return 0; |
210 | } |
211 | |
212 | /* Unmap pages of 1 << order size */ |
213 | static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, |
214 | unsigned int num_pages) |
215 | { |
216 | /* Unmapped while freeing the page */ |
217 | if (pool->use_dma_alloc) |
218 | return; |
219 | |
220 | dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT, |
221 | DMA_BIDIRECTIONAL); |
222 | } |
223 | |
224 | /* Give pages into a specific pool_type */ |
225 | static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) |
226 | { |
227 | unsigned int i, num_pages = 1 << pt->order; |
228 | |
229 | for (i = 0; i < num_pages; ++i) { |
230 | if (PageHighMem(page: p)) |
231 | clear_highpage(page: p + i); |
232 | else |
233 | clear_page(page_address(p + i)); |
234 | } |
235 | |
236 | spin_lock(lock: &pt->lock); |
237 | list_add(new: &p->lru, head: &pt->pages); |
238 | spin_unlock(lock: &pt->lock); |
239 | atomic_long_add(i: 1 << pt->order, v: &allocated_pages); |
240 | } |
241 | |
242 | /* Take pages from a specific pool_type, return NULL when nothing available */ |
243 | static struct page *ttm_pool_type_take(struct ttm_pool_type *pt) |
244 | { |
245 | struct page *p; |
246 | |
247 | spin_lock(lock: &pt->lock); |
248 | p = list_first_entry_or_null(&pt->pages, typeof(*p), lru); |
249 | if (p) { |
250 | atomic_long_sub(i: 1 << pt->order, v: &allocated_pages); |
251 | list_del(entry: &p->lru); |
252 | } |
253 | spin_unlock(lock: &pt->lock); |
254 | |
255 | return p; |
256 | } |
257 | |
258 | /* Initialize and add a pool type to the global shrinker list */ |
259 | static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool, |
260 | enum ttm_caching caching, unsigned int order) |
261 | { |
262 | pt->pool = pool; |
263 | pt->caching = caching; |
264 | pt->order = order; |
265 | spin_lock_init(&pt->lock); |
266 | INIT_LIST_HEAD(list: &pt->pages); |
267 | |
268 | spin_lock(lock: &shrinker_lock); |
269 | list_add_tail(new: &pt->shrinker_list, head: &shrinker_list); |
270 | spin_unlock(lock: &shrinker_lock); |
271 | } |
272 | |
273 | /* Remove a pool_type from the global shrinker list and free all pages */ |
274 | static void ttm_pool_type_fini(struct ttm_pool_type *pt) |
275 | { |
276 | struct page *p; |
277 | |
278 | spin_lock(lock: &shrinker_lock); |
279 | list_del(entry: &pt->shrinker_list); |
280 | spin_unlock(lock: &shrinker_lock); |
281 | |
282 | while ((p = ttm_pool_type_take(pt))) |
283 | ttm_pool_free_page(pool: pt->pool, caching: pt->caching, order: pt->order, p); |
284 | } |
285 | |
286 | /* Return the pool_type to use for the given caching and order */ |
287 | static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool, |
288 | enum ttm_caching caching, |
289 | unsigned int order) |
290 | { |
291 | if (pool->use_dma_alloc) |
292 | return &pool->caching[caching].orders[order]; |
293 | |
294 | #ifdef CONFIG_X86 |
295 | switch (caching) { |
296 | case ttm_write_combined: |
297 | if (pool->nid != NUMA_NO_NODE) |
298 | return &pool->caching[caching].orders[order]; |
299 | |
300 | if (pool->use_dma32) |
301 | return &global_dma32_write_combined[order]; |
302 | |
303 | return &global_write_combined[order]; |
304 | case ttm_uncached: |
305 | if (pool->nid != NUMA_NO_NODE) |
306 | return &pool->caching[caching].orders[order]; |
307 | |
308 | if (pool->use_dma32) |
309 | return &global_dma32_uncached[order]; |
310 | |
311 | return &global_uncached[order]; |
312 | default: |
313 | break; |
314 | } |
315 | #endif |
316 | |
317 | return NULL; |
318 | } |
319 | |
320 | /* Free pages using the global shrinker list */ |
321 | static unsigned int ttm_pool_shrink(void) |
322 | { |
323 | struct ttm_pool_type *pt; |
324 | unsigned int num_pages; |
325 | struct page *p; |
326 | |
327 | down_read(sem: &pool_shrink_rwsem); |
328 | spin_lock(lock: &shrinker_lock); |
329 | pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list); |
330 | list_move_tail(list: &pt->shrinker_list, head: &shrinker_list); |
331 | spin_unlock(lock: &shrinker_lock); |
332 | |
333 | p = ttm_pool_type_take(pt); |
334 | if (p) { |
335 | ttm_pool_free_page(pool: pt->pool, caching: pt->caching, order: pt->order, p); |
336 | num_pages = 1 << pt->order; |
337 | } else { |
338 | num_pages = 0; |
339 | } |
340 | up_read(sem: &pool_shrink_rwsem); |
341 | |
342 | return num_pages; |
343 | } |
344 | |
345 | /* Return the allocation order based for a page */ |
346 | static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) |
347 | { |
348 | if (pool->use_dma_alloc) { |
349 | struct ttm_pool_dma *dma = (void *)p->private; |
350 | |
351 | return dma->vaddr & ~PAGE_MASK; |
352 | } |
353 | |
354 | return p->private; |
355 | } |
356 | |
357 | /* Called when we got a page, either from a pool or newly allocated */ |
358 | static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, |
359 | struct page *p, dma_addr_t **dma_addr, |
360 | unsigned long *num_pages, |
361 | struct page ***pages) |
362 | { |
363 | unsigned int i; |
364 | int r; |
365 | |
366 | if (*dma_addr) { |
367 | r = ttm_pool_map(pool, order, p, dma_addr); |
368 | if (r) |
369 | return r; |
370 | } |
371 | |
372 | *num_pages -= 1 << order; |
373 | for (i = 1 << order; i; --i, ++(*pages), ++p) |
374 | **pages = p; |
375 | |
376 | return 0; |
377 | } |
378 | |
379 | /** |
380 | * ttm_pool_free_range() - Free a range of TTM pages |
381 | * @pool: The pool used for allocating. |
382 | * @tt: The struct ttm_tt holding the page pointers. |
383 | * @caching: The page caching mode used by the range. |
384 | * @start_page: index for first page to free. |
385 | * @end_page: index for last page to free + 1. |
386 | * |
387 | * During allocation the ttm_tt page-vector may be populated with ranges of |
388 | * pages with different attributes if allocation hit an error without being |
389 | * able to completely fulfill the allocation. This function can be used |
390 | * to free these individual ranges. |
391 | */ |
392 | static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, |
393 | enum ttm_caching caching, |
394 | pgoff_t start_page, pgoff_t end_page) |
395 | { |
396 | struct page **pages = &tt->pages[start_page]; |
397 | unsigned int order; |
398 | pgoff_t i, nr; |
399 | |
400 | for (i = start_page; i < end_page; i += nr, pages += nr) { |
401 | struct ttm_pool_type *pt = NULL; |
402 | |
403 | order = ttm_pool_page_order(pool, p: *pages); |
404 | nr = (1UL << order); |
405 | if (tt->dma_address) |
406 | ttm_pool_unmap(pool, dma_addr: tt->dma_address[i], num_pages: nr); |
407 | |
408 | pt = ttm_pool_select_type(pool, caching, order); |
409 | if (pt) |
410 | ttm_pool_type_give(pt, p: *pages); |
411 | else |
412 | ttm_pool_free_page(pool, caching, order, p: *pages); |
413 | } |
414 | } |
415 | |
416 | /** |
417 | * ttm_pool_alloc - Fill a ttm_tt object |
418 | * |
419 | * @pool: ttm_pool to use |
420 | * @tt: ttm_tt object to fill |
421 | * @ctx: operation context |
422 | * |
423 | * Fill the ttm_tt object with pages and also make sure to DMA map them when |
424 | * necessary. |
425 | * |
426 | * Returns: 0 on successe, negative error code otherwise. |
427 | */ |
428 | int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, |
429 | struct ttm_operation_ctx *ctx) |
430 | { |
431 | pgoff_t num_pages = tt->num_pages; |
432 | dma_addr_t *dma_addr = tt->dma_address; |
433 | struct page **caching = tt->pages; |
434 | struct page **pages = tt->pages; |
435 | enum ttm_caching page_caching; |
436 | gfp_t gfp_flags = GFP_USER; |
437 | pgoff_t caching_divide; |
438 | unsigned int order; |
439 | struct page *p; |
440 | int r; |
441 | |
442 | WARN_ON(!num_pages || ttm_tt_is_populated(tt)); |
443 | WARN_ON(dma_addr && !pool->dev); |
444 | |
445 | if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC) |
446 | gfp_flags |= __GFP_ZERO; |
447 | |
448 | if (ctx->gfp_retry_mayfail) |
449 | gfp_flags |= __GFP_RETRY_MAYFAIL; |
450 | |
451 | if (pool->use_dma32) |
452 | gfp_flags |= GFP_DMA32; |
453 | else |
454 | gfp_flags |= GFP_HIGHUSER; |
455 | |
456 | for (order = min_t(unsigned int, MAX_PAGE_ORDER, __fls(num_pages)); |
457 | num_pages; |
458 | order = min_t(unsigned int, order, __fls(num_pages))) { |
459 | struct ttm_pool_type *pt; |
460 | |
461 | page_caching = tt->caching; |
462 | pt = ttm_pool_select_type(pool, caching: tt->caching, order); |
463 | p = pt ? ttm_pool_type_take(pt) : NULL; |
464 | if (p) { |
465 | r = ttm_pool_apply_caching(first: caching, last: pages, |
466 | caching: tt->caching); |
467 | if (r) |
468 | goto error_free_page; |
469 | |
470 | caching = pages; |
471 | do { |
472 | r = ttm_pool_page_allocated(pool, order, p, |
473 | dma_addr: &dma_addr, |
474 | num_pages: &num_pages, |
475 | pages: &pages); |
476 | if (r) |
477 | goto error_free_page; |
478 | |
479 | caching = pages; |
480 | if (num_pages < (1 << order)) |
481 | break; |
482 | |
483 | p = ttm_pool_type_take(pt); |
484 | } while (p); |
485 | } |
486 | |
487 | page_caching = ttm_cached; |
488 | while (num_pages >= (1 << order) && |
489 | (p = ttm_pool_alloc_page(pool, gfp_flags, order))) { |
490 | |
491 | if (PageHighMem(page: p)) { |
492 | r = ttm_pool_apply_caching(first: caching, last: pages, |
493 | caching: tt->caching); |
494 | if (r) |
495 | goto error_free_page; |
496 | caching = pages; |
497 | } |
498 | r = ttm_pool_page_allocated(pool, order, p, dma_addr: &dma_addr, |
499 | num_pages: &num_pages, pages: &pages); |
500 | if (r) |
501 | goto error_free_page; |
502 | if (PageHighMem(page: p)) |
503 | caching = pages; |
504 | } |
505 | |
506 | if (!p) { |
507 | if (order) { |
508 | --order; |
509 | continue; |
510 | } |
511 | r = -ENOMEM; |
512 | goto error_free_all; |
513 | } |
514 | } |
515 | |
516 | r = ttm_pool_apply_caching(first: caching, last: pages, caching: tt->caching); |
517 | if (r) |
518 | goto error_free_all; |
519 | |
520 | return 0; |
521 | |
522 | error_free_page: |
523 | ttm_pool_free_page(pool, caching: page_caching, order, p); |
524 | |
525 | error_free_all: |
526 | num_pages = tt->num_pages - num_pages; |
527 | caching_divide = caching - tt->pages; |
528 | ttm_pool_free_range(pool, tt, caching: tt->caching, start_page: 0, end_page: caching_divide); |
529 | ttm_pool_free_range(pool, tt, caching: ttm_cached, start_page: caching_divide, end_page: num_pages); |
530 | |
531 | return r; |
532 | } |
533 | EXPORT_SYMBOL(ttm_pool_alloc); |
534 | |
535 | /** |
536 | * ttm_pool_free - Free the backing pages from a ttm_tt object |
537 | * |
538 | * @pool: Pool to give pages back to. |
539 | * @tt: ttm_tt object to unpopulate |
540 | * |
541 | * Give the packing pages back to a pool or free them |
542 | */ |
543 | void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) |
544 | { |
545 | ttm_pool_free_range(pool, tt, caching: tt->caching, start_page: 0, end_page: tt->num_pages); |
546 | |
547 | while (atomic_long_read(v: &allocated_pages) > page_pool_size) |
548 | ttm_pool_shrink(); |
549 | } |
550 | EXPORT_SYMBOL(ttm_pool_free); |
551 | |
552 | /** |
553 | * ttm_pool_init - Initialize a pool |
554 | * |
555 | * @pool: the pool to initialize |
556 | * @dev: device for DMA allocations and mappings |
557 | * @nid: NUMA node to use for allocations |
558 | * @use_dma_alloc: true if coherent DMA alloc should be used |
559 | * @use_dma32: true if GFP_DMA32 should be used |
560 | * |
561 | * Initialize the pool and its pool types. |
562 | */ |
563 | void ttm_pool_init(struct ttm_pool *pool, struct device *dev, |
564 | int nid, bool use_dma_alloc, bool use_dma32) |
565 | { |
566 | unsigned int i, j; |
567 | |
568 | WARN_ON(!dev && use_dma_alloc); |
569 | |
570 | pool->dev = dev; |
571 | pool->nid = nid; |
572 | pool->use_dma_alloc = use_dma_alloc; |
573 | pool->use_dma32 = use_dma32; |
574 | |
575 | for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { |
576 | for (j = 0; j < NR_PAGE_ORDERS; ++j) { |
577 | struct ttm_pool_type *pt; |
578 | |
579 | /* Initialize only pool types which are actually used */ |
580 | pt = ttm_pool_select_type(pool, caching: i, order: j); |
581 | if (pt != &pool->caching[i].orders[j]) |
582 | continue; |
583 | |
584 | ttm_pool_type_init(pt, pool, caching: i, order: j); |
585 | } |
586 | } |
587 | } |
588 | EXPORT_SYMBOL(ttm_pool_init); |
589 | |
590 | /** |
591 | * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete. |
592 | * |
593 | * This is useful to guarantee that all shrinker invocations have seen an |
594 | * update, before freeing memory, similar to rcu. |
595 | */ |
596 | static void ttm_pool_synchronize_shrinkers(void) |
597 | { |
598 | down_write(sem: &pool_shrink_rwsem); |
599 | up_write(sem: &pool_shrink_rwsem); |
600 | } |
601 | |
602 | /** |
603 | * ttm_pool_fini - Cleanup a pool |
604 | * |
605 | * @pool: the pool to clean up |
606 | * |
607 | * Free all pages in the pool and unregister the types from the global |
608 | * shrinker. |
609 | */ |
610 | void ttm_pool_fini(struct ttm_pool *pool) |
611 | { |
612 | unsigned int i, j; |
613 | |
614 | for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { |
615 | for (j = 0; j < NR_PAGE_ORDERS; ++j) { |
616 | struct ttm_pool_type *pt; |
617 | |
618 | pt = ttm_pool_select_type(pool, caching: i, order: j); |
619 | if (pt != &pool->caching[i].orders[j]) |
620 | continue; |
621 | |
622 | ttm_pool_type_fini(pt); |
623 | } |
624 | } |
625 | |
626 | /* We removed the pool types from the LRU, but we need to also make sure |
627 | * that no shrinker is concurrently freeing pages from the pool. |
628 | */ |
629 | ttm_pool_synchronize_shrinkers(); |
630 | } |
631 | EXPORT_SYMBOL(ttm_pool_fini); |
632 | |
633 | /* As long as pages are available make sure to release at least one */ |
634 | static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, |
635 | struct shrink_control *sc) |
636 | { |
637 | unsigned long num_freed = 0; |
638 | |
639 | do |
640 | num_freed += ttm_pool_shrink(); |
641 | while (!num_freed && atomic_long_read(v: &allocated_pages)); |
642 | |
643 | return num_freed; |
644 | } |
645 | |
646 | /* Return the number of pages available or SHRINK_EMPTY if we have none */ |
647 | static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, |
648 | struct shrink_control *sc) |
649 | { |
650 | unsigned long num_pages = atomic_long_read(v: &allocated_pages); |
651 | |
652 | return num_pages ? num_pages : SHRINK_EMPTY; |
653 | } |
654 | |
655 | #ifdef CONFIG_DEBUG_FS |
656 | /* Count the number of pages available in a pool_type */ |
657 | static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) |
658 | { |
659 | unsigned int count = 0; |
660 | struct page *p; |
661 | |
662 | spin_lock(lock: &pt->lock); |
663 | /* Only used for debugfs, the overhead doesn't matter */ |
664 | list_for_each_entry(p, &pt->pages, lru) |
665 | ++count; |
666 | spin_unlock(lock: &pt->lock); |
667 | |
668 | return count; |
669 | } |
670 | |
671 | /* Print a nice header for the order */ |
672 | static void (struct seq_file *m) |
673 | { |
674 | unsigned int i; |
675 | |
676 | seq_puts(m, s: "\t " ); |
677 | for (i = 0; i < NR_PAGE_ORDERS; ++i) |
678 | seq_printf(m, fmt: " ---%2u---" , i); |
679 | seq_puts(m, s: "\n" ); |
680 | } |
681 | |
682 | /* Dump information about the different pool types */ |
683 | static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, |
684 | struct seq_file *m) |
685 | { |
686 | unsigned int i; |
687 | |
688 | for (i = 0; i < NR_PAGE_ORDERS; ++i) |
689 | seq_printf(m, fmt: " %8u" , ttm_pool_type_count(pt: &pt[i])); |
690 | seq_puts(m, s: "\n" ); |
691 | } |
692 | |
693 | /* Dump the total amount of allocated pages */ |
694 | static void (struct seq_file *m) |
695 | { |
696 | seq_printf(m, fmt: "\ntotal\t: %8lu of %8lu\n" , |
697 | atomic_long_read(v: &allocated_pages), page_pool_size); |
698 | } |
699 | |
700 | /* Dump the information for the global pools */ |
701 | static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data) |
702 | { |
703 | ttm_pool_debugfs_header(m); |
704 | |
705 | spin_lock(lock: &shrinker_lock); |
706 | seq_puts(m, s: "wc\t:" ); |
707 | ttm_pool_debugfs_orders(pt: global_write_combined, m); |
708 | seq_puts(m, s: "uc\t:" ); |
709 | ttm_pool_debugfs_orders(pt: global_uncached, m); |
710 | seq_puts(m, s: "wc 32\t:" ); |
711 | ttm_pool_debugfs_orders(pt: global_dma32_write_combined, m); |
712 | seq_puts(m, s: "uc 32\t:" ); |
713 | ttm_pool_debugfs_orders(pt: global_dma32_uncached, m); |
714 | spin_unlock(lock: &shrinker_lock); |
715 | |
716 | ttm_pool_debugfs_footer(m); |
717 | |
718 | return 0; |
719 | } |
720 | DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals); |
721 | |
722 | /** |
723 | * ttm_pool_debugfs - Debugfs dump function for a pool |
724 | * |
725 | * @pool: the pool to dump the information for |
726 | * @m: seq_file to dump to |
727 | * |
728 | * Make a debugfs dump with the per pool and global information. |
729 | */ |
730 | int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) |
731 | { |
732 | unsigned int i; |
733 | |
734 | if (!pool->use_dma_alloc) { |
735 | seq_puts(m, s: "unused\n" ); |
736 | return 0; |
737 | } |
738 | |
739 | ttm_pool_debugfs_header(m); |
740 | |
741 | spin_lock(lock: &shrinker_lock); |
742 | for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { |
743 | seq_puts(m, s: "DMA " ); |
744 | switch (i) { |
745 | case ttm_cached: |
746 | seq_puts(m, s: "\t:" ); |
747 | break; |
748 | case ttm_write_combined: |
749 | seq_puts(m, s: "wc\t:" ); |
750 | break; |
751 | case ttm_uncached: |
752 | seq_puts(m, s: "uc\t:" ); |
753 | break; |
754 | } |
755 | ttm_pool_debugfs_orders(pt: pool->caching[i].orders, m); |
756 | } |
757 | spin_unlock(lock: &shrinker_lock); |
758 | |
759 | ttm_pool_debugfs_footer(m); |
760 | return 0; |
761 | } |
762 | EXPORT_SYMBOL(ttm_pool_debugfs); |
763 | |
764 | /* Test the shrinker functions and dump the result */ |
765 | static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) |
766 | { |
767 | struct shrink_control sc = { .gfp_mask = GFP_NOFS }; |
768 | |
769 | fs_reclaim_acquire(GFP_KERNEL); |
770 | seq_printf(m, fmt: "%lu/%lu\n" , ttm_pool_shrinker_count(shrink: mm_shrinker, sc: &sc), |
771 | ttm_pool_shrinker_scan(shrink: mm_shrinker, sc: &sc)); |
772 | fs_reclaim_release(GFP_KERNEL); |
773 | |
774 | return 0; |
775 | } |
776 | DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); |
777 | |
778 | #endif |
779 | |
780 | /** |
781 | * ttm_pool_mgr_init - Initialize globals |
782 | * |
783 | * @num_pages: default number of pages |
784 | * |
785 | * Initialize the global locks and lists for the MM shrinker. |
786 | */ |
787 | int ttm_pool_mgr_init(unsigned long num_pages) |
788 | { |
789 | unsigned int i; |
790 | |
791 | if (!page_pool_size) |
792 | page_pool_size = num_pages; |
793 | |
794 | spin_lock_init(&shrinker_lock); |
795 | INIT_LIST_HEAD(list: &shrinker_list); |
796 | |
797 | for (i = 0; i < NR_PAGE_ORDERS; ++i) { |
798 | ttm_pool_type_init(pt: &global_write_combined[i], NULL, |
799 | caching: ttm_write_combined, order: i); |
800 | ttm_pool_type_init(pt: &global_uncached[i], NULL, caching: ttm_uncached, order: i); |
801 | |
802 | ttm_pool_type_init(pt: &global_dma32_write_combined[i], NULL, |
803 | caching: ttm_write_combined, order: i); |
804 | ttm_pool_type_init(pt: &global_dma32_uncached[i], NULL, |
805 | caching: ttm_uncached, order: i); |
806 | } |
807 | |
808 | #ifdef CONFIG_DEBUG_FS |
809 | debugfs_create_file(name: "page_pool" , mode: 0444, parent: ttm_debugfs_root, NULL, |
810 | fops: &ttm_pool_debugfs_globals_fops); |
811 | debugfs_create_file(name: "page_pool_shrink" , mode: 0400, parent: ttm_debugfs_root, NULL, |
812 | fops: &ttm_pool_debugfs_shrink_fops); |
813 | #endif |
814 | |
815 | mm_shrinker = shrinker_alloc(flags: 0, fmt: "drm-ttm_pool" ); |
816 | if (!mm_shrinker) |
817 | return -ENOMEM; |
818 | |
819 | mm_shrinker->count_objects = ttm_pool_shrinker_count; |
820 | mm_shrinker->scan_objects = ttm_pool_shrinker_scan; |
821 | mm_shrinker->seeks = 1; |
822 | |
823 | shrinker_register(shrinker: mm_shrinker); |
824 | |
825 | return 0; |
826 | } |
827 | |
828 | /** |
829 | * ttm_pool_mgr_fini - Finalize globals |
830 | * |
831 | * Cleanup the global pools and unregister the MM shrinker. |
832 | */ |
833 | void ttm_pool_mgr_fini(void) |
834 | { |
835 | unsigned int i; |
836 | |
837 | for (i = 0; i < NR_PAGE_ORDERS; ++i) { |
838 | ttm_pool_type_fini(pt: &global_write_combined[i]); |
839 | ttm_pool_type_fini(pt: &global_uncached[i]); |
840 | |
841 | ttm_pool_type_fini(pt: &global_dma32_write_combined[i]); |
842 | ttm_pool_type_fini(pt: &global_dma32_uncached[i]); |
843 | } |
844 | |
845 | shrinker_free(shrinker: mm_shrinker); |
846 | WARN_ON(!list_empty(&shrinker_list)); |
847 | } |
848 | |