1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2021 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/shmem_fs.h> |
7 | |
8 | #include <drm/ttm/ttm_placement.h> |
9 | #include <drm/ttm/ttm_tt.h> |
10 | #include <drm/drm_buddy.h> |
11 | |
12 | #include "i915_drv.h" |
13 | #include "i915_ttm_buddy_manager.h" |
14 | #include "intel_memory_region.h" |
15 | #include "intel_region_ttm.h" |
16 | |
17 | #include "gem/i915_gem_mman.h" |
18 | #include "gem/i915_gem_object.h" |
19 | #include "gem/i915_gem_region.h" |
20 | #include "gem/i915_gem_ttm.h" |
21 | #include "gem/i915_gem_ttm_move.h" |
22 | #include "gem/i915_gem_ttm_pm.h" |
23 | #include "gt/intel_gpu_commands.h" |
24 | |
25 | #define I915_TTM_PRIO_PURGE 0 |
26 | #define I915_TTM_PRIO_NO_PAGES 1 |
27 | #define I915_TTM_PRIO_HAS_PAGES 2 |
28 | #define I915_TTM_PRIO_NEEDS_CPU_ACCESS 3 |
29 | |
30 | /* |
31 | * Size of struct ttm_place vector in on-stack struct ttm_placement allocs |
32 | */ |
33 | #define I915_TTM_MAX_PLACEMENTS INTEL_REGION_UNKNOWN |
34 | |
35 | /** |
36 | * struct i915_ttm_tt - TTM page vector with additional private information |
37 | * @ttm: The base TTM page vector. |
38 | * @dev: The struct device used for dma mapping and unmapping. |
39 | * @cached_rsgt: The cached scatter-gather table. |
40 | * @is_shmem: Set if using shmem. |
41 | * @filp: The shmem file, if using shmem backend. |
42 | * |
43 | * Note that DMA may be going on right up to the point where the page- |
44 | * vector is unpopulated in delayed destroy. Hence keep the |
45 | * scatter-gather table mapped and cached up to that point. This is |
46 | * different from the cached gem object io scatter-gather table which |
47 | * doesn't have an associated dma mapping. |
48 | */ |
49 | struct i915_ttm_tt { |
50 | struct ttm_tt ttm; |
51 | struct device *dev; |
52 | struct i915_refct_sgt cached_rsgt; |
53 | |
54 | bool is_shmem; |
55 | struct file *filp; |
56 | }; |
57 | |
58 | static const struct ttm_place sys_placement_flags = { |
59 | .fpfn = 0, |
60 | .lpfn = 0, |
61 | .mem_type = I915_PL_SYSTEM, |
62 | .flags = 0, |
63 | }; |
64 | |
65 | static struct ttm_placement i915_sys_placement = { |
66 | .num_placement = 1, |
67 | .placement = &sys_placement_flags, |
68 | }; |
69 | |
70 | /** |
71 | * i915_ttm_sys_placement - Return the struct ttm_placement to be |
72 | * used for an object in system memory. |
73 | * |
74 | * Rather than making the struct extern, use this |
75 | * function. |
76 | * |
77 | * Return: A pointer to a static variable for sys placement. |
78 | */ |
79 | struct ttm_placement *i915_ttm_sys_placement(void) |
80 | { |
81 | return &i915_sys_placement; |
82 | } |
83 | |
84 | static int i915_ttm_err_to_gem(int err) |
85 | { |
86 | /* Fastpath */ |
87 | if (likely(!err)) |
88 | return 0; |
89 | |
90 | switch (err) { |
91 | case -EBUSY: |
92 | /* |
93 | * TTM likes to convert -EDEADLK to -EBUSY, and wants us to |
94 | * restart the operation, since we don't record the contending |
95 | * lock. We use -EAGAIN to restart. |
96 | */ |
97 | return -EAGAIN; |
98 | case -ENOSPC: |
99 | /* |
100 | * Memory type / region is full, and we can't evict. |
101 | * Except possibly system, that returns -ENOMEM; |
102 | */ |
103 | return -ENXIO; |
104 | default: |
105 | break; |
106 | } |
107 | |
108 | return err; |
109 | } |
110 | |
111 | static enum ttm_caching |
112 | i915_ttm_select_tt_caching(const struct drm_i915_gem_object *obj) |
113 | { |
114 | /* |
115 | * Objects only allowed in system get cached cpu-mappings, or when |
116 | * evicting lmem-only buffers to system for swapping. Other objects get |
117 | * WC mapping for now. Even if in system. |
118 | */ |
119 | if (obj->mm.n_placements <= 1) |
120 | return ttm_cached; |
121 | |
122 | return ttm_write_combined; |
123 | } |
124 | |
125 | static void |
126 | i915_ttm_place_from_region(const struct intel_memory_region *mr, |
127 | struct ttm_place *place, |
128 | resource_size_t offset, |
129 | resource_size_t size, |
130 | unsigned int flags) |
131 | { |
132 | memset(place, 0, sizeof(*place)); |
133 | place->mem_type = intel_region_to_ttm_type(mem: mr); |
134 | |
135 | if (mr->type == INTEL_MEMORY_SYSTEM) |
136 | return; |
137 | |
138 | if (flags & I915_BO_ALLOC_CONTIGUOUS) |
139 | place->flags |= TTM_PL_FLAG_CONTIGUOUS; |
140 | if (offset != I915_BO_INVALID_OFFSET) { |
141 | WARN_ON(overflows_type(offset >> PAGE_SHIFT, place->fpfn)); |
142 | place->fpfn = offset >> PAGE_SHIFT; |
143 | WARN_ON(overflows_type(place->fpfn + (size >> PAGE_SHIFT), place->lpfn)); |
144 | place->lpfn = place->fpfn + (size >> PAGE_SHIFT); |
145 | } else if (resource_size(res: &mr->io) && resource_size(res: &mr->io) < mr->total) { |
146 | if (flags & I915_BO_ALLOC_GPU_ONLY) { |
147 | place->flags |= TTM_PL_FLAG_TOPDOWN; |
148 | } else { |
149 | place->fpfn = 0; |
150 | WARN_ON(overflows_type(resource_size(&mr->io) >> PAGE_SHIFT, place->lpfn)); |
151 | place->lpfn = resource_size(res: &mr->io) >> PAGE_SHIFT; |
152 | } |
153 | } |
154 | } |
155 | |
156 | static void |
157 | i915_ttm_placement_from_obj(const struct drm_i915_gem_object *obj, |
158 | struct ttm_place *places, |
159 | struct ttm_placement *placement) |
160 | { |
161 | unsigned int num_allowed = obj->mm.n_placements; |
162 | unsigned int flags = obj->flags; |
163 | unsigned int i; |
164 | |
165 | i915_ttm_place_from_region(mr: num_allowed ? obj->mm.placements[0] : |
166 | obj->mm.region, place: &places[0], offset: obj->bo_offset, |
167 | size: obj->base.size, flags); |
168 | places[0].flags |= TTM_PL_FLAG_DESIRED; |
169 | |
170 | /* Cache this on object? */ |
171 | for (i = 0; i < num_allowed; ++i) { |
172 | i915_ttm_place_from_region(mr: obj->mm.placements[i], |
173 | place: &places[i + 1], offset: obj->bo_offset, |
174 | size: obj->base.size, flags); |
175 | places[i + 1].flags |= TTM_PL_FLAG_FALLBACK; |
176 | } |
177 | |
178 | placement->num_placement = num_allowed + 1; |
179 | placement->placement = places; |
180 | } |
181 | |
182 | static int i915_ttm_tt_shmem_populate(struct ttm_device *bdev, |
183 | struct ttm_tt *ttm, |
184 | struct ttm_operation_ctx *ctx) |
185 | { |
186 | struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); |
187 | struct intel_memory_region *mr = i915->mm.regions[INTEL_MEMORY_SYSTEM]; |
188 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); |
189 | const unsigned int max_segment = i915_sg_segment_size(dev: i915->drm.dev); |
190 | const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT; |
191 | struct file *filp = i915_tt->filp; |
192 | struct sgt_iter sgt_iter; |
193 | struct sg_table *st; |
194 | struct page *page; |
195 | unsigned long i; |
196 | int err; |
197 | |
198 | if (!filp) { |
199 | struct address_space *mapping; |
200 | gfp_t mask; |
201 | |
202 | filp = shmem_file_setup(name: "i915-shmem-tt" , size, VM_NORESERVE); |
203 | if (IS_ERR(ptr: filp)) |
204 | return PTR_ERR(ptr: filp); |
205 | |
206 | mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; |
207 | |
208 | mapping = filp->f_mapping; |
209 | mapping_set_gfp_mask(m: mapping, mask); |
210 | GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM)); |
211 | |
212 | i915_tt->filp = filp; |
213 | } |
214 | |
215 | st = &i915_tt->cached_rsgt.table; |
216 | err = shmem_sg_alloc_table(i915, st, size, mr, mapping: filp->f_mapping, |
217 | max_segment); |
218 | if (err) |
219 | return err; |
220 | |
221 | err = dma_map_sgtable(dev: i915_tt->dev, sgt: st, dir: DMA_BIDIRECTIONAL, |
222 | DMA_ATTR_SKIP_CPU_SYNC); |
223 | if (err) |
224 | goto err_free_st; |
225 | |
226 | i = 0; |
227 | for_each_sgt_page(page, sgt_iter, st) |
228 | ttm->pages[i++] = page; |
229 | |
230 | if (ttm->page_flags & TTM_TT_FLAG_SWAPPED) |
231 | ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; |
232 | |
233 | return 0; |
234 | |
235 | err_free_st: |
236 | shmem_sg_free_table(st, mapping: filp->f_mapping, dirty: false, backup: false); |
237 | |
238 | return err; |
239 | } |
240 | |
241 | static void i915_ttm_tt_shmem_unpopulate(struct ttm_tt *ttm) |
242 | { |
243 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); |
244 | bool backup = ttm->page_flags & TTM_TT_FLAG_SWAPPED; |
245 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
246 | |
247 | shmem_sg_free_table(st, mapping: file_inode(f: i915_tt->filp)->i_mapping, |
248 | dirty: backup, backup); |
249 | } |
250 | |
251 | static void i915_ttm_tt_release(struct kref *ref) |
252 | { |
253 | struct i915_ttm_tt *i915_tt = |
254 | container_of(ref, typeof(*i915_tt), cached_rsgt.kref); |
255 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
256 | |
257 | GEM_WARN_ON(st->sgl); |
258 | |
259 | kfree(objp: i915_tt); |
260 | } |
261 | |
262 | static const struct i915_refct_sgt_ops tt_rsgt_ops = { |
263 | .release = i915_ttm_tt_release |
264 | }; |
265 | |
266 | static struct ttm_tt *i915_ttm_tt_create(struct ttm_buffer_object *bo, |
267 | uint32_t page_flags) |
268 | { |
269 | struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915), |
270 | bdev); |
271 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
272 | unsigned long ccs_pages = 0; |
273 | enum ttm_caching caching; |
274 | struct i915_ttm_tt *i915_tt; |
275 | int ret; |
276 | |
277 | if (i915_ttm_is_ghost_object(bo)) |
278 | return NULL; |
279 | |
280 | i915_tt = kzalloc(size: sizeof(*i915_tt), GFP_KERNEL); |
281 | if (!i915_tt) |
282 | return NULL; |
283 | |
284 | if (obj->flags & I915_BO_ALLOC_CPU_CLEAR && (!bo->resource || |
285 | ttm_manager_type(bdev: bo->bdev, mem_type: bo->resource->mem_type)->use_tt)) |
286 | page_flags |= TTM_TT_FLAG_ZERO_ALLOC; |
287 | |
288 | caching = i915_ttm_select_tt_caching(obj); |
289 | if (i915_gem_object_is_shrinkable(obj) && caching == ttm_cached) { |
290 | page_flags |= TTM_TT_FLAG_EXTERNAL | |
291 | TTM_TT_FLAG_EXTERNAL_MAPPABLE; |
292 | i915_tt->is_shmem = true; |
293 | } |
294 | |
295 | if (i915_gem_object_needs_ccs_pages(obj)) |
296 | ccs_pages = DIV_ROUND_UP(DIV_ROUND_UP(bo->base.size, |
297 | NUM_BYTES_PER_CCS_BYTE), |
298 | PAGE_SIZE); |
299 | |
300 | ret = ttm_tt_init(ttm: &i915_tt->ttm, bo, page_flags, caching, extra_pages: ccs_pages); |
301 | if (ret) |
302 | goto err_free; |
303 | |
304 | __i915_refct_sgt_init(rsgt: &i915_tt->cached_rsgt, size: bo->base.size, |
305 | ops: &tt_rsgt_ops); |
306 | |
307 | i915_tt->dev = obj->base.dev->dev; |
308 | |
309 | return &i915_tt->ttm; |
310 | |
311 | err_free: |
312 | kfree(objp: i915_tt); |
313 | return NULL; |
314 | } |
315 | |
316 | static int i915_ttm_tt_populate(struct ttm_device *bdev, |
317 | struct ttm_tt *ttm, |
318 | struct ttm_operation_ctx *ctx) |
319 | { |
320 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); |
321 | |
322 | if (i915_tt->is_shmem) |
323 | return i915_ttm_tt_shmem_populate(bdev, ttm, ctx); |
324 | |
325 | return ttm_pool_alloc(pool: &bdev->pool, tt: ttm, ctx); |
326 | } |
327 | |
328 | static void i915_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) |
329 | { |
330 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); |
331 | struct sg_table *st = &i915_tt->cached_rsgt.table; |
332 | |
333 | if (st->sgl) |
334 | dma_unmap_sgtable(dev: i915_tt->dev, sgt: st, dir: DMA_BIDIRECTIONAL, attrs: 0); |
335 | |
336 | if (i915_tt->is_shmem) { |
337 | i915_ttm_tt_shmem_unpopulate(ttm); |
338 | } else { |
339 | sg_free_table(st); |
340 | ttm_pool_free(pool: &bdev->pool, tt: ttm); |
341 | } |
342 | } |
343 | |
344 | static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) |
345 | { |
346 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); |
347 | |
348 | if (i915_tt->filp) |
349 | fput(i915_tt->filp); |
350 | |
351 | ttm_tt_fini(ttm); |
352 | i915_refct_sgt_put(rsgt: &i915_tt->cached_rsgt); |
353 | } |
354 | |
355 | static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo, |
356 | const struct ttm_place *place) |
357 | { |
358 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
359 | |
360 | if (i915_ttm_is_ghost_object(bo)) |
361 | return false; |
362 | |
363 | /* |
364 | * EXTERNAL objects should never be swapped out by TTM, instead we need |
365 | * to handle that ourselves. TTM will already skip such objects for us, |
366 | * but we would like to avoid grabbing locks for no good reason. |
367 | */ |
368 | if (bo->ttm && bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL) |
369 | return false; |
370 | |
371 | /* Will do for now. Our pinned objects are still on TTM's LRU lists */ |
372 | if (!i915_gem_object_evictable(obj)) |
373 | return false; |
374 | |
375 | return ttm_bo_eviction_valuable(bo, place); |
376 | } |
377 | |
378 | static void i915_ttm_evict_flags(struct ttm_buffer_object *bo, |
379 | struct ttm_placement *placement) |
380 | { |
381 | *placement = i915_sys_placement; |
382 | } |
383 | |
384 | /** |
385 | * i915_ttm_free_cached_io_rsgt - Free object cached LMEM information |
386 | * @obj: The GEM object |
387 | * This function frees any LMEM-related information that is cached on |
388 | * the object. For example the radix tree for fast page lookup and the |
389 | * cached refcounted sg-table |
390 | */ |
391 | void i915_ttm_free_cached_io_rsgt(struct drm_i915_gem_object *obj) |
392 | { |
393 | struct radix_tree_iter iter; |
394 | void __rcu **slot; |
395 | |
396 | if (!obj->ttm.cached_io_rsgt) |
397 | return; |
398 | |
399 | rcu_read_lock(); |
400 | radix_tree_for_each_slot(slot, &obj->ttm.get_io_page.radix, &iter, 0) |
401 | radix_tree_delete(&obj->ttm.get_io_page.radix, iter.index); |
402 | rcu_read_unlock(); |
403 | |
404 | i915_refct_sgt_put(rsgt: obj->ttm.cached_io_rsgt); |
405 | obj->ttm.cached_io_rsgt = NULL; |
406 | } |
407 | |
408 | /** |
409 | * i915_ttm_purge - Clear an object of its memory |
410 | * @obj: The object |
411 | * |
412 | * This function is called to clear an object of it's memory when it is |
413 | * marked as not needed anymore. |
414 | * |
415 | * Return: 0 on success, negative error code on failure. |
416 | */ |
417 | int i915_ttm_purge(struct drm_i915_gem_object *obj) |
418 | { |
419 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
420 | struct i915_ttm_tt *i915_tt = |
421 | container_of(bo->ttm, typeof(*i915_tt), ttm); |
422 | struct ttm_operation_ctx ctx = { |
423 | .interruptible = true, |
424 | .no_wait_gpu = false, |
425 | }; |
426 | struct ttm_placement place = {}; |
427 | int ret; |
428 | |
429 | if (obj->mm.madv == __I915_MADV_PURGED) |
430 | return 0; |
431 | |
432 | ret = ttm_bo_validate(bo, placement: &place, ctx: &ctx); |
433 | if (ret) |
434 | return ret; |
435 | |
436 | if (bo->ttm && i915_tt->filp) { |
437 | /* |
438 | * The below fput(which eventually calls shmem_truncate) might |
439 | * be delayed by worker, so when directly called to purge the |
440 | * pages(like by the shrinker) we should try to be more |
441 | * aggressive and release the pages immediately. |
442 | */ |
443 | shmem_truncate_range(inode: file_inode(f: i915_tt->filp), |
444 | start: 0, end: (loff_t)-1); |
445 | fput(fetch_and_zero(&i915_tt->filp)); |
446 | } |
447 | |
448 | obj->write_domain = 0; |
449 | obj->read_domains = 0; |
450 | i915_ttm_adjust_gem_after_move(obj); |
451 | i915_ttm_free_cached_io_rsgt(obj); |
452 | obj->mm.madv = __I915_MADV_PURGED; |
453 | |
454 | return 0; |
455 | } |
456 | |
457 | static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags) |
458 | { |
459 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
460 | struct i915_ttm_tt *i915_tt = |
461 | container_of(bo->ttm, typeof(*i915_tt), ttm); |
462 | struct ttm_operation_ctx ctx = { |
463 | .interruptible = true, |
464 | .no_wait_gpu = flags & I915_GEM_OBJECT_SHRINK_NO_GPU_WAIT, |
465 | }; |
466 | struct ttm_placement place = {}; |
467 | int ret; |
468 | |
469 | if (!bo->ttm || i915_ttm_cpu_maps_iomem(mem: bo->resource)) |
470 | return 0; |
471 | |
472 | GEM_BUG_ON(!i915_tt->is_shmem); |
473 | |
474 | if (!i915_tt->filp) |
475 | return 0; |
476 | |
477 | ret = ttm_bo_wait_ctx(bo, ctx: &ctx); |
478 | if (ret) |
479 | return ret; |
480 | |
481 | switch (obj->mm.madv) { |
482 | case I915_MADV_DONTNEED: |
483 | return i915_ttm_purge(obj); |
484 | case __I915_MADV_PURGED: |
485 | return 0; |
486 | } |
487 | |
488 | if (bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) |
489 | return 0; |
490 | |
491 | bo->ttm->page_flags |= TTM_TT_FLAG_SWAPPED; |
492 | ret = ttm_bo_validate(bo, placement: &place, ctx: &ctx); |
493 | if (ret) { |
494 | bo->ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; |
495 | return ret; |
496 | } |
497 | |
498 | if (flags & I915_GEM_OBJECT_SHRINK_WRITEBACK) |
499 | __shmem_writeback(size: obj->base.size, mapping: i915_tt->filp->f_mapping); |
500 | |
501 | return 0; |
502 | } |
503 | |
504 | static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) |
505 | { |
506 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
507 | |
508 | /* |
509 | * This gets called twice by ttm, so long as we have a ttm resource or |
510 | * ttm_tt then we can still safely call this. Due to pipeline-gutting, |
511 | * we maybe have NULL bo->resource, but in that case we should always |
512 | * have a ttm alive (like if the pages are swapped out). |
513 | */ |
514 | if ((bo->resource || bo->ttm) && !i915_ttm_is_ghost_object(bo)) { |
515 | __i915_gem_object_pages_fini(obj); |
516 | i915_ttm_free_cached_io_rsgt(obj); |
517 | } |
518 | } |
519 | |
520 | static struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm) |
521 | { |
522 | struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); |
523 | struct sg_table *st; |
524 | int ret; |
525 | |
526 | if (i915_tt->cached_rsgt.table.sgl) |
527 | return i915_refct_sgt_get(rsgt: &i915_tt->cached_rsgt); |
528 | |
529 | st = &i915_tt->cached_rsgt.table; |
530 | ret = sg_alloc_table_from_pages_segment(sgt: st, |
531 | pages: ttm->pages, n_pages: ttm->num_pages, |
532 | offset: 0, size: (unsigned long)ttm->num_pages << PAGE_SHIFT, |
533 | max_segment: i915_sg_segment_size(dev: i915_tt->dev), GFP_KERNEL); |
534 | if (ret) { |
535 | st->sgl = NULL; |
536 | return ERR_PTR(error: ret); |
537 | } |
538 | |
539 | ret = dma_map_sgtable(dev: i915_tt->dev, sgt: st, dir: DMA_BIDIRECTIONAL, attrs: 0); |
540 | if (ret) { |
541 | sg_free_table(st); |
542 | return ERR_PTR(error: ret); |
543 | } |
544 | |
545 | return i915_refct_sgt_get(rsgt: &i915_tt->cached_rsgt); |
546 | } |
547 | |
548 | /** |
549 | * i915_ttm_resource_get_st - Get a refcounted sg-table pointing to the |
550 | * resource memory |
551 | * @obj: The GEM object used for sg-table caching |
552 | * @res: The struct ttm_resource for which an sg-table is requested. |
553 | * |
554 | * This function returns a refcounted sg-table representing the memory |
555 | * pointed to by @res. If @res is the object's current resource it may also |
556 | * cache the sg_table on the object or attempt to access an already cached |
557 | * sg-table. The refcounted sg-table needs to be put when no-longer in use. |
558 | * |
559 | * Return: A valid pointer to a struct i915_refct_sgt or error pointer on |
560 | * failure. |
561 | */ |
562 | struct i915_refct_sgt * |
563 | i915_ttm_resource_get_st(struct drm_i915_gem_object *obj, |
564 | struct ttm_resource *res) |
565 | { |
566 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
567 | u32 page_alignment; |
568 | |
569 | if (!i915_ttm_gtt_binds_lmem(mem: res)) |
570 | return i915_ttm_tt_get_st(ttm: bo->ttm); |
571 | |
572 | page_alignment = bo->page_alignment << PAGE_SHIFT; |
573 | if (!page_alignment) |
574 | page_alignment = obj->mm.region->min_page_size; |
575 | |
576 | /* |
577 | * If CPU mapping differs, we need to add the ttm_tt pages to |
578 | * the resulting st. Might make sense for GGTT. |
579 | */ |
580 | GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(res)); |
581 | if (bo->resource == res) { |
582 | if (!obj->ttm.cached_io_rsgt) { |
583 | struct i915_refct_sgt *rsgt; |
584 | |
585 | rsgt = intel_region_ttm_resource_to_rsgt(mem: obj->mm.region, |
586 | res, |
587 | page_alignment); |
588 | if (IS_ERR(ptr: rsgt)) |
589 | return rsgt; |
590 | |
591 | obj->ttm.cached_io_rsgt = rsgt; |
592 | } |
593 | return i915_refct_sgt_get(rsgt: obj->ttm.cached_io_rsgt); |
594 | } |
595 | |
596 | return intel_region_ttm_resource_to_rsgt(mem: obj->mm.region, res, |
597 | page_alignment); |
598 | } |
599 | |
600 | static int i915_ttm_truncate(struct drm_i915_gem_object *obj) |
601 | { |
602 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
603 | long err; |
604 | |
605 | WARN_ON_ONCE(obj->mm.madv == I915_MADV_WILLNEED); |
606 | |
607 | err = dma_resv_wait_timeout(obj: bo->base.resv, usage: DMA_RESV_USAGE_BOOKKEEP, |
608 | intr: true, timeout: 15 * HZ); |
609 | if (err < 0) |
610 | return err; |
611 | if (err == 0) |
612 | return -EBUSY; |
613 | |
614 | err = i915_ttm_move_notify(bo); |
615 | if (err) |
616 | return err; |
617 | |
618 | return i915_ttm_purge(obj); |
619 | } |
620 | |
621 | static void i915_ttm_swap_notify(struct ttm_buffer_object *bo) |
622 | { |
623 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
624 | int ret; |
625 | |
626 | if (i915_ttm_is_ghost_object(bo)) |
627 | return; |
628 | |
629 | ret = i915_ttm_move_notify(bo); |
630 | GEM_WARN_ON(ret); |
631 | GEM_WARN_ON(obj->ttm.cached_io_rsgt); |
632 | if (!ret && obj->mm.madv != I915_MADV_WILLNEED) |
633 | i915_ttm_purge(obj); |
634 | } |
635 | |
636 | /** |
637 | * i915_ttm_resource_mappable - Return true if the ttm resource is CPU |
638 | * accessible. |
639 | * @res: The TTM resource to check. |
640 | * |
641 | * This is interesting on small-BAR systems where we may encounter lmem objects |
642 | * that can't be accessed via the CPU. |
643 | */ |
644 | bool i915_ttm_resource_mappable(struct ttm_resource *res) |
645 | { |
646 | struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); |
647 | |
648 | if (!i915_ttm_cpu_maps_iomem(mem: res)) |
649 | return true; |
650 | |
651 | return bman_res->used_visible_size == PFN_UP(bman_res->base.size); |
652 | } |
653 | |
654 | static int i915_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) |
655 | { |
656 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo: mem->bo); |
657 | bool unknown_state; |
658 | |
659 | if (i915_ttm_is_ghost_object(bo: mem->bo)) |
660 | return -EINVAL; |
661 | |
662 | if (!kref_get_unless_zero(kref: &obj->base.refcount)) |
663 | return -EINVAL; |
664 | |
665 | assert_object_held(obj); |
666 | |
667 | unknown_state = i915_gem_object_has_unknown_state(obj); |
668 | i915_gem_object_put(obj); |
669 | if (unknown_state) |
670 | return -EINVAL; |
671 | |
672 | if (!i915_ttm_cpu_maps_iomem(mem)) |
673 | return 0; |
674 | |
675 | if (!i915_ttm_resource_mappable(res: mem)) |
676 | return -EINVAL; |
677 | |
678 | mem->bus.caching = ttm_write_combined; |
679 | mem->bus.is_iomem = true; |
680 | |
681 | return 0; |
682 | } |
683 | |
684 | static unsigned long i915_ttm_io_mem_pfn(struct ttm_buffer_object *bo, |
685 | unsigned long page_offset) |
686 | { |
687 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
688 | struct scatterlist *sg; |
689 | unsigned long base; |
690 | unsigned int ofs; |
691 | |
692 | GEM_BUG_ON(i915_ttm_is_ghost_object(bo)); |
693 | GEM_WARN_ON(bo->ttm); |
694 | |
695 | base = obj->mm.region->iomap.base - obj->mm.region->region.start; |
696 | sg = i915_gem_object_page_iter_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs); |
697 | |
698 | return ((base + sg_dma_address(sg)) >> PAGE_SHIFT) + ofs; |
699 | } |
700 | |
701 | static int i915_ttm_access_memory(struct ttm_buffer_object *bo, |
702 | unsigned long offset, void *buf, |
703 | int len, int write) |
704 | { |
705 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
706 | resource_size_t iomap = obj->mm.region->iomap.base - |
707 | obj->mm.region->region.start; |
708 | unsigned long page = offset >> PAGE_SHIFT; |
709 | unsigned long bytes_left = len; |
710 | |
711 | /* |
712 | * TODO: For now just let it fail if the resource is non-mappable, |
713 | * otherwise we need to perform the memcpy from the gpu here, without |
714 | * interfering with the object (like moving the entire thing). |
715 | */ |
716 | if (!i915_ttm_resource_mappable(res: bo->resource)) |
717 | return -EIO; |
718 | |
719 | offset -= page << PAGE_SHIFT; |
720 | do { |
721 | unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); |
722 | void __iomem *ptr; |
723 | dma_addr_t daddr; |
724 | |
725 | daddr = i915_gem_object_get_dma_address(obj, page); |
726 | ptr = ioremap_wc(offset: iomap + daddr + offset, size: bytes); |
727 | if (!ptr) |
728 | return -EIO; |
729 | |
730 | if (write) |
731 | memcpy_toio(ptr, buf, bytes); |
732 | else |
733 | memcpy_fromio(buf, ptr, bytes); |
734 | iounmap(addr: ptr); |
735 | |
736 | page++; |
737 | buf += bytes; |
738 | bytes_left -= bytes; |
739 | offset = 0; |
740 | } while (bytes_left); |
741 | |
742 | return len; |
743 | } |
744 | |
745 | /* |
746 | * All callbacks need to take care not to downcast a struct ttm_buffer_object |
747 | * without checking its subclass, since it might be a TTM ghost object. |
748 | */ |
749 | static struct ttm_device_funcs i915_ttm_bo_driver = { |
750 | .ttm_tt_create = i915_ttm_tt_create, |
751 | .ttm_tt_populate = i915_ttm_tt_populate, |
752 | .ttm_tt_unpopulate = i915_ttm_tt_unpopulate, |
753 | .ttm_tt_destroy = i915_ttm_tt_destroy, |
754 | .eviction_valuable = i915_ttm_eviction_valuable, |
755 | .evict_flags = i915_ttm_evict_flags, |
756 | .move = i915_ttm_move, |
757 | .swap_notify = i915_ttm_swap_notify, |
758 | .delete_mem_notify = i915_ttm_delete_mem_notify, |
759 | .io_mem_reserve = i915_ttm_io_mem_reserve, |
760 | .io_mem_pfn = i915_ttm_io_mem_pfn, |
761 | .access_memory = i915_ttm_access_memory, |
762 | }; |
763 | |
764 | /** |
765 | * i915_ttm_driver - Return a pointer to the TTM device funcs |
766 | * |
767 | * Return: Pointer to statically allocated TTM device funcs. |
768 | */ |
769 | struct ttm_device_funcs *i915_ttm_driver(void) |
770 | { |
771 | return &i915_ttm_bo_driver; |
772 | } |
773 | |
774 | static int __i915_ttm_get_pages(struct drm_i915_gem_object *obj, |
775 | struct ttm_placement *placement) |
776 | { |
777 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
778 | struct ttm_operation_ctx ctx = { |
779 | .interruptible = true, |
780 | .no_wait_gpu = false, |
781 | }; |
782 | int real_num_busy; |
783 | int ret; |
784 | |
785 | /* First try only the requested placement. No eviction. */ |
786 | real_num_busy = placement->num_placement; |
787 | placement->num_placement = 1; |
788 | ret = ttm_bo_validate(bo, placement, ctx: &ctx); |
789 | if (ret) { |
790 | ret = i915_ttm_err_to_gem(err: ret); |
791 | /* |
792 | * Anything that wants to restart the operation gets to |
793 | * do that. |
794 | */ |
795 | if (ret == -EDEADLK || ret == -EINTR || ret == -ERESTARTSYS || |
796 | ret == -EAGAIN) |
797 | return ret; |
798 | |
799 | /* |
800 | * If the initial attempt fails, allow all accepted placements, |
801 | * evicting if necessary. |
802 | */ |
803 | placement->num_placement = real_num_busy; |
804 | ret = ttm_bo_validate(bo, placement, ctx: &ctx); |
805 | if (ret) |
806 | return i915_ttm_err_to_gem(err: ret); |
807 | } |
808 | |
809 | if (bo->ttm && !ttm_tt_is_populated(tt: bo->ttm)) { |
810 | ret = ttm_tt_populate(bdev: bo->bdev, ttm: bo->ttm, ctx: &ctx); |
811 | if (ret) |
812 | return ret; |
813 | |
814 | i915_ttm_adjust_domains_after_move(obj); |
815 | i915_ttm_adjust_gem_after_move(obj); |
816 | } |
817 | |
818 | if (!i915_gem_object_has_pages(obj)) { |
819 | struct i915_refct_sgt *rsgt = |
820 | i915_ttm_resource_get_st(obj, res: bo->resource); |
821 | |
822 | if (IS_ERR(ptr: rsgt)) |
823 | return PTR_ERR(ptr: rsgt); |
824 | |
825 | GEM_BUG_ON(obj->mm.rsgt); |
826 | obj->mm.rsgt = rsgt; |
827 | __i915_gem_object_set_pages(obj, pages: &rsgt->table); |
828 | } |
829 | |
830 | GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages)); |
831 | i915_ttm_adjust_lru(obj); |
832 | return ret; |
833 | } |
834 | |
835 | static int i915_ttm_get_pages(struct drm_i915_gem_object *obj) |
836 | { |
837 | struct ttm_place places[I915_TTM_MAX_PLACEMENTS + 1]; |
838 | struct ttm_placement placement; |
839 | |
840 | /* restricted by sg_alloc_table */ |
841 | if (overflows_type(obj->base.size >> PAGE_SHIFT, unsigned int)) |
842 | return -E2BIG; |
843 | |
844 | GEM_BUG_ON(obj->mm.n_placements > I915_TTM_MAX_PLACEMENTS); |
845 | |
846 | /* Move to the requested placement. */ |
847 | i915_ttm_placement_from_obj(obj, places, placement: &placement); |
848 | |
849 | return __i915_ttm_get_pages(obj, placement: &placement); |
850 | } |
851 | |
852 | /** |
853 | * DOC: Migration vs eviction |
854 | * |
855 | * GEM migration may not be the same as TTM migration / eviction. If |
856 | * the TTM core decides to evict an object it may be evicted to a |
857 | * TTM memory type that is not in the object's allowable GEM regions, or |
858 | * in fact theoretically to a TTM memory type that doesn't correspond to |
859 | * a GEM memory region. In that case the object's GEM region is not |
860 | * updated, and the data is migrated back to the GEM region at |
861 | * get_pages time. TTM may however set up CPU ptes to the object even |
862 | * when it is evicted. |
863 | * Gem forced migration using the i915_ttm_migrate() op, is allowed even |
864 | * to regions that are not in the object's list of allowable placements. |
865 | */ |
866 | static int __i915_ttm_migrate(struct drm_i915_gem_object *obj, |
867 | struct intel_memory_region *mr, |
868 | unsigned int flags) |
869 | { |
870 | struct ttm_place requested; |
871 | struct ttm_placement placement; |
872 | int ret; |
873 | |
874 | i915_ttm_place_from_region(mr, place: &requested, offset: obj->bo_offset, |
875 | size: obj->base.size, flags); |
876 | placement.num_placement = 1; |
877 | placement.placement = &requested; |
878 | |
879 | ret = __i915_ttm_get_pages(obj, placement: &placement); |
880 | if (ret) |
881 | return ret; |
882 | |
883 | /* |
884 | * Reinitialize the region bindings. This is primarily |
885 | * required for objects where the new region is not in |
886 | * its allowable placements. |
887 | */ |
888 | if (obj->mm.region != mr) { |
889 | i915_gem_object_release_memory_region(obj); |
890 | i915_gem_object_init_memory_region(obj, mem: mr); |
891 | } |
892 | |
893 | return 0; |
894 | } |
895 | |
896 | static int i915_ttm_migrate(struct drm_i915_gem_object *obj, |
897 | struct intel_memory_region *mr, |
898 | unsigned int flags) |
899 | { |
900 | return __i915_ttm_migrate(obj, mr, flags); |
901 | } |
902 | |
903 | static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, |
904 | struct sg_table *st) |
905 | { |
906 | /* |
907 | * We're currently not called from a shrinker, so put_pages() |
908 | * typically means the object is about to destroyed, or called |
909 | * from move_notify(). So just avoid doing much for now. |
910 | * If the object is not destroyed next, The TTM eviction logic |
911 | * and shrinkers will move it out if needed. |
912 | */ |
913 | |
914 | if (obj->mm.rsgt) |
915 | i915_refct_sgt_put(fetch_and_zero(&obj->mm.rsgt)); |
916 | } |
917 | |
918 | /** |
919 | * i915_ttm_adjust_lru - Adjust an object's position on relevant LRU lists. |
920 | * @obj: The object |
921 | */ |
922 | void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj) |
923 | { |
924 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
925 | struct i915_ttm_tt *i915_tt = |
926 | container_of(bo->ttm, typeof(*i915_tt), ttm); |
927 | bool shrinkable = |
928 | bo->ttm && i915_tt->filp && ttm_tt_is_populated(tt: bo->ttm); |
929 | |
930 | /* |
931 | * Don't manipulate the TTM LRUs while in TTM bo destruction. |
932 | * We're called through i915_ttm_delete_mem_notify(). |
933 | */ |
934 | if (!kref_read(kref: &bo->kref)) |
935 | return; |
936 | |
937 | /* |
938 | * We skip managing the shrinker LRU in set_pages() and just manage |
939 | * everything here. This does at least solve the issue with having |
940 | * temporary shmem mappings(like with evicted lmem) not being visible to |
941 | * the shrinker. Only our shmem objects are shrinkable, everything else |
942 | * we keep as unshrinkable. |
943 | * |
944 | * To make sure everything plays nice we keep an extra shrink pin in TTM |
945 | * if the underlying pages are not currently shrinkable. Once we release |
946 | * our pin, like when the pages are moved to shmem, the pages will then |
947 | * be added to the shrinker LRU, assuming the caller isn't also holding |
948 | * a pin. |
949 | * |
950 | * TODO: consider maybe also bumping the shrinker list here when we have |
951 | * already unpinned it, which should give us something more like an LRU. |
952 | * |
953 | * TODO: There is a small window of opportunity for this function to |
954 | * get called from eviction after we've dropped the last GEM refcount, |
955 | * but before the TTM deleted flag is set on the object. Avoid |
956 | * adjusting the shrinker list in such cases, since the object is |
957 | * not available to the shrinker anyway due to its zero refcount. |
958 | * To fix this properly we should move to a TTM shrinker LRU list for |
959 | * these objects. |
960 | */ |
961 | if (kref_get_unless_zero(kref: &obj->base.refcount)) { |
962 | if (shrinkable != obj->mm.ttm_shrinkable) { |
963 | if (shrinkable) { |
964 | if (obj->mm.madv == I915_MADV_WILLNEED) |
965 | __i915_gem_object_make_shrinkable(obj); |
966 | else |
967 | __i915_gem_object_make_purgeable(obj); |
968 | } else { |
969 | i915_gem_object_make_unshrinkable(obj); |
970 | } |
971 | |
972 | obj->mm.ttm_shrinkable = shrinkable; |
973 | } |
974 | i915_gem_object_put(obj); |
975 | } |
976 | |
977 | /* |
978 | * Put on the correct LRU list depending on the MADV status |
979 | */ |
980 | spin_lock(lock: &bo->bdev->lru_lock); |
981 | if (shrinkable) { |
982 | /* Try to keep shmem_tt from being considered for shrinking. */ |
983 | bo->priority = TTM_MAX_BO_PRIORITY - 1; |
984 | } else if (obj->mm.madv != I915_MADV_WILLNEED) { |
985 | bo->priority = I915_TTM_PRIO_PURGE; |
986 | } else if (!i915_gem_object_has_pages(obj)) { |
987 | bo->priority = I915_TTM_PRIO_NO_PAGES; |
988 | } else { |
989 | struct ttm_resource_manager *man = |
990 | ttm_manager_type(bdev: bo->bdev, mem_type: bo->resource->mem_type); |
991 | |
992 | /* |
993 | * If we need to place an LMEM resource which doesn't need CPU |
994 | * access then we should try not to victimize mappable objects |
995 | * first, since we likely end up stealing more of the mappable |
996 | * portion. And likewise when we try to find space for a mappble |
997 | * object, we know not to ever victimize objects that don't |
998 | * occupy any mappable pages. |
999 | */ |
1000 | if (i915_ttm_cpu_maps_iomem(mem: bo->resource) && |
1001 | i915_ttm_buddy_man_visible_size(man) < man->size && |
1002 | !(obj->flags & I915_BO_ALLOC_GPU_ONLY)) |
1003 | bo->priority = I915_TTM_PRIO_NEEDS_CPU_ACCESS; |
1004 | else |
1005 | bo->priority = I915_TTM_PRIO_HAS_PAGES; |
1006 | } |
1007 | |
1008 | ttm_bo_move_to_lru_tail(bo); |
1009 | spin_unlock(lock: &bo->bdev->lru_lock); |
1010 | } |
1011 | |
1012 | /* |
1013 | * TTM-backed gem object destruction requires some clarification. |
1014 | * Basically we have two possibilities here. We can either rely on the |
1015 | * i915 delayed destruction and put the TTM object when the object |
1016 | * is idle. This would be detected by TTM which would bypass the |
1017 | * TTM delayed destroy handling. The other approach is to put the TTM |
1018 | * object early and rely on the TTM destroyed handling, and then free |
1019 | * the leftover parts of the GEM object once TTM's destroyed list handling is |
1020 | * complete. For now, we rely on the latter for two reasons: |
1021 | * a) TTM can evict an object even when it's on the delayed destroy list, |
1022 | * which in theory allows for complete eviction. |
1023 | * b) There is work going on in TTM to allow freeing an object even when |
1024 | * it's not idle, and using the TTM destroyed list handling could help us |
1025 | * benefit from that. |
1026 | */ |
1027 | static void i915_ttm_delayed_free(struct drm_i915_gem_object *obj) |
1028 | { |
1029 | GEM_BUG_ON(!obj->ttm.created); |
1030 | |
1031 | ttm_bo_put(bo: i915_gem_to_ttm(obj)); |
1032 | } |
1033 | |
1034 | static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) |
1035 | { |
1036 | struct vm_area_struct *area = vmf->vma; |
1037 | struct ttm_buffer_object *bo = area->vm_private_data; |
1038 | struct drm_device *dev = bo->base.dev; |
1039 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
1040 | intel_wakeref_t wakeref = 0; |
1041 | vm_fault_t ret; |
1042 | int idx; |
1043 | |
1044 | /* Sanity check that we allow writing into this object */ |
1045 | if (unlikely(i915_gem_object_is_readonly(obj) && |
1046 | area->vm_flags & VM_WRITE)) |
1047 | return VM_FAULT_SIGBUS; |
1048 | |
1049 | ret = ttm_bo_vm_reserve(bo, vmf); |
1050 | if (ret) |
1051 | return ret; |
1052 | |
1053 | if (obj->mm.madv != I915_MADV_WILLNEED) { |
1054 | dma_resv_unlock(obj: bo->base.resv); |
1055 | return VM_FAULT_SIGBUS; |
1056 | } |
1057 | |
1058 | /* |
1059 | * This must be swapped out with shmem ttm_tt (pipeline-gutting). |
1060 | * Calling ttm_bo_validate() here with TTM_PL_SYSTEM should only go as |
1061 | * far as far doing a ttm_bo_move_null(), which should skip all the |
1062 | * other junk. |
1063 | */ |
1064 | if (!bo->resource) { |
1065 | struct ttm_operation_ctx ctx = { |
1066 | .interruptible = true, |
1067 | .no_wait_gpu = true, /* should be idle already */ |
1068 | }; |
1069 | int err; |
1070 | |
1071 | GEM_BUG_ON(!bo->ttm || !(bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED)); |
1072 | |
1073 | err = ttm_bo_validate(bo, placement: i915_ttm_sys_placement(), ctx: &ctx); |
1074 | if (err) { |
1075 | dma_resv_unlock(obj: bo->base.resv); |
1076 | return VM_FAULT_SIGBUS; |
1077 | } |
1078 | } else if (!i915_ttm_resource_mappable(res: bo->resource)) { |
1079 | int err = -ENODEV; |
1080 | int i; |
1081 | |
1082 | for (i = 0; i < obj->mm.n_placements; i++) { |
1083 | struct intel_memory_region *mr = obj->mm.placements[i]; |
1084 | unsigned int flags; |
1085 | |
1086 | if (!resource_size(res: &mr->io) && mr->type != INTEL_MEMORY_SYSTEM) |
1087 | continue; |
1088 | |
1089 | flags = obj->flags; |
1090 | flags &= ~I915_BO_ALLOC_GPU_ONLY; |
1091 | err = __i915_ttm_migrate(obj, mr, flags); |
1092 | if (!err) |
1093 | break; |
1094 | } |
1095 | |
1096 | if (err) { |
1097 | drm_dbg_ratelimited(dev, |
1098 | "Unable to make resource CPU accessible(err = %pe)\n" , |
1099 | ERR_PTR(err)); |
1100 | dma_resv_unlock(obj: bo->base.resv); |
1101 | ret = VM_FAULT_SIGBUS; |
1102 | goto out_rpm; |
1103 | } |
1104 | } |
1105 | |
1106 | if (i915_ttm_cpu_maps_iomem(mem: bo->resource)) |
1107 | wakeref = intel_runtime_pm_get(rpm: &to_i915(dev: obj->base.dev)->runtime_pm); |
1108 | |
1109 | if (drm_dev_enter(dev, idx: &idx)) { |
1110 | ret = ttm_bo_vm_fault_reserved(vmf, prot: vmf->vma->vm_page_prot, |
1111 | TTM_BO_VM_NUM_PREFAULT); |
1112 | drm_dev_exit(idx); |
1113 | } else { |
1114 | ret = ttm_bo_vm_dummy_page(vmf, prot: vmf->vma->vm_page_prot); |
1115 | } |
1116 | |
1117 | if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) |
1118 | goto out_rpm; |
1119 | |
1120 | /* |
1121 | * ttm_bo_vm_reserve() already has dma_resv_lock. |
1122 | * userfault_count is protected by dma_resv lock and rpm wakeref. |
1123 | */ |
1124 | if (ret == VM_FAULT_NOPAGE && wakeref && !obj->userfault_count) { |
1125 | obj->userfault_count = 1; |
1126 | spin_lock(lock: &to_i915(dev: obj->base.dev)->runtime_pm.lmem_userfault_lock); |
1127 | list_add(new: &obj->userfault_link, head: &to_i915(dev: obj->base.dev)->runtime_pm.lmem_userfault_list); |
1128 | spin_unlock(lock: &to_i915(dev: obj->base.dev)->runtime_pm.lmem_userfault_lock); |
1129 | |
1130 | GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(bo->resource)); |
1131 | } |
1132 | |
1133 | if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) |
1134 | intel_wakeref_auto(wf: &to_i915(dev: obj->base.dev)->runtime_pm.userfault_wakeref, |
1135 | timeout: msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
1136 | |
1137 | i915_ttm_adjust_lru(obj); |
1138 | |
1139 | dma_resv_unlock(obj: bo->base.resv); |
1140 | |
1141 | out_rpm: |
1142 | if (wakeref) |
1143 | intel_runtime_pm_put(rpm: &to_i915(dev: obj->base.dev)->runtime_pm, wref: wakeref); |
1144 | |
1145 | return ret; |
1146 | } |
1147 | |
1148 | static int |
1149 | vm_access_ttm(struct vm_area_struct *area, unsigned long addr, |
1150 | void *buf, int len, int write) |
1151 | { |
1152 | struct drm_i915_gem_object *obj = |
1153 | i915_ttm_to_gem(bo: area->vm_private_data); |
1154 | |
1155 | if (i915_gem_object_is_readonly(obj) && write) |
1156 | return -EACCES; |
1157 | |
1158 | return ttm_bo_vm_access(vma: area, addr, buf, len, write); |
1159 | } |
1160 | |
1161 | static void ttm_vm_open(struct vm_area_struct *vma) |
1162 | { |
1163 | struct drm_i915_gem_object *obj = |
1164 | i915_ttm_to_gem(bo: vma->vm_private_data); |
1165 | |
1166 | GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); |
1167 | i915_gem_object_get(obj); |
1168 | } |
1169 | |
1170 | static void ttm_vm_close(struct vm_area_struct *vma) |
1171 | { |
1172 | struct drm_i915_gem_object *obj = |
1173 | i915_ttm_to_gem(bo: vma->vm_private_data); |
1174 | |
1175 | GEM_BUG_ON(i915_ttm_is_ghost_object(vma->vm_private_data)); |
1176 | i915_gem_object_put(obj); |
1177 | } |
1178 | |
1179 | static const struct vm_operations_struct vm_ops_ttm = { |
1180 | .fault = vm_fault_ttm, |
1181 | .access = vm_access_ttm, |
1182 | .open = ttm_vm_open, |
1183 | .close = ttm_vm_close, |
1184 | }; |
1185 | |
1186 | static u64 i915_ttm_mmap_offset(struct drm_i915_gem_object *obj) |
1187 | { |
1188 | /* The ttm_bo must be allocated with I915_BO_ALLOC_USER */ |
1189 | GEM_BUG_ON(!drm_mm_node_allocated(&obj->base.vma_node.vm_node)); |
1190 | |
1191 | return drm_vma_node_offset_addr(node: &obj->base.vma_node); |
1192 | } |
1193 | |
1194 | static void i915_ttm_unmap_virtual(struct drm_i915_gem_object *obj) |
1195 | { |
1196 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
1197 | intel_wakeref_t wakeref = 0; |
1198 | |
1199 | assert_object_held_shared(obj); |
1200 | |
1201 | if (i915_ttm_cpu_maps_iomem(mem: bo->resource)) { |
1202 | wakeref = intel_runtime_pm_get(rpm: &to_i915(dev: obj->base.dev)->runtime_pm); |
1203 | |
1204 | /* userfault_count is protected by obj lock and rpm wakeref. */ |
1205 | if (obj->userfault_count) { |
1206 | spin_lock(lock: &to_i915(dev: obj->base.dev)->runtime_pm.lmem_userfault_lock); |
1207 | list_del(entry: &obj->userfault_link); |
1208 | spin_unlock(lock: &to_i915(dev: obj->base.dev)->runtime_pm.lmem_userfault_lock); |
1209 | obj->userfault_count = 0; |
1210 | } |
1211 | } |
1212 | |
1213 | GEM_WARN_ON(obj->userfault_count); |
1214 | |
1215 | ttm_bo_unmap_virtual(bo: i915_gem_to_ttm(obj)); |
1216 | |
1217 | if (wakeref) |
1218 | intel_runtime_pm_put(rpm: &to_i915(dev: obj->base.dev)->runtime_pm, wref: wakeref); |
1219 | } |
1220 | |
1221 | static const struct drm_i915_gem_object_ops i915_gem_ttm_obj_ops = { |
1222 | .name = "i915_gem_object_ttm" , |
1223 | .flags = I915_GEM_OBJECT_IS_SHRINKABLE | |
1224 | I915_GEM_OBJECT_SELF_MANAGED_SHRINK_LIST, |
1225 | |
1226 | .get_pages = i915_ttm_get_pages, |
1227 | .put_pages = i915_ttm_put_pages, |
1228 | .truncate = i915_ttm_truncate, |
1229 | .shrink = i915_ttm_shrink, |
1230 | |
1231 | .adjust_lru = i915_ttm_adjust_lru, |
1232 | .delayed_free = i915_ttm_delayed_free, |
1233 | .migrate = i915_ttm_migrate, |
1234 | |
1235 | .mmap_offset = i915_ttm_mmap_offset, |
1236 | .unmap_virtual = i915_ttm_unmap_virtual, |
1237 | .mmap_ops = &vm_ops_ttm, |
1238 | }; |
1239 | |
1240 | void i915_ttm_bo_destroy(struct ttm_buffer_object *bo) |
1241 | { |
1242 | struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); |
1243 | |
1244 | i915_gem_object_release_memory_region(obj); |
1245 | mutex_destroy(lock: &obj->ttm.get_io_page.lock); |
1246 | |
1247 | if (obj->ttm.created) { |
1248 | /* |
1249 | * We freely manage the shrinker LRU outide of the mm.pages life |
1250 | * cycle. As a result when destroying the object we should be |
1251 | * extra paranoid and ensure we remove it from the LRU, before |
1252 | * we free the object. |
1253 | * |
1254 | * Touching the ttm_shrinkable outside of the object lock here |
1255 | * should be safe now that the last GEM object ref was dropped. |
1256 | */ |
1257 | if (obj->mm.ttm_shrinkable) |
1258 | i915_gem_object_make_unshrinkable(obj); |
1259 | |
1260 | i915_ttm_backup_free(obj); |
1261 | |
1262 | /* This releases all gem object bindings to the backend. */ |
1263 | __i915_gem_free_object(obj); |
1264 | |
1265 | call_rcu(head: &obj->rcu, func: __i915_gem_free_object_rcu); |
1266 | } else { |
1267 | __i915_gem_object_fini(obj); |
1268 | } |
1269 | } |
1270 | |
1271 | /* |
1272 | * __i915_gem_ttm_object_init - Initialize a ttm-backed i915 gem object |
1273 | * @mem: The initial memory region for the object. |
1274 | * @obj: The gem object. |
1275 | * @size: Object size in bytes. |
1276 | * @flags: gem object flags. |
1277 | * |
1278 | * Return: 0 on success, negative error code on failure. |
1279 | */ |
1280 | int __i915_gem_ttm_object_init(struct intel_memory_region *mem, |
1281 | struct drm_i915_gem_object *obj, |
1282 | resource_size_t offset, |
1283 | resource_size_t size, |
1284 | resource_size_t page_size, |
1285 | unsigned int flags) |
1286 | { |
1287 | static struct lock_class_key lock_class; |
1288 | struct drm_i915_private *i915 = mem->i915; |
1289 | struct ttm_operation_ctx ctx = { |
1290 | .interruptible = true, |
1291 | .no_wait_gpu = false, |
1292 | }; |
1293 | enum ttm_bo_type bo_type; |
1294 | int ret; |
1295 | |
1296 | drm_gem_private_object_init(dev: &i915->drm, obj: &obj->base, size); |
1297 | i915_gem_object_init(obj, ops: &i915_gem_ttm_obj_ops, key: &lock_class, alloc_flags: flags); |
1298 | |
1299 | obj->bo_offset = offset; |
1300 | |
1301 | /* Don't put on a region list until we're either locked or fully initialized. */ |
1302 | obj->mm.region = mem; |
1303 | INIT_LIST_HEAD(list: &obj->mm.region_link); |
1304 | |
1305 | INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN); |
1306 | mutex_init(&obj->ttm.get_io_page.lock); |
1307 | bo_type = (obj->flags & I915_BO_ALLOC_USER) ? ttm_bo_type_device : |
1308 | ttm_bo_type_kernel; |
1309 | |
1310 | obj->base.vma_node.driver_private = i915_gem_to_ttm(obj); |
1311 | |
1312 | /* Forcing the page size is kernel internal only */ |
1313 | GEM_BUG_ON(page_size && obj->mm.n_placements); |
1314 | |
1315 | /* |
1316 | * Keep an extra shrink pin to prevent the object from being made |
1317 | * shrinkable too early. If the ttm_tt is ever allocated in shmem, we |
1318 | * drop the pin. The TTM backend manages the shrinker LRU itself, |
1319 | * outside of the normal mm.pages life cycle. |
1320 | */ |
1321 | i915_gem_object_make_unshrinkable(obj); |
1322 | |
1323 | /* |
1324 | * If this function fails, it will call the destructor, but |
1325 | * our caller still owns the object. So no freeing in the |
1326 | * destructor until obj->ttm.created is true. |
1327 | * Similarly, in delayed_destroy, we can't call ttm_bo_put() |
1328 | * until successful initialization. |
1329 | */ |
1330 | ret = ttm_bo_init_reserved(bdev: &i915->bdev, bo: i915_gem_to_ttm(obj), type: bo_type, |
1331 | placement: &i915_sys_placement, alignment: page_size >> PAGE_SHIFT, |
1332 | ctx: &ctx, NULL, NULL, destroy: i915_ttm_bo_destroy); |
1333 | |
1334 | /* |
1335 | * XXX: The ttm_bo_init_reserved() functions returns -ENOSPC if the size |
1336 | * is too big to add vma. The direct function that returns -ENOSPC is |
1337 | * drm_mm_insert_node_in_range(). To handle the same error as other code |
1338 | * that returns -E2BIG when the size is too large, it converts -ENOSPC to |
1339 | * -E2BIG. |
1340 | */ |
1341 | if (size >> PAGE_SHIFT > INT_MAX && ret == -ENOSPC) |
1342 | ret = -E2BIG; |
1343 | |
1344 | if (ret) |
1345 | return i915_ttm_err_to_gem(err: ret); |
1346 | |
1347 | obj->ttm.created = true; |
1348 | i915_gem_object_release_memory_region(obj); |
1349 | i915_gem_object_init_memory_region(obj, mem); |
1350 | i915_ttm_adjust_domains_after_move(obj); |
1351 | i915_ttm_adjust_gem_after_move(obj); |
1352 | i915_gem_object_unlock(obj); |
1353 | |
1354 | return 0; |
1355 | } |
1356 | |
1357 | static const struct intel_memory_region_ops ttm_system_region_ops = { |
1358 | .init_object = __i915_gem_ttm_object_init, |
1359 | .release = intel_region_ttm_fini, |
1360 | }; |
1361 | |
1362 | struct intel_memory_region * |
1363 | i915_gem_ttm_system_setup(struct drm_i915_private *i915, |
1364 | u16 type, u16 instance) |
1365 | { |
1366 | struct intel_memory_region *mr; |
1367 | |
1368 | mr = intel_memory_region_create(i915, start: 0, |
1369 | size: totalram_pages() << PAGE_SHIFT, |
1370 | PAGE_SIZE, io_start: 0, io_size: 0, |
1371 | type, instance, |
1372 | ops: &ttm_system_region_ops); |
1373 | if (IS_ERR(ptr: mr)) |
1374 | return mr; |
1375 | |
1376 | intel_memory_region_set_name(mem: mr, fmt: "system-ttm" ); |
1377 | return mr; |
1378 | } |
1379 | |