1 | /* |
2 | * Copyright © 2017 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include <linux/highmem.h> |
26 | #include <linux/sched/mm.h> |
27 | |
28 | #include <drm/drm_cache.h> |
29 | |
30 | #include "display/intel_frontbuffer.h" |
31 | #include "pxp/intel_pxp.h" |
32 | |
33 | #include "i915_drv.h" |
34 | #include "i915_file_private.h" |
35 | #include "i915_gem_clflush.h" |
36 | #include "i915_gem_context.h" |
37 | #include "i915_gem_dmabuf.h" |
38 | #include "i915_gem_mman.h" |
39 | #include "i915_gem_object.h" |
40 | #include "i915_gem_object_frontbuffer.h" |
41 | #include "i915_gem_ttm.h" |
42 | #include "i915_memcpy.h" |
43 | #include "i915_trace.h" |
44 | |
45 | static struct kmem_cache *slab_objects; |
46 | |
47 | static const struct drm_gem_object_funcs i915_gem_object_funcs; |
48 | |
49 | unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915, |
50 | enum i915_cache_level level) |
51 | { |
52 | if (drm_WARN_ON(&i915->drm, level >= I915_MAX_CACHE_LEVEL)) |
53 | return 0; |
54 | |
55 | return INTEL_INFO(i915)->cachelevel_to_pat[level]; |
56 | } |
57 | |
58 | bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj, |
59 | enum i915_cache_level lvl) |
60 | { |
61 | /* |
62 | * In case the pat_index is set by user space, this kernel mode |
63 | * driver should leave the coherency to be managed by user space, |
64 | * simply return true here. |
65 | */ |
66 | if (obj->pat_set_by_user) |
67 | return true; |
68 | |
69 | /* |
70 | * Otherwise the pat_index should have been converted from cache_level |
71 | * so that the following comparison is valid. |
72 | */ |
73 | return obj->pat_index == i915_gem_get_pat_index(obj_to_i915(obj), level: lvl); |
74 | } |
75 | |
76 | struct drm_i915_gem_object *i915_gem_object_alloc(void) |
77 | { |
78 | struct drm_i915_gem_object *obj; |
79 | |
80 | obj = kmem_cache_zalloc(k: slab_objects, GFP_KERNEL); |
81 | if (!obj) |
82 | return NULL; |
83 | obj->base.funcs = &i915_gem_object_funcs; |
84 | |
85 | return obj; |
86 | } |
87 | |
88 | void i915_gem_object_free(struct drm_i915_gem_object *obj) |
89 | { |
90 | return kmem_cache_free(s: slab_objects, objp: obj); |
91 | } |
92 | |
93 | void i915_gem_object_init(struct drm_i915_gem_object *obj, |
94 | const struct drm_i915_gem_object_ops *ops, |
95 | struct lock_class_key *key, unsigned flags) |
96 | { |
97 | /* |
98 | * A gem object is embedded both in a struct ttm_buffer_object :/ and |
99 | * in a drm_i915_gem_object. Make sure they are aliased. |
100 | */ |
101 | BUILD_BUG_ON(offsetof(typeof(*obj), base) != |
102 | offsetof(typeof(*obj), __do_not_access.base)); |
103 | |
104 | spin_lock_init(&obj->vma.lock); |
105 | INIT_LIST_HEAD(list: &obj->vma.list); |
106 | |
107 | INIT_LIST_HEAD(list: &obj->mm.link); |
108 | |
109 | #ifdef CONFIG_PROC_FS |
110 | INIT_LIST_HEAD(list: &obj->client_link); |
111 | #endif |
112 | |
113 | INIT_LIST_HEAD(list: &obj->lut_list); |
114 | spin_lock_init(&obj->lut_lock); |
115 | |
116 | spin_lock_init(&obj->mmo.lock); |
117 | obj->mmo.offsets = RB_ROOT; |
118 | |
119 | init_rcu_head(head: &obj->rcu); |
120 | |
121 | obj->ops = ops; |
122 | GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS); |
123 | obj->flags = flags; |
124 | |
125 | obj->mm.madv = I915_MADV_WILLNEED; |
126 | INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); |
127 | mutex_init(&obj->mm.get_page.lock); |
128 | INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN); |
129 | mutex_init(&obj->mm.get_dma_page.lock); |
130 | } |
131 | |
132 | /** |
133 | * __i915_gem_object_fini - Clean up a GEM object initialization |
134 | * @obj: The gem object to cleanup |
135 | * |
136 | * This function cleans up gem object fields that are set up by |
137 | * drm_gem_private_object_init() and i915_gem_object_init(). |
138 | * It's primarily intended as a helper for backends that need to |
139 | * clean up the gem object in separate steps. |
140 | */ |
141 | void __i915_gem_object_fini(struct drm_i915_gem_object *obj) |
142 | { |
143 | mutex_destroy(lock: &obj->mm.get_page.lock); |
144 | mutex_destroy(lock: &obj->mm.get_dma_page.lock); |
145 | dma_resv_fini(obj: &obj->base._resv); |
146 | } |
147 | |
148 | /** |
149 | * i915_gem_object_set_cache_coherency - Mark up the object's coherency levels |
150 | * for a given cache_level |
151 | * @obj: #drm_i915_gem_object |
152 | * @cache_level: cache level |
153 | */ |
154 | void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj, |
155 | unsigned int cache_level) |
156 | { |
157 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
158 | |
159 | obj->pat_index = i915_gem_get_pat_index(i915, level: cache_level); |
160 | |
161 | if (cache_level != I915_CACHE_NONE) |
162 | obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | |
163 | I915_BO_CACHE_COHERENT_FOR_WRITE); |
164 | else if (HAS_LLC(i915)) |
165 | obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; |
166 | else |
167 | obj->cache_coherent = 0; |
168 | |
169 | obj->cache_dirty = |
170 | !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) && |
171 | !IS_DGFX(i915); |
172 | } |
173 | |
174 | /** |
175 | * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode |
176 | * @obj: #drm_i915_gem_object |
177 | * @pat_index: PAT index |
178 | * |
179 | * This is a clone of i915_gem_object_set_cache_coherency taking pat index |
180 | * instead of cache_level as its second argument. |
181 | */ |
182 | void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj, |
183 | unsigned int pat_index) |
184 | { |
185 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
186 | |
187 | if (obj->pat_index == pat_index) |
188 | return; |
189 | |
190 | obj->pat_index = pat_index; |
191 | |
192 | if (pat_index != i915_gem_get_pat_index(i915, level: I915_CACHE_NONE)) |
193 | obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ | |
194 | I915_BO_CACHE_COHERENT_FOR_WRITE); |
195 | else if (HAS_LLC(i915)) |
196 | obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ; |
197 | else |
198 | obj->cache_coherent = 0; |
199 | |
200 | obj->cache_dirty = |
201 | !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) && |
202 | !IS_DGFX(i915); |
203 | } |
204 | |
205 | bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj) |
206 | { |
207 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
208 | |
209 | /* |
210 | * This is purely from a security perspective, so we simply don't care |
211 | * about non-userspace objects being able to bypass the LLC. |
212 | */ |
213 | if (!(obj->flags & I915_BO_ALLOC_USER)) |
214 | return false; |
215 | |
216 | /* |
217 | * Always flush cache for UMD objects at creation time. |
218 | */ |
219 | if (obj->pat_set_by_user) |
220 | return true; |
221 | |
222 | /* |
223 | * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it |
224 | * possible for userspace to bypass the GTT caching bits set by the |
225 | * kernel, as per the given object cache_level. This is troublesome |
226 | * since the heavy flush we apply when first gathering the pages is |
227 | * skipped if the kernel thinks the object is coherent with the GPU. As |
228 | * a result it might be possible to bypass the cache and read the |
229 | * contents of the page directly, which could be stale data. If it's |
230 | * just a case of userspace shooting themselves in the foot then so be |
231 | * it, but since i915 takes the stance of always zeroing memory before |
232 | * handing it to userspace, we need to prevent this. |
233 | */ |
234 | return (IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)); |
235 | } |
236 | |
237 | static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) |
238 | { |
239 | struct drm_i915_gem_object *obj = to_intel_bo(gem); |
240 | struct drm_i915_file_private *fpriv = file->driver_priv; |
241 | struct i915_lut_handle bookmark = {}; |
242 | struct i915_mmap_offset *mmo, *mn; |
243 | struct i915_lut_handle *lut, *ln; |
244 | LIST_HEAD(close); |
245 | |
246 | spin_lock(lock: &obj->lut_lock); |
247 | list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) { |
248 | struct i915_gem_context *ctx = lut->ctx; |
249 | |
250 | if (ctx && ctx->file_priv == fpriv) { |
251 | i915_gem_context_get(ctx); |
252 | list_move(list: &lut->obj_link, head: &close); |
253 | } |
254 | |
255 | /* Break long locks, and carefully continue on from this spot */ |
256 | if (&ln->obj_link != &obj->lut_list) { |
257 | list_add_tail(new: &bookmark.obj_link, head: &ln->obj_link); |
258 | if (cond_resched_lock(&obj->lut_lock)) |
259 | list_safe_reset_next(&bookmark, ln, obj_link); |
260 | __list_del_entry(entry: &bookmark.obj_link); |
261 | } |
262 | } |
263 | spin_unlock(lock: &obj->lut_lock); |
264 | |
265 | spin_lock(lock: &obj->mmo.lock); |
266 | rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset) |
267 | drm_vma_node_revoke(node: &mmo->vma_node, tag: file); |
268 | spin_unlock(lock: &obj->mmo.lock); |
269 | |
270 | list_for_each_entry_safe(lut, ln, &close, obj_link) { |
271 | struct i915_gem_context *ctx = lut->ctx; |
272 | struct i915_vma *vma; |
273 | |
274 | /* |
275 | * We allow the process to have multiple handles to the same |
276 | * vma, in the same fd namespace, by virtue of flink/open. |
277 | */ |
278 | |
279 | mutex_lock(&ctx->lut_mutex); |
280 | vma = radix_tree_delete(&ctx->handles_vma, lut->handle); |
281 | if (vma) { |
282 | GEM_BUG_ON(vma->obj != obj); |
283 | GEM_BUG_ON(!atomic_read(&vma->open_count)); |
284 | i915_vma_close(vma); |
285 | } |
286 | mutex_unlock(lock: &ctx->lut_mutex); |
287 | |
288 | i915_gem_context_put(ctx: lut->ctx); |
289 | i915_lut_handle_free(lut); |
290 | i915_gem_object_put(obj); |
291 | } |
292 | } |
293 | |
294 | void __i915_gem_free_object_rcu(struct rcu_head *head) |
295 | { |
296 | struct drm_i915_gem_object *obj = |
297 | container_of(head, typeof(*obj), rcu); |
298 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
299 | |
300 | /* We need to keep this alive for RCU read access from fdinfo. */ |
301 | if (obj->mm.n_placements > 1) |
302 | kfree(objp: obj->mm.placements); |
303 | |
304 | i915_gem_object_free(obj); |
305 | |
306 | GEM_BUG_ON(!atomic_read(&i915->mm.free_count)); |
307 | atomic_dec(v: &i915->mm.free_count); |
308 | } |
309 | |
310 | static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) |
311 | { |
312 | /* Skip serialisation and waking the device if known to be not used. */ |
313 | |
314 | if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev))) |
315 | i915_gem_object_release_mmap_gtt(obj); |
316 | |
317 | if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) { |
318 | struct i915_mmap_offset *mmo, *mn; |
319 | |
320 | i915_gem_object_release_mmap_offset(obj); |
321 | |
322 | rbtree_postorder_for_each_entry_safe(mmo, mn, |
323 | &obj->mmo.offsets, |
324 | offset) { |
325 | drm_vma_offset_remove(mgr: obj->base.dev->vma_offset_manager, |
326 | node: &mmo->vma_node); |
327 | kfree(objp: mmo); |
328 | } |
329 | obj->mmo.offsets = RB_ROOT; |
330 | } |
331 | } |
332 | |
333 | /** |
334 | * __i915_gem_object_pages_fini - Clean up pages use of a gem object |
335 | * @obj: The gem object to clean up |
336 | * |
337 | * This function cleans up usage of the object mm.pages member. It |
338 | * is intended for backends that need to clean up a gem object in |
339 | * separate steps and needs to be called when the object is idle before |
340 | * the object's backing memory is freed. |
341 | */ |
342 | void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj) |
343 | { |
344 | assert_object_held_shared(obj); |
345 | |
346 | if (!list_empty(head: &obj->vma.list)) { |
347 | struct i915_vma *vma; |
348 | |
349 | spin_lock(lock: &obj->vma.lock); |
350 | while ((vma = list_first_entry_or_null(&obj->vma.list, |
351 | struct i915_vma, |
352 | obj_link))) { |
353 | GEM_BUG_ON(vma->obj != obj); |
354 | spin_unlock(lock: &obj->vma.lock); |
355 | |
356 | i915_vma_destroy(vma); |
357 | |
358 | spin_lock(lock: &obj->vma.lock); |
359 | } |
360 | spin_unlock(lock: &obj->vma.lock); |
361 | } |
362 | |
363 | __i915_gem_object_free_mmaps(obj); |
364 | |
365 | atomic_set(v: &obj->mm.pages_pin_count, i: 0); |
366 | |
367 | /* |
368 | * dma_buf_unmap_attachment() requires reservation to be |
369 | * locked. The imported GEM shouldn't share reservation lock |
370 | * and ttm_bo_cleanup_memtype_use() shouldn't be invoked for |
371 | * dma-buf, so it's safe to take the lock. |
372 | */ |
373 | if (obj->base.import_attach) |
374 | i915_gem_object_lock(obj, NULL); |
375 | |
376 | __i915_gem_object_put_pages(obj); |
377 | |
378 | if (obj->base.import_attach) |
379 | i915_gem_object_unlock(obj); |
380 | |
381 | GEM_BUG_ON(i915_gem_object_has_pages(obj)); |
382 | } |
383 | |
384 | void __i915_gem_free_object(struct drm_i915_gem_object *obj) |
385 | { |
386 | trace_i915_gem_object_destroy(obj); |
387 | |
388 | GEM_BUG_ON(!list_empty(&obj->lut_list)); |
389 | |
390 | bitmap_free(bitmap: obj->bit_17); |
391 | |
392 | if (obj->base.import_attach) |
393 | drm_prime_gem_destroy(obj: &obj->base, NULL); |
394 | |
395 | drm_gem_free_mmap_offset(obj: &obj->base); |
396 | |
397 | if (obj->ops->release) |
398 | obj->ops->release(obj); |
399 | |
400 | if (obj->shares_resv_from) |
401 | i915_vm_resv_put(vm: obj->shares_resv_from); |
402 | |
403 | __i915_gem_object_fini(obj); |
404 | } |
405 | |
406 | static void __i915_gem_free_objects(struct drm_i915_private *i915, |
407 | struct llist_node *freed) |
408 | { |
409 | struct drm_i915_gem_object *obj, *on; |
410 | |
411 | llist_for_each_entry_safe(obj, on, freed, freed) { |
412 | might_sleep(); |
413 | if (obj->ops->delayed_free) { |
414 | obj->ops->delayed_free(obj); |
415 | continue; |
416 | } |
417 | |
418 | __i915_gem_object_pages_fini(obj); |
419 | __i915_gem_free_object(obj); |
420 | |
421 | /* But keep the pointer alive for RCU-protected lookups */ |
422 | call_rcu(head: &obj->rcu, func: __i915_gem_free_object_rcu); |
423 | cond_resched(); |
424 | } |
425 | } |
426 | |
427 | void i915_gem_flush_free_objects(struct drm_i915_private *i915) |
428 | { |
429 | struct llist_node *freed = llist_del_all(head: &i915->mm.free_list); |
430 | |
431 | if (unlikely(freed)) |
432 | __i915_gem_free_objects(i915, freed); |
433 | } |
434 | |
435 | static void __i915_gem_free_work(struct work_struct *work) |
436 | { |
437 | struct drm_i915_private *i915 = |
438 | container_of(work, struct drm_i915_private, mm.free_work); |
439 | |
440 | i915_gem_flush_free_objects(i915); |
441 | } |
442 | |
443 | static void i915_gem_free_object(struct drm_gem_object *gem_obj) |
444 | { |
445 | struct drm_i915_gem_object *obj = to_intel_bo(gem: gem_obj); |
446 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
447 | |
448 | GEM_BUG_ON(i915_gem_object_is_framebuffer(obj)); |
449 | |
450 | i915_drm_client_remove_object(obj); |
451 | |
452 | /* |
453 | * Before we free the object, make sure any pure RCU-only |
454 | * read-side critical sections are complete, e.g. |
455 | * i915_gem_busy_ioctl(). For the corresponding synchronized |
456 | * lookup see i915_gem_object_lookup_rcu(). |
457 | */ |
458 | atomic_inc(v: &i915->mm.free_count); |
459 | |
460 | /* |
461 | * Since we require blocking on struct_mutex to unbind the freed |
462 | * object from the GPU before releasing resources back to the |
463 | * system, we can not do that directly from the RCU callback (which may |
464 | * be a softirq context), but must instead then defer that work onto a |
465 | * kthread. We use the RCU callback rather than move the freed object |
466 | * directly onto the work queue so that we can mix between using the |
467 | * worker and performing frees directly from subsequent allocations for |
468 | * crude but effective memory throttling. |
469 | */ |
470 | |
471 | if (llist_add(new: &obj->freed, head: &i915->mm.free_list)) |
472 | queue_work(wq: i915->wq, work: &i915->mm.free_work); |
473 | } |
474 | |
475 | void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj, |
476 | enum fb_op_origin origin) |
477 | { |
478 | struct intel_frontbuffer *front; |
479 | |
480 | front = i915_gem_object_get_frontbuffer(obj); |
481 | if (front) { |
482 | intel_frontbuffer_flush(front, origin); |
483 | intel_frontbuffer_put(front); |
484 | } |
485 | } |
486 | |
487 | void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj, |
488 | enum fb_op_origin origin) |
489 | { |
490 | struct intel_frontbuffer *front; |
491 | |
492 | front = i915_gem_object_get_frontbuffer(obj); |
493 | if (front) { |
494 | intel_frontbuffer_invalidate(front, origin); |
495 | intel_frontbuffer_put(front); |
496 | } |
497 | } |
498 | |
499 | static void |
500 | i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) |
501 | { |
502 | pgoff_t idx = offset >> PAGE_SHIFT; |
503 | void *src_ptr; |
504 | |
505 | src_ptr = kmap_local_page(i915_gem_object_get_page(obj, idx)) |
506 | + offset_in_page(offset); |
507 | if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) |
508 | drm_clflush_virt_range(addr: src_ptr, length: size); |
509 | memcpy(dst, src_ptr, size); |
510 | |
511 | kunmap_local(src_ptr); |
512 | } |
513 | |
514 | static void |
515 | i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) |
516 | { |
517 | pgoff_t idx = offset >> PAGE_SHIFT; |
518 | dma_addr_t dma = i915_gem_object_get_dma_address(obj, idx); |
519 | void __iomem *src_map; |
520 | void __iomem *src_ptr; |
521 | |
522 | src_map = io_mapping_map_wc(mapping: &obj->mm.region->iomap, |
523 | offset: dma - obj->mm.region->region.start, |
524 | PAGE_SIZE); |
525 | |
526 | src_ptr = src_map + offset_in_page(offset); |
527 | if (!i915_memcpy_from_wc(dst, src: (void __force *)src_ptr, len: size)) |
528 | memcpy_fromio(dst, src_ptr, size); |
529 | |
530 | io_mapping_unmap(vaddr: src_map); |
531 | } |
532 | |
533 | static bool object_has_mappable_iomem(struct drm_i915_gem_object *obj) |
534 | { |
535 | GEM_BUG_ON(!i915_gem_object_has_iomem(obj)); |
536 | |
537 | if (IS_DGFX(to_i915(obj->base.dev))) |
538 | return i915_ttm_resource_mappable(res: i915_gem_to_ttm(obj)->resource); |
539 | |
540 | return true; |
541 | } |
542 | |
543 | /** |
544 | * i915_gem_object_read_from_page - read data from the page of a GEM object |
545 | * @obj: GEM object to read from |
546 | * @offset: offset within the object |
547 | * @dst: buffer to store the read data |
548 | * @size: size to read |
549 | * |
550 | * Reads data from @obj at the specified offset. The requested region to read |
551 | * from can't cross a page boundary. The caller must ensure that @obj pages |
552 | * are pinned and that @obj is synced wrt. any related writes. |
553 | * |
554 | * Return: %0 on success or -ENODEV if the type of @obj's backing store is |
555 | * unsupported. |
556 | */ |
557 | int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size) |
558 | { |
559 | GEM_BUG_ON(overflows_type(offset >> PAGE_SHIFT, pgoff_t)); |
560 | GEM_BUG_ON(offset >= obj->base.size); |
561 | GEM_BUG_ON(offset_in_page(offset) > PAGE_SIZE - size); |
562 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); |
563 | |
564 | if (i915_gem_object_has_struct_page(obj)) |
565 | i915_gem_object_read_from_page_kmap(obj, offset, dst, size); |
566 | else if (i915_gem_object_has_iomem(obj) && object_has_mappable_iomem(obj)) |
567 | i915_gem_object_read_from_page_iomap(obj, offset, dst, size); |
568 | else |
569 | return -ENODEV; |
570 | |
571 | return 0; |
572 | } |
573 | |
574 | /** |
575 | * i915_gem_object_evictable - Whether object is likely evictable after unbind. |
576 | * @obj: The object to check |
577 | * |
578 | * This function checks whether the object is likely unvictable after unbind. |
579 | * If the object is not locked when checking, the result is only advisory. |
580 | * If the object is locked when checking, and the function returns true, |
581 | * then an eviction should indeed be possible. But since unlocked vma |
582 | * unpinning and unbinding is currently possible, the object can actually |
583 | * become evictable even if this function returns false. |
584 | * |
585 | * Return: true if the object may be evictable. False otherwise. |
586 | */ |
587 | bool i915_gem_object_evictable(struct drm_i915_gem_object *obj) |
588 | { |
589 | struct i915_vma *vma; |
590 | int pin_count = atomic_read(v: &obj->mm.pages_pin_count); |
591 | |
592 | if (!pin_count) |
593 | return true; |
594 | |
595 | spin_lock(lock: &obj->vma.lock); |
596 | list_for_each_entry(vma, &obj->vma.list, obj_link) { |
597 | if (i915_vma_is_pinned(vma)) { |
598 | spin_unlock(lock: &obj->vma.lock); |
599 | return false; |
600 | } |
601 | if (atomic_read(v: &vma->pages_count)) |
602 | pin_count--; |
603 | } |
604 | spin_unlock(lock: &obj->vma.lock); |
605 | GEM_WARN_ON(pin_count < 0); |
606 | |
607 | return pin_count == 0; |
608 | } |
609 | |
610 | /** |
611 | * i915_gem_object_migratable - Whether the object is migratable out of the |
612 | * current region. |
613 | * @obj: Pointer to the object. |
614 | * |
615 | * Return: Whether the object is allowed to be resident in other |
616 | * regions than the current while pages are present. |
617 | */ |
618 | bool i915_gem_object_migratable(struct drm_i915_gem_object *obj) |
619 | { |
620 | struct intel_memory_region *mr = READ_ONCE(obj->mm.region); |
621 | |
622 | if (!mr) |
623 | return false; |
624 | |
625 | return obj->mm.n_placements > 1; |
626 | } |
627 | |
628 | /** |
629 | * i915_gem_object_has_struct_page - Whether the object is page-backed |
630 | * @obj: The object to query. |
631 | * |
632 | * This function should only be called while the object is locked or pinned, |
633 | * otherwise the page backing may change under the caller. |
634 | * |
635 | * Return: True if page-backed, false otherwise. |
636 | */ |
637 | bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) |
638 | { |
639 | #ifdef CONFIG_LOCKDEP |
640 | if (IS_DGFX(to_i915(obj->base.dev)) && |
641 | i915_gem_object_evictable(obj: (void __force *)obj)) |
642 | assert_object_held_shared(obj); |
643 | #endif |
644 | return obj->mem_flags & I915_BO_FLAG_STRUCT_PAGE; |
645 | } |
646 | |
647 | /** |
648 | * i915_gem_object_has_iomem - Whether the object is iomem-backed |
649 | * @obj: The object to query. |
650 | * |
651 | * This function should only be called while the object is locked or pinned, |
652 | * otherwise the iomem backing may change under the caller. |
653 | * |
654 | * Return: True if iomem-backed, false otherwise. |
655 | */ |
656 | bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj) |
657 | { |
658 | #ifdef CONFIG_LOCKDEP |
659 | if (IS_DGFX(to_i915(obj->base.dev)) && |
660 | i915_gem_object_evictable(obj: (void __force *)obj)) |
661 | assert_object_held_shared(obj); |
662 | #endif |
663 | return obj->mem_flags & I915_BO_FLAG_IOMEM; |
664 | } |
665 | |
666 | /** |
667 | * i915_gem_object_can_migrate - Whether an object likely can be migrated |
668 | * |
669 | * @obj: The object to migrate |
670 | * @id: The region intended to migrate to |
671 | * |
672 | * Check whether the object backend supports migration to the |
673 | * given region. Note that pinning may affect the ability to migrate as |
674 | * returned by this function. |
675 | * |
676 | * This function is primarily intended as a helper for checking the |
677 | * possibility to migrate objects and might be slightly less permissive |
678 | * than i915_gem_object_migrate() when it comes to objects with the |
679 | * I915_BO_ALLOC_USER flag set. |
680 | * |
681 | * Return: true if migration is possible, false otherwise. |
682 | */ |
683 | bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, |
684 | enum intel_region_id id) |
685 | { |
686 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
687 | unsigned int num_allowed = obj->mm.n_placements; |
688 | struct intel_memory_region *mr; |
689 | unsigned int i; |
690 | |
691 | GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN); |
692 | GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); |
693 | |
694 | mr = i915->mm.regions[id]; |
695 | if (!mr) |
696 | return false; |
697 | |
698 | if (!IS_ALIGNED(obj->base.size, mr->min_page_size)) |
699 | return false; |
700 | |
701 | if (obj->mm.region == mr) |
702 | return true; |
703 | |
704 | if (!i915_gem_object_evictable(obj)) |
705 | return false; |
706 | |
707 | if (!obj->ops->migrate) |
708 | return false; |
709 | |
710 | if (!(obj->flags & I915_BO_ALLOC_USER)) |
711 | return true; |
712 | |
713 | if (num_allowed == 0) |
714 | return false; |
715 | |
716 | for (i = 0; i < num_allowed; ++i) { |
717 | if (mr == obj->mm.placements[i]) |
718 | return true; |
719 | } |
720 | |
721 | return false; |
722 | } |
723 | |
724 | /** |
725 | * i915_gem_object_migrate - Migrate an object to the desired region id |
726 | * @obj: The object to migrate. |
727 | * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may |
728 | * not be successful in evicting other objects to make room for this object. |
729 | * @id: The region id to migrate to. |
730 | * |
731 | * Attempt to migrate the object to the desired memory region. The |
732 | * object backend must support migration and the object may not be |
733 | * pinned, (explicitly pinned pages or pinned vmas). The object must |
734 | * be locked. |
735 | * On successful completion, the object will have pages pointing to |
736 | * memory in the new region, but an async migration task may not have |
737 | * completed yet, and to accomplish that, i915_gem_object_wait_migration() |
738 | * must be called. |
739 | * |
740 | * Note: the @ww parameter is not used yet, but included to make sure |
741 | * callers put some effort into obtaining a valid ww ctx if one is |
742 | * available. |
743 | * |
744 | * Return: 0 on success. Negative error code on failure. In particular may |
745 | * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance |
746 | * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and |
747 | * -EBUSY if the object is pinned. |
748 | */ |
749 | int i915_gem_object_migrate(struct drm_i915_gem_object *obj, |
750 | struct i915_gem_ww_ctx *ww, |
751 | enum intel_region_id id) |
752 | { |
753 | return __i915_gem_object_migrate(obj, ww, id, flags: obj->flags); |
754 | } |
755 | |
756 | /** |
757 | * __i915_gem_object_migrate - Migrate an object to the desired region id, with |
758 | * control of the extra flags |
759 | * @obj: The object to migrate. |
760 | * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may |
761 | * not be successful in evicting other objects to make room for this object. |
762 | * @id: The region id to migrate to. |
763 | * @flags: The object flags. Normally just obj->flags. |
764 | * |
765 | * Attempt to migrate the object to the desired memory region. The |
766 | * object backend must support migration and the object may not be |
767 | * pinned, (explicitly pinned pages or pinned vmas). The object must |
768 | * be locked. |
769 | * On successful completion, the object will have pages pointing to |
770 | * memory in the new region, but an async migration task may not have |
771 | * completed yet, and to accomplish that, i915_gem_object_wait_migration() |
772 | * must be called. |
773 | * |
774 | * Note: the @ww parameter is not used yet, but included to make sure |
775 | * callers put some effort into obtaining a valid ww ctx if one is |
776 | * available. |
777 | * |
778 | * Return: 0 on success. Negative error code on failure. In particular may |
779 | * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance |
780 | * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and |
781 | * -EBUSY if the object is pinned. |
782 | */ |
783 | int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, |
784 | struct i915_gem_ww_ctx *ww, |
785 | enum intel_region_id id, |
786 | unsigned int flags) |
787 | { |
788 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
789 | struct intel_memory_region *mr; |
790 | |
791 | GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN); |
792 | GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED); |
793 | assert_object_held(obj); |
794 | |
795 | mr = i915->mm.regions[id]; |
796 | GEM_BUG_ON(!mr); |
797 | |
798 | if (!i915_gem_object_can_migrate(obj, id)) |
799 | return -EINVAL; |
800 | |
801 | if (!obj->ops->migrate) { |
802 | if (GEM_WARN_ON(obj->mm.region != mr)) |
803 | return -EINVAL; |
804 | return 0; |
805 | } |
806 | |
807 | return obj->ops->migrate(obj, mr, flags); |
808 | } |
809 | |
810 | /** |
811 | * i915_gem_object_placement_possible - Check whether the object can be |
812 | * placed at certain memory type |
813 | * @obj: Pointer to the object |
814 | * @type: The memory type to check |
815 | * |
816 | * Return: True if the object can be placed in @type. False otherwise. |
817 | */ |
818 | bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj, |
819 | enum intel_memory_type type) |
820 | { |
821 | unsigned int i; |
822 | |
823 | if (!obj->mm.n_placements) { |
824 | switch (type) { |
825 | case INTEL_MEMORY_LOCAL: |
826 | return i915_gem_object_has_iomem(obj); |
827 | case INTEL_MEMORY_SYSTEM: |
828 | return i915_gem_object_has_pages(obj); |
829 | default: |
830 | /* Ignore stolen for now */ |
831 | GEM_BUG_ON(1); |
832 | return false; |
833 | } |
834 | } |
835 | |
836 | for (i = 0; i < obj->mm.n_placements; i++) { |
837 | if (obj->mm.placements[i]->type == type) |
838 | return true; |
839 | } |
840 | |
841 | return false; |
842 | } |
843 | |
844 | /** |
845 | * i915_gem_object_needs_ccs_pages - Check whether the object requires extra |
846 | * pages when placed in system-memory, in order to save and later restore the |
847 | * flat-CCS aux state when the object is moved between local-memory and |
848 | * system-memory |
849 | * @obj: Pointer to the object |
850 | * |
851 | * Return: True if the object needs extra ccs pages. False otherwise. |
852 | */ |
853 | bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj) |
854 | { |
855 | bool lmem_placement = false; |
856 | int i; |
857 | |
858 | if (!HAS_FLAT_CCS(to_i915(obj->base.dev))) |
859 | return false; |
860 | |
861 | if (obj->flags & I915_BO_ALLOC_CCS_AUX) |
862 | return true; |
863 | |
864 | for (i = 0; i < obj->mm.n_placements; i++) { |
865 | /* Compression is not allowed for the objects with smem placement */ |
866 | if (obj->mm.placements[i]->type == INTEL_MEMORY_SYSTEM) |
867 | return false; |
868 | if (!lmem_placement && |
869 | obj->mm.placements[i]->type == INTEL_MEMORY_LOCAL) |
870 | lmem_placement = true; |
871 | } |
872 | |
873 | return lmem_placement; |
874 | } |
875 | |
876 | void i915_gem_init__objects(struct drm_i915_private *i915) |
877 | { |
878 | INIT_WORK(&i915->mm.free_work, __i915_gem_free_work); |
879 | } |
880 | |
881 | void i915_objects_module_exit(void) |
882 | { |
883 | kmem_cache_destroy(s: slab_objects); |
884 | } |
885 | |
886 | int __init i915_objects_module_init(void) |
887 | { |
888 | slab_objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); |
889 | if (!slab_objects) |
890 | return -ENOMEM; |
891 | |
892 | return 0; |
893 | } |
894 | |
895 | static const struct drm_gem_object_funcs i915_gem_object_funcs = { |
896 | .free = i915_gem_free_object, |
897 | .close = i915_gem_close_object, |
898 | .export = i915_gem_prime_export, |
899 | }; |
900 | |
901 | /** |
902 | * i915_gem_object_get_moving_fence - Get the object's moving fence if any |
903 | * @obj: The object whose moving fence to get. |
904 | * @fence: The resulting fence |
905 | * |
906 | * A non-signaled moving fence means that there is an async operation |
907 | * pending on the object that needs to be waited on before setting up |
908 | * any GPU- or CPU PTEs to the object's pages. |
909 | * |
910 | * Return: Negative error code or 0 for success. |
911 | */ |
912 | int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj, |
913 | struct dma_fence **fence) |
914 | { |
915 | return dma_resv_get_singleton(obj: obj->base.resv, usage: DMA_RESV_USAGE_KERNEL, |
916 | fence); |
917 | } |
918 | |
919 | /** |
920 | * i915_gem_object_wait_moving_fence - Wait for the object's moving fence if any |
921 | * @obj: The object whose moving fence to wait for. |
922 | * @intr: Whether to wait interruptible. |
923 | * |
924 | * If the moving fence signaled without an error, it is detached from the |
925 | * object and put. |
926 | * |
927 | * Return: 0 if successful, -ERESTARTSYS if the wait was interrupted, |
928 | * negative error code if the async operation represented by the |
929 | * moving fence failed. |
930 | */ |
931 | int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj, |
932 | bool intr) |
933 | { |
934 | long ret; |
935 | |
936 | assert_object_held(obj); |
937 | |
938 | ret = dma_resv_wait_timeout(obj: obj->base. resv, usage: DMA_RESV_USAGE_KERNEL, |
939 | intr, MAX_SCHEDULE_TIMEOUT); |
940 | if (!ret) |
941 | ret = -ETIME; |
942 | else if (ret > 0 && i915_gem_object_has_unknown_state(obj)) |
943 | ret = -EIO; |
944 | |
945 | return ret < 0 ? ret : 0; |
946 | } |
947 | |
948 | /* |
949 | * i915_gem_object_has_unknown_state - Return true if the object backing pages are |
950 | * in an unknown_state. This means that userspace must NEVER be allowed to touch |
951 | * the pages, with either the GPU or CPU. |
952 | * |
953 | * ONLY valid to be called after ensuring that all kernel fences have signalled |
954 | * (in particular the fence for moving/clearing the object). |
955 | */ |
956 | bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj) |
957 | { |
958 | /* |
959 | * The below barrier pairs with the dma_fence_signal() in |
960 | * __memcpy_work(). We should only sample the unknown_state after all |
961 | * the kernel fences have signalled. |
962 | */ |
963 | smp_rmb(); |
964 | return obj->mm.unknown_state; |
965 | } |
966 | |
967 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
968 | #include "selftests/huge_gem_object.c" |
969 | #include "selftests/huge_pages.c" |
970 | #include "selftests/i915_gem_migrate.c" |
971 | #include "selftests/i915_gem_object.c" |
972 | #include "selftests/i915_gem_coherency.c" |
973 | #endif |
974 | |