1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * Copyright © 2014-2016 Intel Corporation |
5 | */ |
6 | |
7 | #include <linux/anon_inodes.h> |
8 | #include <linux/mman.h> |
9 | #include <linux/pfn_t.h> |
10 | #include <linux/sizes.h> |
11 | |
12 | #include <drm/drm_cache.h> |
13 | |
14 | #include "gt/intel_gt.h" |
15 | #include "gt/intel_gt_requests.h" |
16 | |
17 | #include "i915_drv.h" |
18 | #include "i915_gem_evict.h" |
19 | #include "i915_gem_gtt.h" |
20 | #include "i915_gem_ioctls.h" |
21 | #include "i915_gem_object.h" |
22 | #include "i915_gem_mman.h" |
23 | #include "i915_mm.h" |
24 | #include "i915_trace.h" |
25 | #include "i915_user_extensions.h" |
26 | #include "i915_gem_ttm.h" |
27 | #include "i915_vma.h" |
28 | |
29 | static inline bool |
30 | __vma_matches(struct vm_area_struct *vma, struct file *filp, |
31 | unsigned long addr, unsigned long size) |
32 | { |
33 | if (vma->vm_file != filp) |
34 | return false; |
35 | |
36 | return vma->vm_start == addr && |
37 | (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); |
38 | } |
39 | |
40 | /** |
41 | * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address |
42 | * it is mapped to. |
43 | * @dev: drm device |
44 | * @data: ioctl data blob |
45 | * @file: drm file |
46 | * |
47 | * While the mapping holds a reference on the contents of the object, it doesn't |
48 | * imply a ref on the object itself. |
49 | * |
50 | * IMPORTANT: |
51 | * |
52 | * DRM driver writers who look a this function as an example for how to do GEM |
53 | * mmap support, please don't implement mmap support like here. The modern way |
54 | * to implement DRM mmap support is with an mmap offset ioctl (like |
55 | * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. |
56 | * That way debug tooling like valgrind will understand what's going on, hiding |
57 | * the mmap call in a driver private ioctl will break that. The i915 driver only |
58 | * does cpu mmaps this way because we didn't know better. |
59 | */ |
60 | int |
61 | i915_gem_mmap_ioctl(struct drm_device *dev, void *data, |
62 | struct drm_file *file) |
63 | { |
64 | struct drm_i915_private *i915 = to_i915(dev); |
65 | struct drm_i915_gem_mmap *args = data; |
66 | struct drm_i915_gem_object *obj; |
67 | unsigned long addr; |
68 | |
69 | /* |
70 | * mmap ioctl is disallowed for all discrete platforms, |
71 | * and for all platforms with GRAPHICS_VER > 12. |
72 | */ |
73 | if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0)) |
74 | return -EOPNOTSUPP; |
75 | |
76 | if (args->flags & ~(I915_MMAP_WC)) |
77 | return -EINVAL; |
78 | |
79 | if (args->flags & I915_MMAP_WC && !pat_enabled()) |
80 | return -ENODEV; |
81 | |
82 | obj = i915_gem_object_lookup(file, handle: args->handle); |
83 | if (!obj) |
84 | return -ENOENT; |
85 | |
86 | /* prime objects have no backing filp to GEM mmap |
87 | * pages from. |
88 | */ |
89 | if (!obj->base.filp) { |
90 | addr = -ENXIO; |
91 | goto err; |
92 | } |
93 | |
94 | if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { |
95 | addr = -EINVAL; |
96 | goto err; |
97 | } |
98 | |
99 | addr = vm_mmap(obj->base.filp, 0, args->size, |
100 | PROT_READ | PROT_WRITE, MAP_SHARED, |
101 | args->offset); |
102 | if (IS_ERR_VALUE(addr)) |
103 | goto err; |
104 | |
105 | if (args->flags & I915_MMAP_WC) { |
106 | struct mm_struct *mm = current->mm; |
107 | struct vm_area_struct *vma; |
108 | |
109 | if (mmap_write_lock_killable(mm)) { |
110 | addr = -EINTR; |
111 | goto err; |
112 | } |
113 | vma = find_vma(mm, addr); |
114 | if (vma && __vma_matches(vma, filp: obj->base.filp, addr, size: args->size)) |
115 | vma->vm_page_prot = |
116 | pgprot_writecombine(prot: vm_get_page_prot(vm_flags: vma->vm_flags)); |
117 | else |
118 | addr = -ENOMEM; |
119 | mmap_write_unlock(mm); |
120 | if (IS_ERR_VALUE(addr)) |
121 | goto err; |
122 | } |
123 | i915_gem_object_put(obj); |
124 | |
125 | args->addr_ptr = (u64)addr; |
126 | return 0; |
127 | |
128 | err: |
129 | i915_gem_object_put(obj); |
130 | return addr; |
131 | } |
132 | |
133 | static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) |
134 | { |
135 | return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; |
136 | } |
137 | |
138 | /** |
139 | * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps |
140 | * |
141 | * A history of the GTT mmap interface: |
142 | * |
143 | * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to |
144 | * aligned and suitable for fencing, and still fit into the available |
145 | * mappable space left by the pinned display objects. A classic problem |
146 | * we called the page-fault-of-doom where we would ping-pong between |
147 | * two objects that could not fit inside the GTT and so the memcpy |
148 | * would page one object in at the expense of the other between every |
149 | * single byte. |
150 | * |
151 | * 1 - Objects can be any size, and have any compatible fencing (X Y, or none |
152 | * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the |
153 | * object is too large for the available space (or simply too large |
154 | * for the mappable aperture!), a view is created instead and faulted |
155 | * into userspace. (This view is aligned and sized appropriately for |
156 | * fenced access.) |
157 | * |
158 | * 2 - Recognise WC as a separate cache domain so that we can flush the |
159 | * delayed writes via GTT before performing direct access via WC. |
160 | * |
161 | * 3 - Remove implicit set-domain(GTT) and synchronisation on initial |
162 | * pagefault; swapin remains transparent. |
163 | * |
164 | * 4 - Support multiple fault handlers per object depending on object's |
165 | * backing storage (a.k.a. MMAP_OFFSET). |
166 | * |
167 | * Restrictions: |
168 | * |
169 | * * snoopable objects cannot be accessed via the GTT. It can cause machine |
170 | * hangs on some architectures, corruption on others. An attempt to service |
171 | * a GTT page fault from a snoopable object will generate a SIGBUS. |
172 | * |
173 | * * the object must be able to fit into RAM (physical memory, though no |
174 | * limited to the mappable aperture). |
175 | * |
176 | * |
177 | * Caveats: |
178 | * |
179 | * * a new GTT page fault will synchronize rendering from the GPU and flush |
180 | * all data to system memory. Subsequent access will not be synchronized. |
181 | * |
182 | * * all mappings are revoked on runtime device suspend. |
183 | * |
184 | * * there are only 8, 16 or 32 fence registers to share between all users |
185 | * (older machines require fence register for display and blitter access |
186 | * as well). Contention of the fence registers will cause the previous users |
187 | * to be unmapped and any new access will generate new page faults. |
188 | * |
189 | * * running out of memory while servicing a fault may generate a SIGBUS, |
190 | * rather than the expected SIGSEGV. |
191 | */ |
192 | int i915_gem_mmap_gtt_version(void) |
193 | { |
194 | return 4; |
195 | } |
196 | |
197 | static inline struct i915_gtt_view |
198 | compute_partial_view(const struct drm_i915_gem_object *obj, |
199 | pgoff_t page_offset, |
200 | unsigned int chunk) |
201 | { |
202 | struct i915_gtt_view view; |
203 | |
204 | if (i915_gem_object_is_tiled(obj)) |
205 | chunk = roundup(chunk, tile_row_pages(obj) ?: 1); |
206 | |
207 | view.type = I915_GTT_VIEW_PARTIAL; |
208 | view.partial.offset = rounddown(page_offset, chunk); |
209 | view.partial.size = |
210 | min_t(unsigned int, chunk, |
211 | (obj->base.size >> PAGE_SHIFT) - view.partial.offset); |
212 | |
213 | /* If the partial covers the entire object, just create a normal VMA. */ |
214 | if (chunk >= obj->base.size >> PAGE_SHIFT) |
215 | view.type = I915_GTT_VIEW_NORMAL; |
216 | |
217 | return view; |
218 | } |
219 | |
220 | static vm_fault_t i915_error_to_vmf_fault(int err) |
221 | { |
222 | switch (err) { |
223 | default: |
224 | WARN_ONCE(err, "unhandled error in %s: %i\n" , __func__, err); |
225 | fallthrough; |
226 | case -EIO: /* shmemfs failure from swap device */ |
227 | case -EFAULT: /* purged object */ |
228 | case -ENODEV: /* bad object, how did you get here! */ |
229 | case -ENXIO: /* unable to access backing store (on device) */ |
230 | return VM_FAULT_SIGBUS; |
231 | |
232 | case -ENOMEM: /* our allocation failure */ |
233 | return VM_FAULT_OOM; |
234 | |
235 | case 0: |
236 | case -EAGAIN: |
237 | case -ENOSPC: /* transient failure to evict? */ |
238 | case -ENOBUFS: /* temporarily out of fences? */ |
239 | case -ERESTARTSYS: |
240 | case -EINTR: |
241 | case -EBUSY: |
242 | /* |
243 | * EBUSY is ok: this just means that another thread |
244 | * already did the job. |
245 | */ |
246 | return VM_FAULT_NOPAGE; |
247 | } |
248 | } |
249 | |
250 | static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) |
251 | { |
252 | struct vm_area_struct *area = vmf->vma; |
253 | struct i915_mmap_offset *mmo = area->vm_private_data; |
254 | struct drm_i915_gem_object *obj = mmo->obj; |
255 | resource_size_t iomap; |
256 | int err; |
257 | |
258 | /* Sanity check that we allow writing into this object */ |
259 | if (unlikely(i915_gem_object_is_readonly(obj) && |
260 | area->vm_flags & VM_WRITE)) |
261 | return VM_FAULT_SIGBUS; |
262 | |
263 | if (i915_gem_object_lock_interruptible(obj, NULL)) |
264 | return VM_FAULT_NOPAGE; |
265 | |
266 | err = i915_gem_object_pin_pages(obj); |
267 | if (err) |
268 | goto out; |
269 | |
270 | iomap = -1; |
271 | if (!i915_gem_object_has_struct_page(obj)) { |
272 | iomap = obj->mm.region->iomap.base; |
273 | iomap -= obj->mm.region->region.start; |
274 | } |
275 | |
276 | /* PTEs are revoked in obj->ops->put_pages() */ |
277 | err = remap_io_sg(vma: area, |
278 | addr: area->vm_start, size: area->vm_end - area->vm_start, |
279 | sgl: obj->mm.pages->sgl, iobase: iomap); |
280 | |
281 | if (area->vm_flags & VM_WRITE) { |
282 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); |
283 | obj->mm.dirty = true; |
284 | } |
285 | |
286 | i915_gem_object_unpin_pages(obj); |
287 | |
288 | out: |
289 | i915_gem_object_unlock(obj); |
290 | return i915_error_to_vmf_fault(err); |
291 | } |
292 | |
293 | static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) |
294 | { |
295 | #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) |
296 | struct vm_area_struct *area = vmf->vma; |
297 | struct i915_mmap_offset *mmo = area->vm_private_data; |
298 | struct drm_i915_gem_object *obj = mmo->obj; |
299 | struct drm_device *dev = obj->base.dev; |
300 | struct drm_i915_private *i915 = to_i915(dev); |
301 | struct intel_runtime_pm *rpm = &i915->runtime_pm; |
302 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
303 | bool write = area->vm_flags & VM_WRITE; |
304 | struct i915_gem_ww_ctx ww; |
305 | intel_wakeref_t wakeref; |
306 | struct i915_vma *vma; |
307 | pgoff_t page_offset; |
308 | int srcu; |
309 | int ret; |
310 | |
311 | /* We don't use vmf->pgoff since that has the fake offset */ |
312 | page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; |
313 | |
314 | trace_i915_gem_object_fault(obj, index: page_offset, gtt: true, write); |
315 | |
316 | wakeref = intel_runtime_pm_get(rpm); |
317 | |
318 | i915_gem_ww_ctx_init(ctx: &ww, intr: true); |
319 | retry: |
320 | ret = i915_gem_object_lock(obj, ww: &ww); |
321 | if (ret) |
322 | goto err_rpm; |
323 | |
324 | /* Sanity check that we allow writing into this object */ |
325 | if (i915_gem_object_is_readonly(obj) && write) { |
326 | ret = -EFAULT; |
327 | goto err_rpm; |
328 | } |
329 | |
330 | ret = i915_gem_object_pin_pages(obj); |
331 | if (ret) |
332 | goto err_rpm; |
333 | |
334 | ret = intel_gt_reset_lock_interruptible(gt: ggtt->vm.gt, srcu: &srcu); |
335 | if (ret) |
336 | goto err_pages; |
337 | |
338 | /* Now pin it into the GTT as needed */ |
339 | vma = i915_gem_object_ggtt_pin_ww(obj, ww: &ww, NULL, size: 0, alignment: 0, |
340 | PIN_MAPPABLE | |
341 | PIN_NONBLOCK /* NOWARN */ | |
342 | PIN_NOEVICT); |
343 | if (IS_ERR(ptr: vma) && vma != ERR_PTR(error: -EDEADLK)) { |
344 | /* Use a partial view if it is bigger than available space */ |
345 | struct i915_gtt_view view = |
346 | compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); |
347 | unsigned int flags; |
348 | |
349 | flags = PIN_MAPPABLE | PIN_NOSEARCH; |
350 | if (view.type == I915_GTT_VIEW_NORMAL) |
351 | flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ |
352 | |
353 | /* |
354 | * Userspace is now writing through an untracked VMA, abandon |
355 | * all hope that the hardware is able to track future writes. |
356 | */ |
357 | |
358 | vma = i915_gem_object_ggtt_pin_ww(obj, ww: &ww, view: &view, size: 0, alignment: 0, flags); |
359 | if (IS_ERR(ptr: vma) && vma != ERR_PTR(error: -EDEADLK)) { |
360 | flags = PIN_MAPPABLE; |
361 | view.type = I915_GTT_VIEW_PARTIAL; |
362 | vma = i915_gem_object_ggtt_pin_ww(obj, ww: &ww, view: &view, size: 0, alignment: 0, flags); |
363 | } |
364 | |
365 | /* |
366 | * The entire mappable GGTT is pinned? Unexpected! |
367 | * Try to evict the object we locked too, as normally we skip it |
368 | * due to lack of short term pinning inside execbuf. |
369 | */ |
370 | if (vma == ERR_PTR(error: -ENOSPC)) { |
371 | ret = mutex_lock_interruptible(&ggtt->vm.mutex); |
372 | if (!ret) { |
373 | ret = i915_gem_evict_vm(vm: &ggtt->vm, ww: &ww, NULL); |
374 | mutex_unlock(lock: &ggtt->vm.mutex); |
375 | } |
376 | if (ret) |
377 | goto err_reset; |
378 | vma = i915_gem_object_ggtt_pin_ww(obj, ww: &ww, view: &view, size: 0, alignment: 0, flags); |
379 | } |
380 | } |
381 | if (IS_ERR(ptr: vma)) { |
382 | ret = PTR_ERR(ptr: vma); |
383 | goto err_reset; |
384 | } |
385 | |
386 | /* Access to snoopable pages through the GTT is incoherent. */ |
387 | /* |
388 | * For objects created by userspace through GEM_CREATE with pat_index |
389 | * set by set_pat extension, coherency is managed by userspace, make |
390 | * sure we don't fail handling the vm fault by calling |
391 | * i915_gem_object_has_cache_level() which always return true for such |
392 | * objects. Otherwise this helper function would fall back to checking |
393 | * whether the object is un-cached. |
394 | */ |
395 | if (!(i915_gem_object_has_cache_level(obj, lvl: I915_CACHE_NONE) || |
396 | HAS_LLC(i915))) { |
397 | ret = -EFAULT; |
398 | goto err_unpin; |
399 | } |
400 | |
401 | ret = i915_vma_pin_fence(vma); |
402 | if (ret) |
403 | goto err_unpin; |
404 | |
405 | /* Finally, remap it using the new GTT offset */ |
406 | ret = remap_io_mapping(vma: area, |
407 | addr: area->vm_start + (vma->gtt_view.partial.offset << PAGE_SHIFT), |
408 | pfn: (ggtt->gmadr.start + i915_ggtt_offset(vma)) >> PAGE_SHIFT, |
409 | min_t(u64, vma->size, area->vm_end - area->vm_start), |
410 | iomap: &ggtt->iomap); |
411 | if (ret) |
412 | goto err_fence; |
413 | |
414 | assert_rpm_wakelock_held(rpm); |
415 | |
416 | /* Mark as being mmapped into userspace for later revocation */ |
417 | mutex_lock(&to_gt(i915)->ggtt->vm.mutex); |
418 | if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) |
419 | list_add(new: &obj->userfault_link, head: &to_gt(i915)->ggtt->userfault_list); |
420 | mutex_unlock(lock: &to_gt(i915)->ggtt->vm.mutex); |
421 | |
422 | /* Track the mmo associated with the fenced vma */ |
423 | vma->mmo = mmo; |
424 | |
425 | if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) |
426 | intel_wakeref_auto(wf: &i915->runtime_pm.userfault_wakeref, |
427 | timeout: msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
428 | |
429 | if (write) { |
430 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); |
431 | i915_vma_set_ggtt_write(vma); |
432 | obj->mm.dirty = true; |
433 | } |
434 | |
435 | err_fence: |
436 | i915_vma_unpin_fence(vma); |
437 | err_unpin: |
438 | __i915_vma_unpin(vma); |
439 | err_reset: |
440 | intel_gt_reset_unlock(gt: ggtt->vm.gt, tag: srcu); |
441 | err_pages: |
442 | i915_gem_object_unpin_pages(obj); |
443 | err_rpm: |
444 | if (ret == -EDEADLK) { |
445 | ret = i915_gem_ww_ctx_backoff(ctx: &ww); |
446 | if (!ret) |
447 | goto retry; |
448 | } |
449 | i915_gem_ww_ctx_fini(ctx: &ww); |
450 | intel_runtime_pm_put(rpm, wref: wakeref); |
451 | return i915_error_to_vmf_fault(err: ret); |
452 | } |
453 | |
454 | static int |
455 | vm_access(struct vm_area_struct *area, unsigned long addr, |
456 | void *buf, int len, int write) |
457 | { |
458 | struct i915_mmap_offset *mmo = area->vm_private_data; |
459 | struct drm_i915_gem_object *obj = mmo->obj; |
460 | struct i915_gem_ww_ctx ww; |
461 | void *vaddr; |
462 | int err = 0; |
463 | |
464 | if (i915_gem_object_is_readonly(obj) && write) |
465 | return -EACCES; |
466 | |
467 | addr -= area->vm_start; |
468 | if (range_overflows_t(u64, addr, len, obj->base.size)) |
469 | return -EINVAL; |
470 | |
471 | i915_gem_ww_ctx_init(ctx: &ww, intr: true); |
472 | retry: |
473 | err = i915_gem_object_lock(obj, ww: &ww); |
474 | if (err) |
475 | goto out; |
476 | |
477 | /* As this is primarily for debugging, let's focus on simplicity */ |
478 | vaddr = i915_gem_object_pin_map(obj, type: I915_MAP_FORCE_WC); |
479 | if (IS_ERR(ptr: vaddr)) { |
480 | err = PTR_ERR(ptr: vaddr); |
481 | goto out; |
482 | } |
483 | |
484 | if (write) { |
485 | memcpy(vaddr + addr, buf, len); |
486 | __i915_gem_object_flush_map(obj, offset: addr, size: len); |
487 | } else { |
488 | memcpy(buf, vaddr + addr, len); |
489 | } |
490 | |
491 | i915_gem_object_unpin_map(obj); |
492 | out: |
493 | if (err == -EDEADLK) { |
494 | err = i915_gem_ww_ctx_backoff(ctx: &ww); |
495 | if (!err) |
496 | goto retry; |
497 | } |
498 | i915_gem_ww_ctx_fini(ctx: &ww); |
499 | |
500 | if (err) |
501 | return err; |
502 | |
503 | return len; |
504 | } |
505 | |
506 | void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
507 | { |
508 | struct i915_vma *vma; |
509 | |
510 | GEM_BUG_ON(!obj->userfault_count); |
511 | |
512 | for_each_ggtt_vma(vma, obj) |
513 | i915_vma_revoke_mmap(vma); |
514 | |
515 | GEM_BUG_ON(obj->userfault_count); |
516 | } |
517 | |
518 | /* |
519 | * It is vital that we remove the page mapping if we have mapped a tiled |
520 | * object through the GTT and then lose the fence register due to |
521 | * resource pressure. Similarly if the object has been moved out of the |
522 | * aperture, than pages mapped into userspace must be revoked. Removing the |
523 | * mapping will then trigger a page fault on the next user access, allowing |
524 | * fixup by vm_fault_gtt(). |
525 | */ |
526 | void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
527 | { |
528 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
529 | intel_wakeref_t wakeref; |
530 | |
531 | /* |
532 | * Serialisation between user GTT access and our code depends upon |
533 | * revoking the CPU's PTE whilst the mutex is held. The next user |
534 | * pagefault then has to wait until we release the mutex. |
535 | * |
536 | * Note that RPM complicates somewhat by adding an additional |
537 | * requirement that operations to the GGTT be made holding the RPM |
538 | * wakeref. |
539 | */ |
540 | wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm); |
541 | mutex_lock(&to_gt(i915)->ggtt->vm.mutex); |
542 | |
543 | if (!obj->userfault_count) |
544 | goto out; |
545 | |
546 | __i915_gem_object_release_mmap_gtt(obj); |
547 | |
548 | /* |
549 | * Ensure that the CPU's PTE are revoked and there are not outstanding |
550 | * memory transactions from userspace before we return. The TLB |
551 | * flushing implied above by changing the PTE above *should* be |
552 | * sufficient, an extra barrier here just provides us with a bit |
553 | * of paranoid documentation about our requirement to serialise |
554 | * memory writes before touching registers / GSM. |
555 | */ |
556 | wmb(); |
557 | |
558 | out: |
559 | mutex_unlock(lock: &to_gt(i915)->ggtt->vm.mutex); |
560 | intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref); |
561 | } |
562 | |
563 | void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj) |
564 | { |
565 | struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); |
566 | struct ttm_device *bdev = bo->bdev; |
567 | |
568 | drm_vma_node_unmap(node: &bo->base.vma_node, file_mapping: bdev->dev_mapping); |
569 | |
570 | /* |
571 | * We have exclusive access here via runtime suspend. All other callers |
572 | * must first grab the rpm wakeref. |
573 | */ |
574 | GEM_BUG_ON(!obj->userfault_count); |
575 | list_del(entry: &obj->userfault_link); |
576 | obj->userfault_count = 0; |
577 | } |
578 | |
579 | void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) |
580 | { |
581 | struct i915_mmap_offset *mmo, *mn; |
582 | |
583 | if (obj->ops->unmap_virtual) |
584 | obj->ops->unmap_virtual(obj); |
585 | |
586 | spin_lock(lock: &obj->mmo.lock); |
587 | rbtree_postorder_for_each_entry_safe(mmo, mn, |
588 | &obj->mmo.offsets, offset) { |
589 | /* |
590 | * vma_node_unmap for GTT mmaps handled already in |
591 | * __i915_gem_object_release_mmap_gtt |
592 | */ |
593 | if (mmo->mmap_type == I915_MMAP_TYPE_GTT) |
594 | continue; |
595 | |
596 | spin_unlock(lock: &obj->mmo.lock); |
597 | drm_vma_node_unmap(node: &mmo->vma_node, |
598 | file_mapping: obj->base.dev->anon_inode->i_mapping); |
599 | spin_lock(lock: &obj->mmo.lock); |
600 | } |
601 | spin_unlock(lock: &obj->mmo.lock); |
602 | } |
603 | |
604 | static struct i915_mmap_offset * |
605 | lookup_mmo(struct drm_i915_gem_object *obj, |
606 | enum i915_mmap_type mmap_type) |
607 | { |
608 | struct rb_node *rb; |
609 | |
610 | spin_lock(lock: &obj->mmo.lock); |
611 | rb = obj->mmo.offsets.rb_node; |
612 | while (rb) { |
613 | struct i915_mmap_offset *mmo = |
614 | rb_entry(rb, typeof(*mmo), offset); |
615 | |
616 | if (mmo->mmap_type == mmap_type) { |
617 | spin_unlock(lock: &obj->mmo.lock); |
618 | return mmo; |
619 | } |
620 | |
621 | if (mmo->mmap_type < mmap_type) |
622 | rb = rb->rb_right; |
623 | else |
624 | rb = rb->rb_left; |
625 | } |
626 | spin_unlock(lock: &obj->mmo.lock); |
627 | |
628 | return NULL; |
629 | } |
630 | |
631 | static struct i915_mmap_offset * |
632 | insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo) |
633 | { |
634 | struct rb_node *rb, **p; |
635 | |
636 | spin_lock(lock: &obj->mmo.lock); |
637 | rb = NULL; |
638 | p = &obj->mmo.offsets.rb_node; |
639 | while (*p) { |
640 | struct i915_mmap_offset *pos; |
641 | |
642 | rb = *p; |
643 | pos = rb_entry(rb, typeof(*pos), offset); |
644 | |
645 | if (pos->mmap_type == mmo->mmap_type) { |
646 | spin_unlock(lock: &obj->mmo.lock); |
647 | drm_vma_offset_remove(mgr: obj->base.dev->vma_offset_manager, |
648 | node: &mmo->vma_node); |
649 | kfree(objp: mmo); |
650 | return pos; |
651 | } |
652 | |
653 | if (pos->mmap_type < mmo->mmap_type) |
654 | p = &rb->rb_right; |
655 | else |
656 | p = &rb->rb_left; |
657 | } |
658 | rb_link_node(node: &mmo->offset, parent: rb, rb_link: p); |
659 | rb_insert_color(&mmo->offset, &obj->mmo.offsets); |
660 | spin_unlock(lock: &obj->mmo.lock); |
661 | |
662 | return mmo; |
663 | } |
664 | |
665 | static struct i915_mmap_offset * |
666 | mmap_offset_attach(struct drm_i915_gem_object *obj, |
667 | enum i915_mmap_type mmap_type, |
668 | struct drm_file *file) |
669 | { |
670 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
671 | struct i915_mmap_offset *mmo; |
672 | int err; |
673 | |
674 | GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops); |
675 | |
676 | mmo = lookup_mmo(obj, mmap_type); |
677 | if (mmo) |
678 | goto out; |
679 | |
680 | mmo = kmalloc(size: sizeof(*mmo), GFP_KERNEL); |
681 | if (!mmo) |
682 | return ERR_PTR(error: -ENOMEM); |
683 | |
684 | mmo->obj = obj; |
685 | mmo->mmap_type = mmap_type; |
686 | drm_vma_node_reset(node: &mmo->vma_node); |
687 | |
688 | err = drm_vma_offset_add(mgr: obj->base.dev->vma_offset_manager, |
689 | node: &mmo->vma_node, pages: obj->base.size / PAGE_SIZE); |
690 | if (likely(!err)) |
691 | goto insert; |
692 | |
693 | /* Attempt to reap some mmap space from dead objects */ |
694 | err = intel_gt_retire_requests_timeout(gt: to_gt(i915), MAX_SCHEDULE_TIMEOUT, |
695 | NULL); |
696 | if (err) |
697 | goto err; |
698 | |
699 | i915_gem_drain_freed_objects(i915); |
700 | err = drm_vma_offset_add(mgr: obj->base.dev->vma_offset_manager, |
701 | node: &mmo->vma_node, pages: obj->base.size / PAGE_SIZE); |
702 | if (err) |
703 | goto err; |
704 | |
705 | insert: |
706 | mmo = insert_mmo(obj, mmo); |
707 | GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo); |
708 | out: |
709 | if (file) |
710 | drm_vma_node_allow_once(node: &mmo->vma_node, tag: file); |
711 | return mmo; |
712 | |
713 | err: |
714 | kfree(objp: mmo); |
715 | return ERR_PTR(error: err); |
716 | } |
717 | |
718 | static int |
719 | __assign_mmap_offset(struct drm_i915_gem_object *obj, |
720 | enum i915_mmap_type mmap_type, |
721 | u64 *offset, struct drm_file *file) |
722 | { |
723 | struct i915_mmap_offset *mmo; |
724 | |
725 | if (i915_gem_object_never_mmap(obj)) |
726 | return -ENODEV; |
727 | |
728 | if (obj->ops->mmap_offset) { |
729 | if (mmap_type != I915_MMAP_TYPE_FIXED) |
730 | return -ENODEV; |
731 | |
732 | *offset = obj->ops->mmap_offset(obj); |
733 | return 0; |
734 | } |
735 | |
736 | if (mmap_type == I915_MMAP_TYPE_FIXED) |
737 | return -ENODEV; |
738 | |
739 | if (mmap_type != I915_MMAP_TYPE_GTT && |
740 | !i915_gem_object_has_struct_page(obj) && |
741 | !i915_gem_object_has_iomem(obj)) |
742 | return -ENODEV; |
743 | |
744 | mmo = mmap_offset_attach(obj, mmap_type, file); |
745 | if (IS_ERR(ptr: mmo)) |
746 | return PTR_ERR(ptr: mmo); |
747 | |
748 | *offset = drm_vma_node_offset_addr(node: &mmo->vma_node); |
749 | return 0; |
750 | } |
751 | |
752 | static int |
753 | __assign_mmap_offset_handle(struct drm_file *file, |
754 | u32 handle, |
755 | enum i915_mmap_type mmap_type, |
756 | u64 *offset) |
757 | { |
758 | struct drm_i915_gem_object *obj; |
759 | int err; |
760 | |
761 | obj = i915_gem_object_lookup(file, handle); |
762 | if (!obj) |
763 | return -ENOENT; |
764 | |
765 | err = i915_gem_object_lock_interruptible(obj, NULL); |
766 | if (err) |
767 | goto out_put; |
768 | err = __assign_mmap_offset(obj, mmap_type, offset, file); |
769 | i915_gem_object_unlock(obj); |
770 | out_put: |
771 | i915_gem_object_put(obj); |
772 | return err; |
773 | } |
774 | |
775 | int |
776 | i915_gem_dumb_mmap_offset(struct drm_file *file, |
777 | struct drm_device *dev, |
778 | u32 handle, |
779 | u64 *offset) |
780 | { |
781 | struct drm_i915_private *i915 = to_i915(dev); |
782 | enum i915_mmap_type mmap_type; |
783 | |
784 | if (HAS_LMEM(to_i915(dev))) |
785 | mmap_type = I915_MMAP_TYPE_FIXED; |
786 | else if (pat_enabled()) |
787 | mmap_type = I915_MMAP_TYPE_WC; |
788 | else if (!i915_ggtt_has_aperture(ggtt: to_gt(i915)->ggtt)) |
789 | return -ENODEV; |
790 | else |
791 | mmap_type = I915_MMAP_TYPE_GTT; |
792 | |
793 | return __assign_mmap_offset_handle(file, handle, mmap_type, offset); |
794 | } |
795 | |
796 | /** |
797 | * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing |
798 | * @dev: DRM device |
799 | * @data: GTT mapping ioctl data |
800 | * @file: GEM object info |
801 | * |
802 | * Simply returns the fake offset to userspace so it can mmap it. |
803 | * The mmap call will end up in drm_gem_mmap(), which will set things |
804 | * up so we can get faults in the handler above. |
805 | * |
806 | * The fault handler will take care of binding the object into the GTT |
807 | * (since it may have been evicted to make room for something), allocating |
808 | * a fence register, and mapping the appropriate aperture address into |
809 | * userspace. |
810 | */ |
811 | int |
812 | i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, |
813 | struct drm_file *file) |
814 | { |
815 | struct drm_i915_private *i915 = to_i915(dev); |
816 | struct drm_i915_gem_mmap_offset *args = data; |
817 | enum i915_mmap_type type; |
818 | int err; |
819 | |
820 | /* |
821 | * Historically we failed to check args.pad and args.offset |
822 | * and so we cannot use those fields for user input and we cannot |
823 | * add -EINVAL for them as the ABI is fixed, i.e. old userspace |
824 | * may be feeding in garbage in those fields. |
825 | * |
826 | * if (args->pad) return -EINVAL; is verbotten! |
827 | */ |
828 | |
829 | err = i915_user_extensions(u64_to_user_ptr(args->extensions), |
830 | NULL, count: 0, NULL); |
831 | if (err) |
832 | return err; |
833 | |
834 | switch (args->flags) { |
835 | case I915_MMAP_OFFSET_GTT: |
836 | if (!i915_ggtt_has_aperture(ggtt: to_gt(i915)->ggtt)) |
837 | return -ENODEV; |
838 | type = I915_MMAP_TYPE_GTT; |
839 | break; |
840 | |
841 | case I915_MMAP_OFFSET_WC: |
842 | if (!pat_enabled()) |
843 | return -ENODEV; |
844 | type = I915_MMAP_TYPE_WC; |
845 | break; |
846 | |
847 | case I915_MMAP_OFFSET_WB: |
848 | type = I915_MMAP_TYPE_WB; |
849 | break; |
850 | |
851 | case I915_MMAP_OFFSET_UC: |
852 | if (!pat_enabled()) |
853 | return -ENODEV; |
854 | type = I915_MMAP_TYPE_UC; |
855 | break; |
856 | |
857 | case I915_MMAP_OFFSET_FIXED: |
858 | type = I915_MMAP_TYPE_FIXED; |
859 | break; |
860 | |
861 | default: |
862 | return -EINVAL; |
863 | } |
864 | |
865 | return __assign_mmap_offset_handle(file, handle: args->handle, mmap_type: type, offset: &args->offset); |
866 | } |
867 | |
868 | static void vm_open(struct vm_area_struct *vma) |
869 | { |
870 | struct i915_mmap_offset *mmo = vma->vm_private_data; |
871 | struct drm_i915_gem_object *obj = mmo->obj; |
872 | |
873 | GEM_BUG_ON(!obj); |
874 | i915_gem_object_get(obj); |
875 | } |
876 | |
877 | static void vm_close(struct vm_area_struct *vma) |
878 | { |
879 | struct i915_mmap_offset *mmo = vma->vm_private_data; |
880 | struct drm_i915_gem_object *obj = mmo->obj; |
881 | |
882 | GEM_BUG_ON(!obj); |
883 | i915_gem_object_put(obj); |
884 | } |
885 | |
886 | static const struct vm_operations_struct vm_ops_gtt = { |
887 | .fault = vm_fault_gtt, |
888 | .access = vm_access, |
889 | .open = vm_open, |
890 | .close = vm_close, |
891 | }; |
892 | |
893 | static const struct vm_operations_struct vm_ops_cpu = { |
894 | .fault = vm_fault_cpu, |
895 | .access = vm_access, |
896 | .open = vm_open, |
897 | .close = vm_close, |
898 | }; |
899 | |
900 | static int singleton_release(struct inode *inode, struct file *file) |
901 | { |
902 | struct drm_i915_private *i915 = file->private_data; |
903 | |
904 | cmpxchg(&i915->gem.mmap_singleton, file, NULL); |
905 | drm_dev_put(dev: &i915->drm); |
906 | |
907 | return 0; |
908 | } |
909 | |
910 | static const struct file_operations singleton_fops = { |
911 | .owner = THIS_MODULE, |
912 | .release = singleton_release, |
913 | }; |
914 | |
915 | static struct file *mmap_singleton(struct drm_i915_private *i915) |
916 | { |
917 | struct file *file; |
918 | |
919 | file = get_file_active(f: &i915->gem.mmap_singleton); |
920 | if (file) |
921 | return file; |
922 | |
923 | file = anon_inode_getfile(name: "i915.gem" , fops: &singleton_fops, priv: i915, O_RDWR); |
924 | if (IS_ERR(ptr: file)) |
925 | return file; |
926 | |
927 | /* Everyone shares a single global address space */ |
928 | file->f_mapping = i915->drm.anon_inode->i_mapping; |
929 | |
930 | smp_store_mb(i915->gem.mmap_singleton, file); |
931 | drm_dev_get(dev: &i915->drm); |
932 | |
933 | return file; |
934 | } |
935 | |
936 | static int |
937 | i915_gem_object_mmap(struct drm_i915_gem_object *obj, |
938 | struct i915_mmap_offset *mmo, |
939 | struct vm_area_struct *vma) |
940 | { |
941 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
942 | struct drm_device *dev = &i915->drm; |
943 | struct file *anon; |
944 | |
945 | if (i915_gem_object_is_readonly(obj)) { |
946 | if (vma->vm_flags & VM_WRITE) { |
947 | i915_gem_object_put(obj); |
948 | return -EINVAL; |
949 | } |
950 | vm_flags_clear(vma, VM_MAYWRITE); |
951 | } |
952 | |
953 | anon = mmap_singleton(i915: to_i915(dev)); |
954 | if (IS_ERR(ptr: anon)) { |
955 | i915_gem_object_put(obj); |
956 | return PTR_ERR(ptr: anon); |
957 | } |
958 | |
959 | vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); |
960 | |
961 | /* |
962 | * We keep the ref on mmo->obj, not vm_file, but we require |
963 | * vma->vm_file->f_mapping, see vma_link(), for later revocation. |
964 | * Our userspace is accustomed to having per-file resource cleanup |
965 | * (i.e. contexts, objects and requests) on their close(fd), which |
966 | * requires avoiding extraneous references to their filp, hence why |
967 | * we prefer to use an anonymous file for their mmaps. |
968 | */ |
969 | vma_set_file(vma, file: anon); |
970 | /* Drop the initial creation reference, the vma is now holding one. */ |
971 | fput(anon); |
972 | |
973 | if (obj->ops->mmap_ops) { |
974 | vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); |
975 | vma->vm_ops = obj->ops->mmap_ops; |
976 | vma->vm_private_data = obj->base.vma_node.driver_private; |
977 | return 0; |
978 | } |
979 | |
980 | vma->vm_private_data = mmo; |
981 | |
982 | switch (mmo->mmap_type) { |
983 | case I915_MMAP_TYPE_WC: |
984 | vma->vm_page_prot = |
985 | pgprot_writecombine(prot: vm_get_page_prot(vm_flags: vma->vm_flags)); |
986 | vma->vm_ops = &vm_ops_cpu; |
987 | break; |
988 | |
989 | case I915_MMAP_TYPE_FIXED: |
990 | GEM_WARN_ON(1); |
991 | fallthrough; |
992 | case I915_MMAP_TYPE_WB: |
993 | vma->vm_page_prot = vm_get_page_prot(vm_flags: vma->vm_flags); |
994 | vma->vm_ops = &vm_ops_cpu; |
995 | break; |
996 | |
997 | case I915_MMAP_TYPE_UC: |
998 | vma->vm_page_prot = |
999 | pgprot_noncached(vm_get_page_prot(vma->vm_flags)); |
1000 | vma->vm_ops = &vm_ops_cpu; |
1001 | break; |
1002 | |
1003 | case I915_MMAP_TYPE_GTT: |
1004 | vma->vm_page_prot = |
1005 | pgprot_writecombine(prot: vm_get_page_prot(vm_flags: vma->vm_flags)); |
1006 | vma->vm_ops = &vm_ops_gtt; |
1007 | break; |
1008 | } |
1009 | vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); |
1010 | |
1011 | return 0; |
1012 | } |
1013 | |
1014 | /* |
1015 | * This overcomes the limitation in drm_gem_mmap's assignment of a |
1016 | * drm_gem_object as the vma->vm_private_data. Since we need to |
1017 | * be able to resolve multiple mmap offsets which could be tied |
1018 | * to a single gem object. |
1019 | */ |
1020 | int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) |
1021 | { |
1022 | struct drm_vma_offset_node *node; |
1023 | struct drm_file *priv = filp->private_data; |
1024 | struct drm_device *dev = priv->minor->dev; |
1025 | struct drm_i915_gem_object *obj = NULL; |
1026 | struct i915_mmap_offset *mmo = NULL; |
1027 | |
1028 | if (drm_dev_is_unplugged(dev)) |
1029 | return -ENODEV; |
1030 | |
1031 | rcu_read_lock(); |
1032 | drm_vma_offset_lock_lookup(mgr: dev->vma_offset_manager); |
1033 | node = drm_vma_offset_exact_lookup_locked(mgr: dev->vma_offset_manager, |
1034 | start: vma->vm_pgoff, |
1035 | pages: vma_pages(vma)); |
1036 | if (node && drm_vma_node_is_allowed(node, tag: priv)) { |
1037 | /* |
1038 | * Skip 0-refcnted objects as it is in the process of being |
1039 | * destroyed and will be invalid when the vma manager lock |
1040 | * is released. |
1041 | */ |
1042 | if (!node->driver_private) { |
1043 | mmo = container_of(node, struct i915_mmap_offset, vma_node); |
1044 | obj = i915_gem_object_get_rcu(obj: mmo->obj); |
1045 | |
1046 | GEM_BUG_ON(obj && obj->ops->mmap_ops); |
1047 | } else { |
1048 | obj = i915_gem_object_get_rcu |
1049 | (container_of(node, struct drm_i915_gem_object, |
1050 | base.vma_node)); |
1051 | |
1052 | GEM_BUG_ON(obj && !obj->ops->mmap_ops); |
1053 | } |
1054 | } |
1055 | drm_vma_offset_unlock_lookup(mgr: dev->vma_offset_manager); |
1056 | rcu_read_unlock(); |
1057 | if (!obj) |
1058 | return node ? -EACCES : -EINVAL; |
1059 | |
1060 | return i915_gem_object_mmap(obj, mmo, vma); |
1061 | } |
1062 | |
1063 | int i915_gem_fb_mmap(struct drm_i915_gem_object *obj, struct vm_area_struct *vma) |
1064 | { |
1065 | struct drm_i915_private *i915 = to_i915(dev: obj->base.dev); |
1066 | struct drm_device *dev = &i915->drm; |
1067 | struct i915_mmap_offset *mmo = NULL; |
1068 | enum i915_mmap_type mmap_type; |
1069 | struct i915_ggtt *ggtt = to_gt(i915)->ggtt; |
1070 | |
1071 | if (drm_dev_is_unplugged(dev)) |
1072 | return -ENODEV; |
1073 | |
1074 | /* handle ttm object */ |
1075 | if (obj->ops->mmap_ops) { |
1076 | /* |
1077 | * ttm fault handler, ttm_bo_vm_fault_reserved() uses fake offset |
1078 | * to calculate page offset so set that up. |
1079 | */ |
1080 | vma->vm_pgoff += drm_vma_node_start(node: &obj->base.vma_node); |
1081 | } else { |
1082 | /* handle stolen and smem objects */ |
1083 | mmap_type = i915_ggtt_has_aperture(ggtt) ? I915_MMAP_TYPE_GTT : I915_MMAP_TYPE_WC; |
1084 | mmo = mmap_offset_attach(obj, mmap_type, NULL); |
1085 | if (IS_ERR(ptr: mmo)) |
1086 | return PTR_ERR(ptr: mmo); |
1087 | } |
1088 | |
1089 | /* |
1090 | * When we install vm_ops for mmap we are too late for |
1091 | * the vm_ops->open() which increases the ref_count of |
1092 | * this obj and then it gets decreased by the vm_ops->close(). |
1093 | * To balance this increase the obj ref_count here. |
1094 | */ |
1095 | obj = i915_gem_object_get(obj); |
1096 | return i915_gem_object_mmap(obj, mmo, vma); |
1097 | } |
1098 | |
1099 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
1100 | #include "selftests/i915_gem_mman.c" |
1101 | #endif |
1102 | |