1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2021 Intel Corporation |
4 | */ |
5 | |
6 | #include "gem/i915_gem_domain.h" |
7 | #include "gem/i915_gem_internal.h" |
8 | #include "gem/i915_gem_lmem.h" |
9 | #include "gt/gen8_ppgtt.h" |
10 | |
11 | #include "i915_drv.h" |
12 | #include "intel_display_types.h" |
13 | #include "intel_dpt.h" |
14 | #include "intel_fb.h" |
15 | |
16 | struct i915_dpt { |
17 | struct i915_address_space vm; |
18 | |
19 | struct drm_i915_gem_object *obj; |
20 | struct i915_vma *vma; |
21 | void __iomem *iomem; |
22 | }; |
23 | |
24 | #define i915_is_dpt(vm) ((vm)->is_dpt) |
25 | |
26 | static inline struct i915_dpt * |
27 | i915_vm_to_dpt(struct i915_address_space *vm) |
28 | { |
29 | BUILD_BUG_ON(offsetof(struct i915_dpt, vm)); |
30 | drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm)); |
31 | return container_of(vm, struct i915_dpt, vm); |
32 | } |
33 | |
34 | #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT) |
35 | |
36 | static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) |
37 | { |
38 | writeq(val: pte, addr); |
39 | } |
40 | |
41 | static void dpt_insert_page(struct i915_address_space *vm, |
42 | dma_addr_t addr, |
43 | u64 offset, |
44 | unsigned int pat_index, |
45 | u32 flags) |
46 | { |
47 | struct i915_dpt *dpt = i915_vm_to_dpt(vm); |
48 | gen8_pte_t __iomem *base = dpt->iomem; |
49 | |
50 | gen8_set_pte(addr: base + offset / I915_GTT_PAGE_SIZE, |
51 | pte: vm->pte_encode(addr, pat_index, flags)); |
52 | } |
53 | |
54 | static void dpt_insert_entries(struct i915_address_space *vm, |
55 | struct i915_vma_resource *vma_res, |
56 | unsigned int pat_index, |
57 | u32 flags) |
58 | { |
59 | struct i915_dpt *dpt = i915_vm_to_dpt(vm); |
60 | gen8_pte_t __iomem *base = dpt->iomem; |
61 | const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); |
62 | struct sgt_iter sgt_iter; |
63 | dma_addr_t addr; |
64 | int i; |
65 | |
66 | /* |
67 | * Note that we ignore PTE_READ_ONLY here. The caller must be careful |
68 | * not to allow the user to override access to a read only page. |
69 | */ |
70 | |
71 | i = vma_res->start / I915_GTT_PAGE_SIZE; |
72 | for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) |
73 | gen8_set_pte(addr: &base[i++], pte: pte_encode | addr); |
74 | } |
75 | |
76 | static void dpt_clear_range(struct i915_address_space *vm, |
77 | u64 start, u64 length) |
78 | { |
79 | } |
80 | |
81 | static void dpt_bind_vma(struct i915_address_space *vm, |
82 | struct i915_vm_pt_stash *stash, |
83 | struct i915_vma_resource *vma_res, |
84 | unsigned int pat_index, |
85 | u32 flags) |
86 | { |
87 | u32 pte_flags; |
88 | |
89 | if (vma_res->bound_flags) |
90 | return; |
91 | |
92 | /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ |
93 | pte_flags = 0; |
94 | if (vm->has_read_only && vma_res->bi.readonly) |
95 | pte_flags |= PTE_READ_ONLY; |
96 | if (vma_res->bi.lmem) |
97 | pte_flags |= PTE_LM; |
98 | |
99 | vm->insert_entries(vm, vma_res, pat_index, pte_flags); |
100 | |
101 | vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; |
102 | |
103 | /* |
104 | * Without aliasing PPGTT there's no difference between |
105 | * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally |
106 | * upgrade to both bound if we bind either to avoid double-binding. |
107 | */ |
108 | vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; |
109 | } |
110 | |
111 | static void dpt_unbind_vma(struct i915_address_space *vm, |
112 | struct i915_vma_resource *vma_res) |
113 | { |
114 | vm->clear_range(vm, vma_res->start, vma_res->vma_size); |
115 | } |
116 | |
117 | static void dpt_cleanup(struct i915_address_space *vm) |
118 | { |
119 | struct i915_dpt *dpt = i915_vm_to_dpt(vm); |
120 | |
121 | i915_gem_object_put(obj: dpt->obj); |
122 | } |
123 | |
124 | struct i915_vma *intel_dpt_pin(struct i915_address_space *vm) |
125 | { |
126 | struct drm_i915_private *i915 = vm->i915; |
127 | struct i915_dpt *dpt = i915_vm_to_dpt(vm); |
128 | intel_wakeref_t wakeref; |
129 | struct i915_vma *vma; |
130 | void __iomem *iomem; |
131 | struct i915_gem_ww_ctx ww; |
132 | u64 pin_flags = 0; |
133 | int err; |
134 | |
135 | if (i915_gem_object_is_stolen(obj: dpt->obj)) |
136 | pin_flags |= PIN_MAPPABLE; |
137 | |
138 | wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm); |
139 | atomic_inc(v: &i915->gpu_error.pending_fb_pin); |
140 | |
141 | for_i915_gem_ww(&ww, err, true) { |
142 | err = i915_gem_object_lock(obj: dpt->obj, ww: &ww); |
143 | if (err) |
144 | continue; |
145 | |
146 | vma = i915_gem_object_ggtt_pin_ww(obj: dpt->obj, ww: &ww, NULL, size: 0, alignment: 4096, |
147 | flags: pin_flags); |
148 | if (IS_ERR(ptr: vma)) { |
149 | err = PTR_ERR(ptr: vma); |
150 | continue; |
151 | } |
152 | |
153 | iomem = i915_vma_pin_iomap(vma); |
154 | i915_vma_unpin(vma); |
155 | |
156 | if (IS_ERR(ptr: iomem)) { |
157 | err = PTR_ERR(ptr: iomem); |
158 | continue; |
159 | } |
160 | |
161 | dpt->vma = vma; |
162 | dpt->iomem = iomem; |
163 | |
164 | i915_vma_get(vma); |
165 | } |
166 | |
167 | dpt->obj->mm.dirty = true; |
168 | |
169 | atomic_dec(v: &i915->gpu_error.pending_fb_pin); |
170 | intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref); |
171 | |
172 | return err ? ERR_PTR(error: err) : vma; |
173 | } |
174 | |
175 | void intel_dpt_unpin(struct i915_address_space *vm) |
176 | { |
177 | struct i915_dpt *dpt = i915_vm_to_dpt(vm); |
178 | |
179 | i915_vma_unpin_iomap(vma: dpt->vma); |
180 | i915_vma_put(vma: dpt->vma); |
181 | } |
182 | |
183 | /** |
184 | * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume |
185 | * @i915: device instance |
186 | * |
187 | * Restore the memory mapping during system resume for all framebuffers which |
188 | * are mapped to HW via a GGTT->DPT page table. The content of these page |
189 | * tables are not stored in the hibernation image during S4 and S3RST->S4 |
190 | * transitions, so here we reprogram the PTE entries in those tables. |
191 | * |
192 | * This function must be called after the mappings in GGTT have been restored calling |
193 | * i915_ggtt_resume(). |
194 | */ |
195 | void intel_dpt_resume(struct drm_i915_private *i915) |
196 | { |
197 | struct drm_framebuffer *drm_fb; |
198 | |
199 | if (!HAS_DISPLAY(i915)) |
200 | return; |
201 | |
202 | mutex_lock(&i915->drm.mode_config.fb_lock); |
203 | drm_for_each_fb(drm_fb, &i915->drm) { |
204 | struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); |
205 | |
206 | if (fb->dpt_vm) |
207 | i915_ggtt_resume_vm(vm: fb->dpt_vm); |
208 | } |
209 | mutex_unlock(lock: &i915->drm.mode_config.fb_lock); |
210 | } |
211 | |
212 | /** |
213 | * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend |
214 | * @i915: device instance |
215 | * |
216 | * Suspend the memory mapping during system suspend for all framebuffers which |
217 | * are mapped to HW via a GGTT->DPT page table. |
218 | * |
219 | * This function must be called before the mappings in GGTT are suspended calling |
220 | * i915_ggtt_suspend(). |
221 | */ |
222 | void intel_dpt_suspend(struct drm_i915_private *i915) |
223 | { |
224 | struct drm_framebuffer *drm_fb; |
225 | |
226 | if (!HAS_DISPLAY(i915)) |
227 | return; |
228 | |
229 | mutex_lock(&i915->drm.mode_config.fb_lock); |
230 | |
231 | drm_for_each_fb(drm_fb, &i915->drm) { |
232 | struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); |
233 | |
234 | if (fb->dpt_vm) |
235 | i915_ggtt_suspend_vm(vm: fb->dpt_vm); |
236 | } |
237 | |
238 | mutex_unlock(lock: &i915->drm.mode_config.fb_lock); |
239 | } |
240 | |
241 | struct i915_address_space * |
242 | intel_dpt_create(struct intel_framebuffer *fb) |
243 | { |
244 | struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base; |
245 | struct drm_i915_private *i915 = to_i915(dev: obj->dev); |
246 | struct drm_i915_gem_object *dpt_obj; |
247 | struct i915_address_space *vm; |
248 | struct i915_dpt *dpt; |
249 | size_t size; |
250 | int ret; |
251 | |
252 | if (intel_fb_needs_pot_stride_remap(fb)) |
253 | size = intel_remapped_info_size(rem_info: &fb->remapped_view.gtt.remapped); |
254 | else |
255 | size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); |
256 | |
257 | size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); |
258 | |
259 | dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS); |
260 | if (IS_ERR(ptr: dpt_obj) && i915_ggtt_has_aperture(ggtt: to_gt(i915)->ggtt)) |
261 | dpt_obj = i915_gem_object_create_stolen(dev_priv: i915, size); |
262 | if (IS_ERR(ptr: dpt_obj) && !HAS_LMEM(i915)) { |
263 | drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n" ); |
264 | dpt_obj = i915_gem_object_create_shmem(i915, size); |
265 | } |
266 | if (IS_ERR(ptr: dpt_obj)) |
267 | return ERR_CAST(ptr: dpt_obj); |
268 | |
269 | ret = i915_gem_object_lock_interruptible(obj: dpt_obj, NULL); |
270 | if (!ret) { |
271 | ret = i915_gem_object_set_cache_level(obj: dpt_obj, cache_level: I915_CACHE_NONE); |
272 | i915_gem_object_unlock(obj: dpt_obj); |
273 | } |
274 | if (ret) { |
275 | i915_gem_object_put(obj: dpt_obj); |
276 | return ERR_PTR(error: ret); |
277 | } |
278 | |
279 | dpt = kzalloc(size: sizeof(*dpt), GFP_KERNEL); |
280 | if (!dpt) { |
281 | i915_gem_object_put(obj: dpt_obj); |
282 | return ERR_PTR(error: -ENOMEM); |
283 | } |
284 | |
285 | vm = &dpt->vm; |
286 | |
287 | vm->gt = to_gt(i915); |
288 | vm->i915 = i915; |
289 | vm->dma = i915->drm.dev; |
290 | vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; |
291 | vm->is_dpt = true; |
292 | |
293 | i915_address_space_init(vm, VM_CLASS_DPT); |
294 | |
295 | vm->insert_page = dpt_insert_page; |
296 | vm->clear_range = dpt_clear_range; |
297 | vm->insert_entries = dpt_insert_entries; |
298 | vm->cleanup = dpt_cleanup; |
299 | |
300 | vm->vma_ops.bind_vma = dpt_bind_vma; |
301 | vm->vma_ops.unbind_vma = dpt_unbind_vma; |
302 | |
303 | vm->pte_encode = vm->gt->ggtt->vm.pte_encode; |
304 | |
305 | dpt->obj = dpt_obj; |
306 | dpt->obj->is_dpt = true; |
307 | |
308 | return &dpt->vm; |
309 | } |
310 | |
311 | void intel_dpt_destroy(struct i915_address_space *vm) |
312 | { |
313 | struct i915_dpt *dpt = i915_vm_to_dpt(vm); |
314 | |
315 | dpt->obj->is_dpt = false; |
316 | i915_vm_put(vm: &dpt->vm); |
317 | } |
318 | |
319 | |