1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * Copyright © 2019 Intel Corporation |
5 | */ |
6 | |
7 | #include "gem/i915_gem_pm.h" |
8 | #include "gem/i915_gem_ttm_pm.h" |
9 | #include "gt/intel_gt.h" |
10 | #include "gt/intel_gt_pm.h" |
11 | #include "gt/intel_gt_requests.h" |
12 | |
13 | #include "i915_driver.h" |
14 | #include "i915_drv.h" |
15 | |
16 | #if defined(CONFIG_X86) |
17 | #include <asm/smp.h> |
18 | #else |
19 | #define wbinvd_on_all_cpus() \ |
20 | pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__) |
21 | #endif |
22 | |
23 | void i915_gem_suspend(struct drm_i915_private *i915) |
24 | { |
25 | struct intel_gt *gt; |
26 | unsigned int i; |
27 | |
28 | GEM_TRACE("%s\n" , dev_name(i915->drm.dev)); |
29 | |
30 | intel_wakeref_auto(wf: &i915->runtime_pm.userfault_wakeref, timeout: 0); |
31 | /* |
32 | * On rare occasions, we've observed the fence completion triggers |
33 | * free_engines asynchronously via rcu_call. Ensure those are done. |
34 | * This path is only called on suspend, so it's an acceptable cost. |
35 | */ |
36 | rcu_barrier(); |
37 | |
38 | flush_workqueue(i915->wq); |
39 | |
40 | /* |
41 | * We have to flush all the executing contexts to main memory so |
42 | * that they can saved in the hibernation image. To ensure the last |
43 | * context image is coherent, we have to switch away from it. That |
44 | * leaves the i915->kernel_context still active when |
45 | * we actually suspend, and its image in memory may not match the GPU |
46 | * state. Fortunately, the kernel_context is disposable and we do |
47 | * not rely on its state. |
48 | */ |
49 | for_each_gt(gt, i915, i) |
50 | intel_gt_suspend_prepare(gt); |
51 | |
52 | i915_gem_drain_freed_objects(i915); |
53 | } |
54 | |
55 | static int lmem_restore(struct drm_i915_private *i915, u32 flags) |
56 | { |
57 | struct intel_memory_region *mr; |
58 | int ret = 0, id; |
59 | |
60 | for_each_memory_region(mr, i915, id) { |
61 | if (mr->type == INTEL_MEMORY_LOCAL) { |
62 | ret = i915_ttm_restore_region(mr, flags); |
63 | if (ret) |
64 | break; |
65 | } |
66 | } |
67 | |
68 | return ret; |
69 | } |
70 | |
71 | static int lmem_suspend(struct drm_i915_private *i915, u32 flags) |
72 | { |
73 | struct intel_memory_region *mr; |
74 | int ret = 0, id; |
75 | |
76 | for_each_memory_region(mr, i915, id) { |
77 | if (mr->type == INTEL_MEMORY_LOCAL) { |
78 | ret = i915_ttm_backup_region(mr, flags); |
79 | if (ret) |
80 | break; |
81 | } |
82 | } |
83 | |
84 | return ret; |
85 | } |
86 | |
87 | static void lmem_recover(struct drm_i915_private *i915) |
88 | { |
89 | struct intel_memory_region *mr; |
90 | int id; |
91 | |
92 | for_each_memory_region(mr, i915, id) |
93 | if (mr->type == INTEL_MEMORY_LOCAL) |
94 | i915_ttm_recover_region(mr); |
95 | } |
96 | |
97 | int i915_gem_backup_suspend(struct drm_i915_private *i915) |
98 | { |
99 | int ret; |
100 | |
101 | /* Opportunistically try to evict unpinned objects */ |
102 | ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU); |
103 | if (ret) |
104 | goto out_recover; |
105 | |
106 | i915_gem_suspend(i915); |
107 | |
108 | /* |
109 | * More objects may have become unpinned as requests were |
110 | * retired. Now try to evict again. The gt may be wedged here |
111 | * in which case we automatically fall back to memcpy. |
112 | * We allow also backing up pinned objects that have not been |
113 | * marked for early recover, and that may contain, for example, |
114 | * page-tables for the migrate context. |
115 | */ |
116 | ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU | |
117 | I915_TTM_BACKUP_PINNED); |
118 | if (ret) |
119 | goto out_recover; |
120 | |
121 | /* |
122 | * Remaining objects are backed up using memcpy once we've stopped |
123 | * using the migrate context. |
124 | */ |
125 | ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED); |
126 | if (ret) |
127 | goto out_recover; |
128 | |
129 | return 0; |
130 | |
131 | out_recover: |
132 | lmem_recover(i915); |
133 | |
134 | return ret; |
135 | } |
136 | |
137 | void i915_gem_suspend_late(struct drm_i915_private *i915) |
138 | { |
139 | struct drm_i915_gem_object *obj; |
140 | struct list_head *phases[] = { |
141 | &i915->mm.shrink_list, |
142 | &i915->mm.purge_list, |
143 | NULL |
144 | }, **phase; |
145 | struct intel_gt *gt; |
146 | unsigned long flags; |
147 | unsigned int i; |
148 | bool flush = false; |
149 | |
150 | /* |
151 | * Neither the BIOS, ourselves or any other kernel |
152 | * expects the system to be in execlists mode on startup, |
153 | * so we need to reset the GPU back to legacy mode. And the only |
154 | * known way to disable logical contexts is through a GPU reset. |
155 | * |
156 | * So in order to leave the system in a known default configuration, |
157 | * always reset the GPU upon unload and suspend. Afterwards we then |
158 | * clean up the GEM state tracking, flushing off the requests and |
159 | * leaving the system in a known idle state. |
160 | * |
161 | * Note that is of the upmost importance that the GPU is idle and |
162 | * all stray writes are flushed *before* we dismantle the backing |
163 | * storage for the pinned objects. |
164 | * |
165 | * However, since we are uncertain that resetting the GPU on older |
166 | * machines is a good idea, we don't - just in case it leaves the |
167 | * machine in an unusable condition. |
168 | */ |
169 | |
170 | /* Like i915_gem_suspend, flush tasks staged from fence triggers */ |
171 | rcu_barrier(); |
172 | |
173 | for_each_gt(gt, i915, i) |
174 | intel_gt_suspend_late(gt); |
175 | |
176 | spin_lock_irqsave(&i915->mm.obj_lock, flags); |
177 | for (phase = phases; *phase; phase++) { |
178 | list_for_each_entry(obj, *phase, mm.link) { |
179 | if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)) |
180 | flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0; |
181 | __start_cpu_write(obj); /* presume auto-hibernate */ |
182 | } |
183 | } |
184 | spin_unlock_irqrestore(lock: &i915->mm.obj_lock, flags); |
185 | if (flush) |
186 | wbinvd_on_all_cpus(); |
187 | } |
188 | |
189 | int i915_gem_freeze(struct drm_i915_private *i915) |
190 | { |
191 | /* Discard all purgeable objects, let userspace recover those as |
192 | * required after resuming. |
193 | */ |
194 | i915_gem_shrink_all(i915); |
195 | |
196 | return 0; |
197 | } |
198 | |
199 | int i915_gem_freeze_late(struct drm_i915_private *i915) |
200 | { |
201 | struct drm_i915_gem_object *obj; |
202 | intel_wakeref_t wakeref; |
203 | |
204 | /* |
205 | * Called just before we write the hibernation image. |
206 | * |
207 | * We need to update the domain tracking to reflect that the CPU |
208 | * will be accessing all the pages to create and restore from the |
209 | * hibernation, and so upon restoration those pages will be in the |
210 | * CPU domain. |
211 | * |
212 | * To make sure the hibernation image contains the latest state, |
213 | * we update that state just before writing out the image. |
214 | * |
215 | * To try and reduce the hibernation image, we manually shrink |
216 | * the objects as well, see i915_gem_freeze() |
217 | */ |
218 | |
219 | with_intel_runtime_pm(&i915->runtime_pm, wakeref) |
220 | i915_gem_shrink(NULL, i915, target: -1UL, NULL, flags: ~0); |
221 | i915_gem_drain_freed_objects(i915); |
222 | |
223 | wbinvd_on_all_cpus(); |
224 | list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) |
225 | __start_cpu_write(obj); |
226 | |
227 | return 0; |
228 | } |
229 | |
230 | void i915_gem_resume(struct drm_i915_private *i915) |
231 | { |
232 | struct intel_gt *gt; |
233 | int ret, i, j; |
234 | |
235 | GEM_TRACE("%s\n" , dev_name(i915->drm.dev)); |
236 | |
237 | ret = lmem_restore(i915, flags: 0); |
238 | GEM_WARN_ON(ret); |
239 | |
240 | /* |
241 | * As we didn't flush the kernel context before suspend, we cannot |
242 | * guarantee that the context image is complete. So let's just reset |
243 | * it and start again. |
244 | */ |
245 | for_each_gt(gt, i915, i) |
246 | if (intel_gt_resume(gt)) |
247 | goto err_wedged; |
248 | |
249 | ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU); |
250 | GEM_WARN_ON(ret); |
251 | |
252 | return; |
253 | |
254 | err_wedged: |
255 | for_each_gt(gt, i915, j) { |
256 | if (!intel_gt_is_wedged(gt)) { |
257 | dev_err(i915->drm.dev, |
258 | "Failed to re-initialize GPU[%u], declaring it wedged!\n" , |
259 | j); |
260 | intel_gt_set_wedged(gt); |
261 | } |
262 | |
263 | if (j == i) |
264 | break; |
265 | } |
266 | } |
267 | |