1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2019 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/string_helpers.h> |
7 | #include <linux/suspend.h> |
8 | |
9 | #include "i915_drv.h" |
10 | #include "i915_irq.h" |
11 | #include "i915_params.h" |
12 | #include "intel_context.h" |
13 | #include "intel_engine_pm.h" |
14 | #include "intel_gt.h" |
15 | #include "intel_gt_clock_utils.h" |
16 | #include "intel_gt_mcr.h" |
17 | #include "intel_gt_pm.h" |
18 | #include "intel_gt_print.h" |
19 | #include "intel_gt_requests.h" |
20 | #include "intel_llc.h" |
21 | #include "intel_rc6.h" |
22 | #include "intel_rps.h" |
23 | #include "intel_wakeref.h" |
24 | #include "pxp/intel_pxp_pm.h" |
25 | |
26 | #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2) |
27 | |
28 | static void user_forcewake(struct intel_gt *gt, bool suspend) |
29 | { |
30 | int count = atomic_read(v: >->user_wakeref); |
31 | intel_wakeref_t wakeref; |
32 | |
33 | /* Inside suspend/resume so single threaded, no races to worry about. */ |
34 | if (likely(!count)) |
35 | return; |
36 | |
37 | wakeref = intel_gt_pm_get(gt); |
38 | if (suspend) { |
39 | GEM_BUG_ON(count > atomic_read(>->wakeref.count)); |
40 | atomic_sub(i: count, v: >->wakeref.count); |
41 | } else { |
42 | atomic_add(i: count, v: >->wakeref.count); |
43 | } |
44 | intel_gt_pm_put(gt, handle: wakeref); |
45 | } |
46 | |
47 | static void runtime_begin(struct intel_gt *gt) |
48 | { |
49 | local_irq_disable(); |
50 | write_seqcount_begin(>->stats.lock); |
51 | gt->stats.start = ktime_get(); |
52 | gt->stats.active = true; |
53 | write_seqcount_end(>->stats.lock); |
54 | local_irq_enable(); |
55 | } |
56 | |
57 | static void runtime_end(struct intel_gt *gt) |
58 | { |
59 | local_irq_disable(); |
60 | write_seqcount_begin(>->stats.lock); |
61 | gt->stats.active = false; |
62 | gt->stats.total = |
63 | ktime_add(gt->stats.total, |
64 | ktime_sub(ktime_get(), gt->stats.start)); |
65 | write_seqcount_end(>->stats.lock); |
66 | local_irq_enable(); |
67 | } |
68 | |
69 | static int __gt_unpark(struct intel_wakeref *wf) |
70 | { |
71 | struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); |
72 | struct drm_i915_private *i915 = gt->i915; |
73 | |
74 | GT_TRACE(gt, "\n" ); |
75 | |
76 | /* |
77 | * It seems that the DMC likes to transition between the DC states a lot |
78 | * when there are no connected displays (no active power domains) during |
79 | * command submission. |
80 | * |
81 | * This activity has negative impact on the performance of the chip with |
82 | * huge latencies observed in the interrupt handler and elsewhere. |
83 | * |
84 | * Work around it by grabbing a GT IRQ power domain whilst there is any |
85 | * GT activity, preventing any DC state transitions. |
86 | */ |
87 | gt->awake = intel_display_power_get(dev_priv: i915, domain: POWER_DOMAIN_GT_IRQ); |
88 | GEM_BUG_ON(!gt->awake); |
89 | |
90 | intel_rc6_unpark(rc6: >->rc6); |
91 | intel_rps_unpark(rps: >->rps); |
92 | i915_pmu_gt_unparked(gt); |
93 | intel_guc_busyness_unpark(gt); |
94 | |
95 | intel_gt_unpark_requests(gt); |
96 | runtime_begin(gt); |
97 | |
98 | return 0; |
99 | } |
100 | |
101 | static int __gt_park(struct intel_wakeref *wf) |
102 | { |
103 | struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); |
104 | intel_wakeref_t wakeref = fetch_and_zero(>->awake); |
105 | struct drm_i915_private *i915 = gt->i915; |
106 | |
107 | GT_TRACE(gt, "\n" ); |
108 | |
109 | runtime_end(gt); |
110 | intel_gt_park_requests(gt); |
111 | |
112 | intel_guc_busyness_park(gt); |
113 | i915_vma_parked(gt); |
114 | i915_pmu_gt_parked(gt); |
115 | intel_rps_park(rps: >->rps); |
116 | intel_rc6_park(rc6: >->rc6); |
117 | |
118 | /* Everything switched off, flush any residual interrupt just in case */ |
119 | intel_synchronize_irq(i915); |
120 | |
121 | /* Defer dropping the display power well for 100ms, it's slow! */ |
122 | GEM_BUG_ON(!wakeref); |
123 | intel_display_power_put_async(i915, domain: POWER_DOMAIN_GT_IRQ, wakeref); |
124 | |
125 | return 0; |
126 | } |
127 | |
128 | static const struct intel_wakeref_ops wf_ops = { |
129 | .get = __gt_unpark, |
130 | .put = __gt_park, |
131 | }; |
132 | |
133 | void intel_gt_pm_init_early(struct intel_gt *gt) |
134 | { |
135 | /* |
136 | * We access the runtime_pm structure via gt->i915 here rather than |
137 | * gt->uncore as we do elsewhere in the file because gt->uncore is not |
138 | * yet initialized for all tiles at this point in the driver startup. |
139 | * runtime_pm is per-device rather than per-tile, so this is still the |
140 | * correct structure. |
141 | */ |
142 | intel_wakeref_init(>->wakeref, gt->i915, &wf_ops, "GT" ); |
143 | seqcount_mutex_init(>->stats.lock, >->wakeref.mutex); |
144 | } |
145 | |
146 | void intel_gt_pm_init(struct intel_gt *gt) |
147 | { |
148 | /* |
149 | * Enabling power-management should be "self-healing". If we cannot |
150 | * enable a feature, simply leave it disabled with a notice to the |
151 | * user. |
152 | */ |
153 | intel_rc6_init(rc6: >->rc6); |
154 | intel_rps_init(rps: >->rps); |
155 | } |
156 | |
157 | static bool reset_engines(struct intel_gt *gt) |
158 | { |
159 | if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) |
160 | return false; |
161 | |
162 | return __intel_gt_reset(gt, ALL_ENGINES) == 0; |
163 | } |
164 | |
165 | static void gt_sanitize(struct intel_gt *gt, bool force) |
166 | { |
167 | struct intel_engine_cs *engine; |
168 | enum intel_engine_id id; |
169 | intel_wakeref_t wakeref; |
170 | |
171 | GT_TRACE(gt, "force:%s\n" , str_yes_no(force)); |
172 | |
173 | /* Use a raw wakeref to avoid calling intel_display_power_get early */ |
174 | wakeref = intel_runtime_pm_get(rpm: gt->uncore->rpm); |
175 | intel_uncore_forcewake_get(uncore: gt->uncore, domains: FORCEWAKE_ALL); |
176 | |
177 | intel_gt_check_clock_frequency(gt); |
178 | |
179 | /* |
180 | * As we have just resumed the machine and woken the device up from |
181 | * deep PCI sleep (presumably D3_cold), assume the HW has been reset |
182 | * back to defaults, recovering from whatever wedged state we left it |
183 | * in and so worth trying to use the device once more. |
184 | */ |
185 | if (intel_gt_is_wedged(gt)) |
186 | intel_gt_unset_wedged(gt); |
187 | |
188 | /* For GuC mode, ensure submission is disabled before stopping ring */ |
189 | intel_uc_reset_prepare(uc: >->uc); |
190 | |
191 | for_each_engine(engine, gt, id) { |
192 | if (engine->reset.prepare) |
193 | engine->reset.prepare(engine); |
194 | |
195 | if (engine->sanitize) |
196 | engine->sanitize(engine); |
197 | } |
198 | |
199 | if (reset_engines(gt) || force) { |
200 | for_each_engine(engine, gt, id) |
201 | __intel_engine_reset(engine, stalled: false); |
202 | } |
203 | |
204 | intel_uc_reset(uc: >->uc, stalled: false); |
205 | |
206 | for_each_engine(engine, gt, id) |
207 | if (engine->reset.finish) |
208 | engine->reset.finish(engine); |
209 | |
210 | intel_rps_sanitize(rps: >->rps); |
211 | |
212 | intel_uncore_forcewake_put(uncore: gt->uncore, domains: FORCEWAKE_ALL); |
213 | intel_runtime_pm_put(rpm: gt->uncore->rpm, wref: wakeref); |
214 | } |
215 | |
216 | void intel_gt_pm_fini(struct intel_gt *gt) |
217 | { |
218 | intel_rc6_fini(rc6: >->rc6); |
219 | } |
220 | |
221 | void intel_gt_resume_early(struct intel_gt *gt) |
222 | { |
223 | /* |
224 | * Sanitize steer semaphores during driver resume. This is necessary |
225 | * to address observed cases of steer semaphores being |
226 | * held after a suspend operation. Confirmation from the hardware team |
227 | * assures the safety of this operation, as no lock acquisitions |
228 | * by other agents occur during driver load/resume process. |
229 | */ |
230 | intel_gt_mcr_lock_sanitize(gt); |
231 | |
232 | intel_uncore_resume_early(uncore: gt->uncore); |
233 | intel_gt_check_and_clear_faults(gt); |
234 | } |
235 | |
236 | int intel_gt_resume(struct intel_gt *gt) |
237 | { |
238 | struct intel_engine_cs *engine; |
239 | enum intel_engine_id id; |
240 | intel_wakeref_t wakeref; |
241 | int err; |
242 | |
243 | err = intel_gt_has_unrecoverable_error(gt); |
244 | if (err) |
245 | return err; |
246 | |
247 | GT_TRACE(gt, "\n" ); |
248 | |
249 | /* |
250 | * After resume, we may need to poke into the pinned kernel |
251 | * contexts to paper over any damage caused by the sudden suspend. |
252 | * Only the kernel contexts should remain pinned over suspend, |
253 | * allowing us to fixup the user contexts on their first pin. |
254 | */ |
255 | gt_sanitize(gt, force: true); |
256 | |
257 | wakeref = intel_gt_pm_get(gt); |
258 | |
259 | intel_uncore_forcewake_get(uncore: gt->uncore, domains: FORCEWAKE_ALL); |
260 | intel_rc6_sanitize(rc6: >->rc6); |
261 | if (intel_gt_is_wedged(gt)) { |
262 | err = -EIO; |
263 | goto out_fw; |
264 | } |
265 | |
266 | /* Only when the HW is re-initialised, can we replay the requests */ |
267 | err = intel_gt_init_hw(gt); |
268 | if (err) { |
269 | gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n" ); |
270 | goto err_wedged; |
271 | } |
272 | |
273 | intel_uc_reset_finish(uc: >->uc); |
274 | |
275 | intel_rps_enable(rps: >->rps); |
276 | intel_llc_enable(llc: >->llc); |
277 | |
278 | for_each_engine(engine, gt, id) { |
279 | intel_engine_pm_get(engine); |
280 | |
281 | engine->serial++; /* kernel context lost */ |
282 | err = intel_engine_resume(engine); |
283 | |
284 | intel_engine_pm_put(engine); |
285 | if (err) { |
286 | gt_err(gt, "Failed to restart %s (%d)\n" , |
287 | engine->name, err); |
288 | goto err_wedged; |
289 | } |
290 | } |
291 | |
292 | intel_rc6_enable(rc6: >->rc6); |
293 | |
294 | intel_uc_resume(uc: >->uc); |
295 | |
296 | user_forcewake(gt, suspend: false); |
297 | |
298 | out_fw: |
299 | intel_uncore_forcewake_put(uncore: gt->uncore, domains: FORCEWAKE_ALL); |
300 | intel_gt_pm_put(gt, handle: wakeref); |
301 | intel_gt_bind_context_set_ready(gt); |
302 | return err; |
303 | |
304 | err_wedged: |
305 | intel_gt_set_wedged(gt); |
306 | goto out_fw; |
307 | } |
308 | |
309 | static void wait_for_suspend(struct intel_gt *gt) |
310 | { |
311 | if (!intel_gt_pm_is_awake(gt)) |
312 | return; |
313 | |
314 | if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) { |
315 | /* |
316 | * Forcibly cancel outstanding work and leave |
317 | * the gpu quiet. |
318 | */ |
319 | intel_gt_set_wedged(gt); |
320 | intel_gt_retire_requests(gt); |
321 | } |
322 | |
323 | intel_gt_pm_wait_for_idle(gt); |
324 | } |
325 | |
326 | void intel_gt_suspend_prepare(struct intel_gt *gt) |
327 | { |
328 | intel_gt_bind_context_set_unready(gt); |
329 | user_forcewake(gt, suspend: true); |
330 | wait_for_suspend(gt); |
331 | } |
332 | |
333 | static suspend_state_t pm_suspend_target(void) |
334 | { |
335 | #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) |
336 | return pm_suspend_target_state; |
337 | #else |
338 | return PM_SUSPEND_TO_IDLE; |
339 | #endif |
340 | } |
341 | |
342 | void intel_gt_suspend_late(struct intel_gt *gt) |
343 | { |
344 | intel_wakeref_t wakeref; |
345 | |
346 | /* We expect to be idle already; but also want to be independent */ |
347 | wait_for_suspend(gt); |
348 | |
349 | if (is_mock_gt(gt)) |
350 | return; |
351 | |
352 | GEM_BUG_ON(gt->awake); |
353 | |
354 | intel_uc_suspend(uc: >->uc); |
355 | |
356 | /* |
357 | * On disabling the device, we want to turn off HW access to memory |
358 | * that we no longer own. |
359 | * |
360 | * However, not all suspend-states disable the device. S0 (s2idle) |
361 | * is effectively runtime-suspend, the device is left powered on |
362 | * but needs to be put into a low power state. We need to keep |
363 | * powermanagement enabled, but we also retain system state and so |
364 | * it remains safe to keep on using our allocated memory. |
365 | */ |
366 | if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) |
367 | return; |
368 | |
369 | with_intel_runtime_pm(gt->uncore->rpm, wakeref) { |
370 | intel_rps_disable(rps: >->rps); |
371 | intel_rc6_disable(rc6: >->rc6); |
372 | intel_llc_disable(llc: >->llc); |
373 | } |
374 | |
375 | gt_sanitize(gt, force: false); |
376 | |
377 | GT_TRACE(gt, "\n" ); |
378 | } |
379 | |
380 | void intel_gt_runtime_suspend(struct intel_gt *gt) |
381 | { |
382 | intel_gt_bind_context_set_unready(gt); |
383 | intel_uc_runtime_suspend(uc: >->uc); |
384 | |
385 | GT_TRACE(gt, "\n" ); |
386 | } |
387 | |
388 | int intel_gt_runtime_resume(struct intel_gt *gt) |
389 | { |
390 | int ret; |
391 | |
392 | GT_TRACE(gt, "\n" ); |
393 | intel_gt_init_swizzling(gt); |
394 | intel_ggtt_restore_fences(ggtt: gt->ggtt); |
395 | |
396 | ret = intel_uc_runtime_resume(uc: >->uc); |
397 | if (ret) |
398 | return ret; |
399 | |
400 | intel_gt_bind_context_set_ready(gt); |
401 | return 0; |
402 | } |
403 | |
404 | static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt) |
405 | { |
406 | ktime_t total = gt->stats.total; |
407 | |
408 | if (gt->stats.active) |
409 | total = ktime_add(total, |
410 | ktime_sub(ktime_get(), gt->stats.start)); |
411 | |
412 | return total; |
413 | } |
414 | |
415 | ktime_t intel_gt_get_awake_time(const struct intel_gt *gt) |
416 | { |
417 | unsigned int seq; |
418 | ktime_t total; |
419 | |
420 | do { |
421 | seq = read_seqcount_begin(>->stats.lock); |
422 | total = __intel_gt_get_awake_time(gt); |
423 | } while (read_seqcount_retry(>->stats.lock, seq)); |
424 | |
425 | return total; |
426 | } |
427 | |
428 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
429 | #include "selftest_gt_pm.c" |
430 | #endif |
431 | |