1	/* SPDX-License-Identifier: MIT */
2	/*
3	* Copyright © 2019 Intel Corporation
4	*/
5
6	#include <linux/string_helpers.h>
7
8	#include "i915_drv.h"
9	#include "i915_irq.h"
10	#include "i915_reg.h"
11	#include "intel_backlight_regs.h"
12	#include "intel_cdclk.h"
13	#include "intel_clock_gating.h"
14	#include "intel_combo_phy.h"
15	#include "intel_de.h"
16	#include "intel_display_power.h"
17	#include "intel_display_power_map.h"
18	#include "intel_display_power_well.h"
19	#include "intel_display_types.h"
20	#include "intel_dmc.h"
21	#include "intel_mchbar_regs.h"
22	#include "intel_pch_refclk.h"
23	#include "intel_pcode.h"
24	#include "intel_pmdemand.h"
25	#include "intel_pps_regs.h"
26	#include "intel_snps_phy.h"
27	#include "skl_watermark.h"
28	#include "skl_watermark_regs.h"
29	#include "vlv_sideband.h"
30
31	#define for_each_power_domain_well(__dev_priv, __power_well, __domain) \
32	for_each_power_well(__dev_priv, __power_well) \
33	for_each_if(test_bit((__domain), (__power_well)->domains.bits))
34
35	#define for_each_power_domain_well_reverse(__dev_priv, __power_well, __domain) \
36	for_each_power_well_reverse(__dev_priv, __power_well) \
37	for_each_if(test_bit((__domain), (__power_well)->domains.bits))
38
39	const char *
40	intel_display_power_domain_str(enum intel_display_power_domain domain)
41	{
42	switch (domain) {
43	case POWER_DOMAIN_DISPLAY_CORE:
44	return "DISPLAY_CORE";
45	case POWER_DOMAIN_PIPE_A:
46	return "PIPE_A";
47	case POWER_DOMAIN_PIPE_B:
48	return "PIPE_B";
49	case POWER_DOMAIN_PIPE_C:
50	return "PIPE_C";
51	case POWER_DOMAIN_PIPE_D:
52	return "PIPE_D";
53	case POWER_DOMAIN_PIPE_PANEL_FITTER_A:
54	return "PIPE_PANEL_FITTER_A";
55	case POWER_DOMAIN_PIPE_PANEL_FITTER_B:
56	return "PIPE_PANEL_FITTER_B";
57	case POWER_DOMAIN_PIPE_PANEL_FITTER_C:
58	return "PIPE_PANEL_FITTER_C";
59	case POWER_DOMAIN_PIPE_PANEL_FITTER_D:
60	return "PIPE_PANEL_FITTER_D";
61	case POWER_DOMAIN_TRANSCODER_A:
62	return "TRANSCODER_A";
63	case POWER_DOMAIN_TRANSCODER_B:
64	return "TRANSCODER_B";
65	case POWER_DOMAIN_TRANSCODER_C:
66	return "TRANSCODER_C";
67	case POWER_DOMAIN_TRANSCODER_D:
68	return "TRANSCODER_D";
69	case POWER_DOMAIN_TRANSCODER_EDP:
70	return "TRANSCODER_EDP";
71	case POWER_DOMAIN_TRANSCODER_DSI_A:
72	return "TRANSCODER_DSI_A";
73	case POWER_DOMAIN_TRANSCODER_DSI_C:
74	return "TRANSCODER_DSI_C";
75	case POWER_DOMAIN_TRANSCODER_VDSC_PW2:
76	return "TRANSCODER_VDSC_PW2";
77	case POWER_DOMAIN_PORT_DDI_LANES_A:
78	return "PORT_DDI_LANES_A";
79	case POWER_DOMAIN_PORT_DDI_LANES_B:
80	return "PORT_DDI_LANES_B";
81	case POWER_DOMAIN_PORT_DDI_LANES_C:
82	return "PORT_DDI_LANES_C";
83	case POWER_DOMAIN_PORT_DDI_LANES_D:
84	return "PORT_DDI_LANES_D";
85	case POWER_DOMAIN_PORT_DDI_LANES_E:
86	return "PORT_DDI_LANES_E";
87	case POWER_DOMAIN_PORT_DDI_LANES_F:
88	return "PORT_DDI_LANES_F";
89	case POWER_DOMAIN_PORT_DDI_LANES_TC1:
90	return "PORT_DDI_LANES_TC1";
91	case POWER_DOMAIN_PORT_DDI_LANES_TC2:
92	return "PORT_DDI_LANES_TC2";
93	case POWER_DOMAIN_PORT_DDI_LANES_TC3:
94	return "PORT_DDI_LANES_TC3";
95	case POWER_DOMAIN_PORT_DDI_LANES_TC4:
96	return "PORT_DDI_LANES_TC4";
97	case POWER_DOMAIN_PORT_DDI_LANES_TC5:
98	return "PORT_DDI_LANES_TC5";
99	case POWER_DOMAIN_PORT_DDI_LANES_TC6:
100	return "PORT_DDI_LANES_TC6";
101	case POWER_DOMAIN_PORT_DDI_IO_A:
102	return "PORT_DDI_IO_A";
103	case POWER_DOMAIN_PORT_DDI_IO_B:
104	return "PORT_DDI_IO_B";
105	case POWER_DOMAIN_PORT_DDI_IO_C:
106	return "PORT_DDI_IO_C";
107	case POWER_DOMAIN_PORT_DDI_IO_D:
108	return "PORT_DDI_IO_D";
109	case POWER_DOMAIN_PORT_DDI_IO_E:
110	return "PORT_DDI_IO_E";
111	case POWER_DOMAIN_PORT_DDI_IO_F:
112	return "PORT_DDI_IO_F";
113	case POWER_DOMAIN_PORT_DDI_IO_TC1:
114	return "PORT_DDI_IO_TC1";
115	case POWER_DOMAIN_PORT_DDI_IO_TC2:
116	return "PORT_DDI_IO_TC2";
117	case POWER_DOMAIN_PORT_DDI_IO_TC3:
118	return "PORT_DDI_IO_TC3";
119	case POWER_DOMAIN_PORT_DDI_IO_TC4:
120	return "PORT_DDI_IO_TC4";
121	case POWER_DOMAIN_PORT_DDI_IO_TC5:
122	return "PORT_DDI_IO_TC5";
123	case POWER_DOMAIN_PORT_DDI_IO_TC6:
124	return "PORT_DDI_IO_TC6";
125	case POWER_DOMAIN_PORT_DSI:
126	return "PORT_DSI";
127	case POWER_DOMAIN_PORT_CRT:
128	return "PORT_CRT";
129	case POWER_DOMAIN_PORT_OTHER:
130	return "PORT_OTHER";
131	case POWER_DOMAIN_VGA:
132	return "VGA";
133	case POWER_DOMAIN_AUDIO_MMIO:
134	return "AUDIO_MMIO";
135	case POWER_DOMAIN_AUDIO_PLAYBACK:
136	return "AUDIO_PLAYBACK";
137	case POWER_DOMAIN_AUX_IO_A:
138	return "AUX_IO_A";
139	case POWER_DOMAIN_AUX_IO_B:
140	return "AUX_IO_B";
141	case POWER_DOMAIN_AUX_IO_C:
142	return "AUX_IO_C";
143	case POWER_DOMAIN_AUX_IO_D:
144	return "AUX_IO_D";
145	case POWER_DOMAIN_AUX_IO_E:
146	return "AUX_IO_E";
147	case POWER_DOMAIN_AUX_IO_F:
148	return "AUX_IO_F";
149	case POWER_DOMAIN_AUX_A:
150	return "AUX_A";
151	case POWER_DOMAIN_AUX_B:
152	return "AUX_B";
153	case POWER_DOMAIN_AUX_C:
154	return "AUX_C";
155	case POWER_DOMAIN_AUX_D:
156	return "AUX_D";
157	case POWER_DOMAIN_AUX_E:
158	return "AUX_E";
159	case POWER_DOMAIN_AUX_F:
160	return "AUX_F";
161	case POWER_DOMAIN_AUX_USBC1:
162	return "AUX_USBC1";
163	case POWER_DOMAIN_AUX_USBC2:
164	return "AUX_USBC2";
165	case POWER_DOMAIN_AUX_USBC3:
166	return "AUX_USBC3";
167	case POWER_DOMAIN_AUX_USBC4:
168	return "AUX_USBC4";
169	case POWER_DOMAIN_AUX_USBC5:
170	return "AUX_USBC5";
171	case POWER_DOMAIN_AUX_USBC6:
172	return "AUX_USBC6";
173	case POWER_DOMAIN_AUX_TBT1:
174	return "AUX_TBT1";
175	case POWER_DOMAIN_AUX_TBT2:
176	return "AUX_TBT2";
177	case POWER_DOMAIN_AUX_TBT3:
178	return "AUX_TBT3";
179	case POWER_DOMAIN_AUX_TBT4:
180	return "AUX_TBT4";
181	case POWER_DOMAIN_AUX_TBT5:
182	return "AUX_TBT5";
183	case POWER_DOMAIN_AUX_TBT6:
184	return "AUX_TBT6";
185	case POWER_DOMAIN_GMBUS:
186	return "GMBUS";
187	case POWER_DOMAIN_INIT:
188	return "INIT";
189	case POWER_DOMAIN_GT_IRQ:
190	return "GT_IRQ";
191	case POWER_DOMAIN_DC_OFF:
192	return "DC_OFF";
193	case POWER_DOMAIN_TC_COLD_OFF:
194	return "TC_COLD_OFF";
195	default:
196	MISSING_CASE(domain);
197	return "?";
198	}
199	}
200
201	/**
202	* __intel_display_power_is_enabled - unlocked check for a power domain
203	* @dev_priv: i915 device instance
204	* @domain: power domain to check
205	*
206	* This is the unlocked version of intel_display_power_is_enabled() and should
207	* only be used from error capture and recovery code where deadlocks are
208	* possible.
209	*
210	* Returns:
211	* True when the power domain is enabled, false otherwise.
212	*/
213	bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
214	enum intel_display_power_domain domain)
215	{
216	struct i915_power_well *power_well;
217	bool is_enabled;
218
219	if (pm_runtime_suspended(dev: dev_priv->drm.dev))
220	return false;
221
222	is_enabled = true;
223
224	for_each_power_domain_well_reverse(dev_priv, power_well, domain) {
225	if (intel_power_well_is_always_on(power_well))
226	continue;
227
228	if (!intel_power_well_is_enabled_cached(power_well)) {
229	is_enabled = false;
230	break;
231	}
232	}
233
234	return is_enabled;
235	}
236
237	/**
238	* intel_display_power_is_enabled - check for a power domain
239	* @dev_priv: i915 device instance
240	* @domain: power domain to check
241	*
242	* This function can be used to check the hw power domain state. It is mostly
243	* used in hardware state readout functions. Everywhere else code should rely
244	* upon explicit power domain reference counting to ensure that the hardware
245	* block is powered up before accessing it.
246	*
247	* Callers must hold the relevant modesetting locks to ensure that concurrent
248	* threads can't disable the power well while the caller tries to read a few
249	* registers.
250	*
251	* Returns:
252	* True when the power domain is enabled, false otherwise.
253	*/
254	bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
255	enum intel_display_power_domain domain)
256	{
257	struct i915_power_domains *power_domains;
258	bool ret;
259
260	power_domains = &dev_priv->display.power.domains;
261
262	mutex_lock(&power_domains->lock);
263	ret = __intel_display_power_is_enabled(dev_priv, domain);
264	mutex_unlock(lock: &power_domains->lock);
265
266	return ret;
267	}
268
269	static u32
270	sanitize_target_dc_state(struct drm_i915_private *i915,
271	u32 target_dc_state)
272	{
273	struct i915_power_domains *power_domains = &i915->display.power.domains;
274	static const u32 states[] = {
275	DC_STATE_EN_UPTO_DC6,
276	DC_STATE_EN_UPTO_DC5,
277	DC_STATE_EN_DC3CO,
278	DC_STATE_DISABLE,
279	};
280	int i;
281
282	for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
283	if (target_dc_state != states[i])
284	continue;
285
286	if (power_domains->allowed_dc_mask & target_dc_state)
287	break;
288
289	target_dc_state = states[i + 1];
290	}
291
292	return target_dc_state;
293	}
294
295	/**
296	* intel_display_power_set_target_dc_state - Set target dc state.
297	* @dev_priv: i915 device
298	* @state: state which needs to be set as target_dc_state.
299	*
300	* This function set the "DC off" power well target_dc_state,
301	* based upon this target_dc_stste, "DC off" power well will
302	* enable desired DC state.
303	*/
304	void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
305	u32 state)
306	{
307	struct i915_power_well *power_well;
308	bool dc_off_enabled;
309	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
310
311	mutex_lock(&power_domains->lock);
312	power_well = lookup_power_well(i915: dev_priv, id: SKL_DISP_DC_OFF);
313
314	if (drm_WARN_ON(&dev_priv->drm, !power_well))
315	goto unlock;
316
317	state = sanitize_target_dc_state(i915: dev_priv, target_dc_state: state);
318
319	if (state == power_domains->target_dc_state)
320	goto unlock;
321
322	dc_off_enabled = intel_power_well_is_enabled(i915: dev_priv, power_well);
323	/*
324	* If DC off power well is disabled, need to enable and disable the
325	* DC off power well to effect target DC state.
326	*/
327	if (!dc_off_enabled)
328	intel_power_well_enable(i915: dev_priv, power_well);
329
330	power_domains->target_dc_state = state;
331
332	if (!dc_off_enabled)
333	intel_power_well_disable(i915: dev_priv, power_well);
334
335	unlock:
336	mutex_unlock(lock: &power_domains->lock);
337	}
338
339	static void __async_put_domains_mask(struct i915_power_domains *power_domains,
340	struct intel_power_domain_mask *mask)
341	{
342	bitmap_or(dst: mask->bits,
343	src1: power_domains->async_put_domains[0].bits,
344	src2: power_domains->async_put_domains[1].bits,
345	nbits: POWER_DOMAIN_NUM);
346	}
347
348	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
349
350	static bool
351	assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
352	{
353	struct drm_i915_private *i915 = container_of(power_domains,
354	struct drm_i915_private,
355	display.power.domains);
356
357	return !drm_WARN_ON(&i915->drm,
358	bitmap_intersects(power_domains->async_put_domains[0].bits,
359	power_domains->async_put_domains[1].bits,
360	POWER_DOMAIN_NUM));
361	}
362
363	static bool
364	__async_put_domains_state_ok(struct i915_power_domains *power_domains)
365	{
366	struct drm_i915_private *i915 = container_of(power_domains,
367	struct drm_i915_private,
368	display.power.domains);
369	struct intel_power_domain_mask async_put_mask;
370	enum intel_display_power_domain domain;
371	bool err = false;
372
373	err |= !assert_async_put_domain_masks_disjoint(power_domains);
374	__async_put_domains_mask(power_domains, mask: &async_put_mask);
375	err |= drm_WARN_ON(&i915->drm,
376	!!power_domains->async_put_wakeref !=
377	!bitmap_empty(async_put_mask.bits, POWER_DOMAIN_NUM));
378
379	for_each_power_domain(domain, &async_put_mask)
380	err |= drm_WARN_ON(&i915->drm,
381	power_domains->domain_use_count[domain] != 1);
382
383	return !err;
384	}
385
386	static void print_power_domains(struct i915_power_domains *power_domains,
387	const char *prefix, struct intel_power_domain_mask *mask)
388	{
389	struct drm_i915_private *i915 = container_of(power_domains,
390	struct drm_i915_private,
391	display.power.domains);
392	enum intel_display_power_domain domain;
393
394	drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
395	for_each_power_domain(domain, mask)
396	drm_dbg(&i915->drm, "%s use_count %d\n",
397	intel_display_power_domain_str(domain),
398	power_domains->domain_use_count[domain]);
399	}
400
401	static void
402	print_async_put_domains_state(struct i915_power_domains *power_domains)
403	{
404	struct drm_i915_private *i915 = container_of(power_domains,
405	struct drm_i915_private,
406	display.power.domains);
407
408	drm_dbg(&i915->drm, "async_put_wakeref: %s\n",
409	str_yes_no(power_domains->async_put_wakeref));
410
411	print_power_domains(power_domains, prefix: "async_put_domains[0]",
412	mask: &power_domains->async_put_domains[0]);
413	print_power_domains(power_domains, prefix: "async_put_domains[1]",
414	mask: &power_domains->async_put_domains[1]);
415	}
416
417	static void
418	verify_async_put_domains_state(struct i915_power_domains *power_domains)
419	{
420	if (!__async_put_domains_state_ok(power_domains))
421	print_async_put_domains_state(power_domains);
422	}
423
424	#else
425
426	static void
427	assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
428	{
429	}
430
431	static void
432	verify_async_put_domains_state(struct i915_power_domains *power_domains)
433	{
434	}
435
436	#endif /* CONFIG_DRM_I915_DEBUG_RUNTIME_PM */
437
438	static void async_put_domains_mask(struct i915_power_domains *power_domains,
439	struct intel_power_domain_mask *mask)
440
441	{
442	assert_async_put_domain_masks_disjoint(power_domains);
443
444	__async_put_domains_mask(power_domains, mask);
445	}
446
447	static void
448	async_put_domains_clear_domain(struct i915_power_domains *power_domains,
449	enum intel_display_power_domain domain)
450	{
451	assert_async_put_domain_masks_disjoint(power_domains);
452
453	clear_bit(nr: domain, addr: power_domains->async_put_domains[0].bits);
454	clear_bit(nr: domain, addr: power_domains->async_put_domains[1].bits);
455	}
456
457	static void
458	cancel_async_put_work(struct i915_power_domains *power_domains, bool sync)
459	{
460	if (sync)
461	cancel_delayed_work_sync(dwork: &power_domains->async_put_work);
462	else
463	cancel_delayed_work(dwork: &power_domains->async_put_work);
464
465	power_domains->async_put_next_delay = 0;
466	}
467
468	static bool
469	intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
470	enum intel_display_power_domain domain)
471	{
472	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
473	struct intel_power_domain_mask async_put_mask;
474	bool ret = false;
475
476	async_put_domains_mask(power_domains, mask: &async_put_mask);
477	if (!test_bit(domain, async_put_mask.bits))
478	goto out_verify;
479
480	async_put_domains_clear_domain(power_domains, domain);
481
482	ret = true;
483
484	async_put_domains_mask(power_domains, mask: &async_put_mask);
485	if (!bitmap_empty(src: async_put_mask.bits, nbits: POWER_DOMAIN_NUM))
486	goto out_verify;
487
488	cancel_async_put_work(power_domains, sync: false);
489	intel_runtime_pm_put_raw(rpm: &dev_priv->runtime_pm,
490	fetch_and_zero(&power_domains->async_put_wakeref));
491	out_verify:
492	verify_async_put_domains_state(power_domains);
493
494	return ret;
495	}
496
497	static void
498	__intel_display_power_get_domain(struct drm_i915_private *dev_priv,
499	enum intel_display_power_domain domain)
500	{
501	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
502	struct i915_power_well *power_well;
503
504	if (intel_display_power_grab_async_put_ref(dev_priv, domain))
505	return;
506
507	for_each_power_domain_well(dev_priv, power_well, domain)
508	intel_power_well_get(i915: dev_priv, power_well);
509
510	power_domains->domain_use_count[domain]++;
511	}
512
513	/**
514	* intel_display_power_get - grab a power domain reference
515	* @dev_priv: i915 device instance
516	* @domain: power domain to reference
517	*
518	* This function grabs a power domain reference for @domain and ensures that the
519	* power domain and all its parents are powered up. Therefore users should only
520	* grab a reference to the innermost power domain they need.
521	*
522	* Any power domain reference obtained by this function must have a symmetric
523	* call to intel_display_power_put() to release the reference again.
524	*/
525	intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
526	enum intel_display_power_domain domain)
527	{
528	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
529	intel_wakeref_t wakeref = intel_runtime_pm_get(rpm: &dev_priv->runtime_pm);
530
531	mutex_lock(&power_domains->lock);
532	__intel_display_power_get_domain(dev_priv, domain);
533	mutex_unlock(lock: &power_domains->lock);
534
535	return wakeref;
536	}
537
538	/**
539	* intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
540	* @dev_priv: i915 device instance
541	* @domain: power domain to reference
542	*
543	* This function grabs a power domain reference for @domain and ensures that the
544	* power domain and all its parents are powered up. Therefore users should only
545	* grab a reference to the innermost power domain they need.
546	*
547	* Any power domain reference obtained by this function must have a symmetric
548	* call to intel_display_power_put() to release the reference again.
549	*/
550	intel_wakeref_t
551	intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
552	enum intel_display_power_domain domain)
553	{
554	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
555	intel_wakeref_t wakeref;
556	bool is_enabled;
557
558	wakeref = intel_runtime_pm_get_if_in_use(rpm: &dev_priv->runtime_pm);
559	if (!wakeref)
560	return false;
561
562	mutex_lock(&power_domains->lock);
563
564	if (__intel_display_power_is_enabled(dev_priv, domain)) {
565	__intel_display_power_get_domain(dev_priv, domain);
566	is_enabled = true;
567	} else {
568	is_enabled = false;
569	}
570
571	mutex_unlock(lock: &power_domains->lock);
572
573	if (!is_enabled) {
574	intel_runtime_pm_put(rpm: &dev_priv->runtime_pm, wref: wakeref);
575	wakeref = 0;
576	}
577
578	return wakeref;
579	}
580
581	static void
582	__intel_display_power_put_domain(struct drm_i915_private *dev_priv,
583	enum intel_display_power_domain domain)
584	{
585	struct i915_power_domains *power_domains;
586	struct i915_power_well *power_well;
587	const char *name = intel_display_power_domain_str(domain);
588	struct intel_power_domain_mask async_put_mask;
589
590	power_domains = &dev_priv->display.power.domains;
591
592	drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
593	"Use count on domain %s is already zero\n",
594	name);
595	async_put_domains_mask(power_domains, mask: &async_put_mask);
596	drm_WARN(&dev_priv->drm,
597	test_bit(domain, async_put_mask.bits),
598	"Async disabling of domain %s is pending\n",
599	name);
600
601	power_domains->domain_use_count[domain]--;
602
603	for_each_power_domain_well_reverse(dev_priv, power_well, domain)
604	intel_power_well_put(i915: dev_priv, power_well);
605	}
606
607	static void __intel_display_power_put(struct drm_i915_private *dev_priv,
608	enum intel_display_power_domain domain)
609	{
610	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
611
612	mutex_lock(&power_domains->lock);
613	__intel_display_power_put_domain(dev_priv, domain);
614	mutex_unlock(lock: &power_domains->lock);
615	}
616
617	static void
618	queue_async_put_domains_work(struct i915_power_domains *power_domains,
619	intel_wakeref_t wakeref,
620	int delay_ms)
621	{
622	struct drm_i915_private *i915 = container_of(power_domains,
623	struct drm_i915_private,
624	display.power.domains);
625	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
626	power_domains->async_put_wakeref = wakeref;
627	drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
628	&power_domains->async_put_work,
629	msecs_to_jiffies(delay_ms)));
630	}
631
632	static void
633	release_async_put_domains(struct i915_power_domains *power_domains,
634	struct intel_power_domain_mask *mask)
635	{
636	struct drm_i915_private *dev_priv =
637	container_of(power_domains, struct drm_i915_private,
638	display.power.domains);
639	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
640	enum intel_display_power_domain domain;
641	intel_wakeref_t wakeref;
642
643	/*
644	* The caller must hold already raw wakeref, upgrade that to a proper
645	* wakeref to make the state checker happy about the HW access during
646	* power well disabling.
647	*/
648	assert_rpm_raw_wakeref_held(rpm);
649	wakeref = intel_runtime_pm_get(rpm);
650
651	for_each_power_domain(domain, mask) {
652	/* Clear before put, so put's sanity check is happy. */
653	async_put_domains_clear_domain(power_domains, domain);
654	__intel_display_power_put_domain(dev_priv, domain);
655	}
656
657	intel_runtime_pm_put(rpm, wref: wakeref);
658	}
659
660	static void
661	intel_display_power_put_async_work(struct work_struct *work)
662	{
663	struct drm_i915_private *dev_priv =
664	container_of(work, struct drm_i915_private,
665	display.power.domains.async_put_work.work);
666	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
667	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
668	intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
669	intel_wakeref_t old_work_wakeref = 0;
670
671	mutex_lock(&power_domains->lock);
672
673	/*
674	* Bail out if all the domain refs pending to be released were grabbed
675	* by subsequent gets or a flush_work.
676	*/
677	old_work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
678	if (!old_work_wakeref)
679	goto out_verify;
680
681	release_async_put_domains(power_domains,
682	mask: &power_domains->async_put_domains[0]);
683
684	/* Requeue the work if more domains were async put meanwhile. */
685	if (!bitmap_empty(src: power_domains->async_put_domains[1].bits, nbits: POWER_DOMAIN_NUM)) {
686	bitmap_copy(dst: power_domains->async_put_domains[0].bits,
687	src: power_domains->async_put_domains[1].bits,
688	nbits: POWER_DOMAIN_NUM);
689	bitmap_zero(dst: power_domains->async_put_domains[1].bits,
690	nbits: POWER_DOMAIN_NUM);
691	queue_async_put_domains_work(power_domains,
692	fetch_and_zero(&new_work_wakeref),
693	delay_ms: power_domains->async_put_next_delay);
694	power_domains->async_put_next_delay = 0;
695	} else {
696	/*
697	* Cancel the work that got queued after this one got dequeued,
698	* since here we released the corresponding async-put reference.
699	*/
700	cancel_async_put_work(power_domains, sync: false);
701	}
702
703	out_verify:
704	verify_async_put_domains_state(power_domains);
705
706	mutex_unlock(lock: &power_domains->lock);
707
708	if (old_work_wakeref)
709	intel_runtime_pm_put_raw(rpm, wref: old_work_wakeref);
710	if (new_work_wakeref)
711	intel_runtime_pm_put_raw(rpm, wref: new_work_wakeref);
712	}
713
714	/**
715	* __intel_display_power_put_async - release a power domain reference asynchronously
716	* @i915: i915 device instance
717	* @domain: power domain to reference
718	* @wakeref: wakeref acquired for the reference that is being released
719	* @delay_ms: delay of powering down the power domain
720	*
721	* This function drops the power domain reference obtained by
722	* intel_display_power_get*() and schedules a work to power down the
723	* corresponding hardware block if this is the last reference.
724	* The power down is delayed by @delay_ms if this is >= 0, or by a default
725	* 100 ms otherwise.
726	*/
727	void __intel_display_power_put_async(struct drm_i915_private *i915,
728	enum intel_display_power_domain domain,
729	intel_wakeref_t wakeref,
730	int delay_ms)
731	{
732	struct i915_power_domains *power_domains = &i915->display.power.domains;
733	struct intel_runtime_pm *rpm = &i915->runtime_pm;
734	intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
735
736	delay_ms = delay_ms >= 0 ? delay_ms : 100;
737
738	mutex_lock(&power_domains->lock);
739
740	if (power_domains->domain_use_count[domain] > 1) {
741	__intel_display_power_put_domain(dev_priv: i915, domain);
742
743	goto out_verify;
744	}
745
746	drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1);
747
748	/* Let a pending work requeue itself or queue a new one. */
749	if (power_domains->async_put_wakeref) {
750	set_bit(nr: domain, addr: power_domains->async_put_domains[1].bits);
751	power_domains->async_put_next_delay = max(power_domains->async_put_next_delay,
752	delay_ms);
753	} else {
754	set_bit(nr: domain, addr: power_domains->async_put_domains[0].bits);
755	queue_async_put_domains_work(power_domains,
756	fetch_and_zero(&work_wakeref),
757	delay_ms);
758	}
759
760	out_verify:
761	verify_async_put_domains_state(power_domains);
762
763	mutex_unlock(lock: &power_domains->lock);
764
765	if (work_wakeref)
766	intel_runtime_pm_put_raw(rpm, wref: work_wakeref);
767
768	intel_runtime_pm_put(rpm, wref: wakeref);
769	}
770
771	/**
772	* intel_display_power_flush_work - flushes the async display power disabling work
773	* @i915: i915 device instance
774	*
775	* Flushes any pending work that was scheduled by a preceding
776	* intel_display_power_put_async() call, completing the disabling of the
777	* corresponding power domains.
778	*
779	* Note that the work handler function may still be running after this
780	* function returns; to ensure that the work handler isn't running use
781	* intel_display_power_flush_work_sync() instead.
782	*/
783	void intel_display_power_flush_work(struct drm_i915_private *i915)
784	{
785	struct i915_power_domains *power_domains = &i915->display.power.domains;
786	struct intel_power_domain_mask async_put_mask;
787	intel_wakeref_t work_wakeref;
788
789	mutex_lock(&power_domains->lock);
790
791	work_wakeref = fetch_and_zero(&power_domains->async_put_wakeref);
792	if (!work_wakeref)
793	goto out_verify;
794
795	async_put_domains_mask(power_domains, mask: &async_put_mask);
796	release_async_put_domains(power_domains, mask: &async_put_mask);
797	cancel_async_put_work(power_domains, sync: false);
798
799	out_verify:
800	verify_async_put_domains_state(power_domains);
801
802	mutex_unlock(lock: &power_domains->lock);
803
804	if (work_wakeref)
805	intel_runtime_pm_put_raw(rpm: &i915->runtime_pm, wref: work_wakeref);
806	}
807
808	/**
809	* intel_display_power_flush_work_sync - flushes and syncs the async display power disabling work
810	* @i915: i915 device instance
811	*
812	* Like intel_display_power_flush_work(), but also ensure that the work
813	* handler function is not running any more when this function returns.
814	*/
815	static void
816	intel_display_power_flush_work_sync(struct drm_i915_private *i915)
817	{
818	struct i915_power_domains *power_domains = &i915->display.power.domains;
819
820	intel_display_power_flush_work(i915);
821	cancel_async_put_work(power_domains, sync: true);
822
823	verify_async_put_domains_state(power_domains);
824
825	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
826	}
827
828	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
829	/**
830	* intel_display_power_put - release a power domain reference
831	* @dev_priv: i915 device instance
832	* @domain: power domain to reference
833	* @wakeref: wakeref acquired for the reference that is being released
834	*
835	* This function drops the power domain reference obtained by
836	* intel_display_power_get() and might power down the corresponding hardware
837	* block right away if this is the last reference.
838	*/
839	void intel_display_power_put(struct drm_i915_private *dev_priv,
840	enum intel_display_power_domain domain,
841	intel_wakeref_t wakeref)
842	{
843	__intel_display_power_put(dev_priv, domain);
844	intel_runtime_pm_put(rpm: &dev_priv->runtime_pm, wref: wakeref);
845	}
846	#else
847	/**
848	* intel_display_power_put_unchecked - release an unchecked power domain reference
849	* @dev_priv: i915 device instance
850	* @domain: power domain to reference
851	*
852	* This function drops the power domain reference obtained by
853	* intel_display_power_get() and might power down the corresponding hardware
854	* block right away if this is the last reference.
855	*
856	* This function is only for the power domain code's internal use to suppress wakeref
857	* tracking when the correspondig debug kconfig option is disabled, should not
858	* be used otherwise.
859	*/
860	void intel_display_power_put_unchecked(struct drm_i915_private *dev_priv,
861	enum intel_display_power_domain domain)
862	{
863	__intel_display_power_put(dev_priv, domain);
864	intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
865	}
866	#endif
867
868	void
869	intel_display_power_get_in_set(struct drm_i915_private *i915,
870	struct intel_display_power_domain_set *power_domain_set,
871	enum intel_display_power_domain domain)
872	{
873	intel_wakeref_t __maybe_unused wf;
874
875	drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
876
877	wf = intel_display_power_get(dev_priv: i915, domain);
878	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
879	power_domain_set->wakerefs[domain] = wf;
880	#endif
881	set_bit(nr: domain, addr: power_domain_set->mask.bits);
882	}
883
884	bool
885	intel_display_power_get_in_set_if_enabled(struct drm_i915_private *i915,
886	struct intel_display_power_domain_set *power_domain_set,
887	enum intel_display_power_domain domain)
888	{
889	intel_wakeref_t wf;
890
891	drm_WARN_ON(&i915->drm, test_bit(domain, power_domain_set->mask.bits));
892
893	wf = intel_display_power_get_if_enabled(dev_priv: i915, domain);
894	if (!wf)
895	return false;
896
897	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
898	power_domain_set->wakerefs[domain] = wf;
899	#endif
900	set_bit(nr: domain, addr: power_domain_set->mask.bits);
901
902	return true;
903	}
904
905	void
906	intel_display_power_put_mask_in_set(struct drm_i915_private *i915,
907	struct intel_display_power_domain_set *power_domain_set,
908	struct intel_power_domain_mask *mask)
909	{
910	enum intel_display_power_domain domain;
911
912	drm_WARN_ON(&i915->drm,
913	!bitmap_subset(mask->bits, power_domain_set->mask.bits, POWER_DOMAIN_NUM));
914
915	for_each_power_domain(domain, mask) {
916	intel_wakeref_t __maybe_unused wf = -1;
917
918	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
919	wf = fetch_and_zero(&power_domain_set->wakerefs[domain]);
920	#endif
921	intel_display_power_put(dev_priv: i915, domain, wakeref: wf);
922	clear_bit(nr: domain, addr: power_domain_set->mask.bits);
923	}
924	}
925
926	static int
927	sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
928	int disable_power_well)
929	{
930	if (disable_power_well >= 0)
931	return !!disable_power_well;
932
933	return 1;
934	}
935
936	static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
937	int enable_dc)
938	{
939	u32 mask;
940	int requested_dc;
941	int max_dc;
942
943	if (!HAS_DISPLAY(dev_priv))
944	return 0;
945
946	if (DISPLAY_VER(dev_priv) >= 20)
947	max_dc = 2;
948	else if (IS_DG2(dev_priv))
949	max_dc = 1;
950	else if (IS_DG1(dev_priv))
951	max_dc = 3;
952	else if (DISPLAY_VER(dev_priv) >= 12)
953	max_dc = 4;
954	else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
955	max_dc = 1;
956	else if (DISPLAY_VER(dev_priv) >= 9)
957	max_dc = 2;
958	else
959	max_dc = 0;
960
961	/*
962	* DC9 has a separate HW flow from the rest of the DC states,
963	* not depending on the DMC firmware. It's needed by system
964	* suspend/resume, so allow it unconditionally.
965	*/
966	mask = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ||
967	DISPLAY_VER(dev_priv) >= 11 ?
968	DC_STATE_EN_DC9 : 0;
969
970	if (!dev_priv->display.params.disable_power_well)
971	max_dc = 0;
972
973	if (enable_dc >= 0 && enable_dc <= max_dc) {
974	requested_dc = enable_dc;
975	} else if (enable_dc == -1) {
976	requested_dc = max_dc;
977	} else if (enable_dc > max_dc && enable_dc <= 4) {
978	drm_dbg_kms(&dev_priv->drm,
979	"Adjusting requested max DC state (%d->%d)\n",
980	enable_dc, max_dc);
981	requested_dc = max_dc;
982	} else {
983	drm_err(&dev_priv->drm,
984	"Unexpected value for enable_dc (%d)\n", enable_dc);
985	requested_dc = max_dc;
986	}
987
988	switch (requested_dc) {
989	case 4:
990	mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
991	break;
992	case 3:
993	mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
994	break;
995	case 2:
996	mask |= DC_STATE_EN_UPTO_DC6;
997	break;
998	case 1:
999	mask |= DC_STATE_EN_UPTO_DC5;
1000	break;
1001	}
1002
1003	drm_dbg_kms(&dev_priv->drm, "Allowed DC state mask %02x\n", mask);
1004
1005	return mask;
1006	}
1007
1008	/**
1009	* intel_power_domains_init - initializes the power domain structures
1010	* @dev_priv: i915 device instance
1011	*
1012	* Initializes the power domain structures for @dev_priv depending upon the
1013	* supported platform.
1014	*/
1015	int intel_power_domains_init(struct drm_i915_private *dev_priv)
1016	{
1017	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1018
1019	dev_priv->display.params.disable_power_well =
1020	sanitize_disable_power_well_option(dev_priv,
1021	disable_power_well: dev_priv->display.params.disable_power_well);
1022	power_domains->allowed_dc_mask =
1023	get_allowed_dc_mask(dev_priv, enable_dc: dev_priv->display.params.enable_dc);
1024
1025	power_domains->target_dc_state =
1026	sanitize_target_dc_state(i915: dev_priv, DC_STATE_EN_UPTO_DC6);
1027
1028	mutex_init(&power_domains->lock);
1029
1030	INIT_DELAYED_WORK(&power_domains->async_put_work,
1031	intel_display_power_put_async_work);
1032
1033	return intel_display_power_map_init(power_domains);
1034	}
1035
1036	/**
1037	* intel_power_domains_cleanup - clean up power domains resources
1038	* @dev_priv: i915 device instance
1039	*
1040	* Release any resources acquired by intel_power_domains_init()
1041	*/
1042	void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
1043	{
1044	intel_display_power_map_cleanup(power_domains: &dev_priv->display.power.domains);
1045	}
1046
1047	static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
1048	{
1049	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1050	struct i915_power_well *power_well;
1051
1052	mutex_lock(&power_domains->lock);
1053	for_each_power_well(dev_priv, power_well)
1054	intel_power_well_sync_hw(i915: dev_priv, power_well);
1055	mutex_unlock(lock: &power_domains->lock);
1056	}
1057
1058	static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
1059	enum dbuf_slice slice, bool enable)
1060	{
1061	i915_reg_t reg = DBUF_CTL_S(slice);
1062	bool state;
1063
1064	intel_de_rmw(i915: dev_priv, reg, DBUF_POWER_REQUEST,
1065	set: enable ? DBUF_POWER_REQUEST : 0);
1066	intel_de_posting_read(i915: dev_priv, reg);
1067	udelay(10);
1068
1069	state = intel_de_read(i915: dev_priv, reg) & DBUF_POWER_STATE;
1070	drm_WARN(&dev_priv->drm, enable != state,
1071	"DBuf slice %d power %s timeout!\n",
1072	slice, str_enable_disable(enable));
1073	}
1074
1075	void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
1076	u8 req_slices)
1077	{
1078	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1079	u8 slice_mask = DISPLAY_INFO(dev_priv)->dbuf.slice_mask;
1080	enum dbuf_slice slice;
1081
1082	drm_WARN(&dev_priv->drm, req_slices & ~slice_mask,
1083	"Invalid set of dbuf slices (0x%x) requested (total dbuf slices 0x%x)\n",
1084	req_slices, slice_mask);
1085
1086	drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n",
1087	req_slices);
1088
1089	/*
1090	* Might be running this in parallel to gen9_dc_off_power_well_enable
1091	* being called from intel_dp_detect for instance,
1092	* which causes assertion triggered by race condition,
1093	* as gen9_assert_dbuf_enabled might preempt this when registers
1094	* were already updated, while dev_priv was not.
1095	*/
1096	mutex_lock(&power_domains->lock);
1097
1098	for_each_dbuf_slice(dev_priv, slice)
1099	gen9_dbuf_slice_set(dev_priv, slice, enable: req_slices & BIT(slice));
1100
1101	dev_priv->display.dbuf.enabled_slices = req_slices;
1102
1103	mutex_unlock(lock: &power_domains->lock);
1104	}
1105
1106	static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
1107	{
1108	u8 slices_mask;
1109
1110	dev_priv->display.dbuf.enabled_slices =
1111	intel_enabled_dbuf_slices_mask(i915: dev_priv);
1112
1113	slices_mask = BIT(DBUF_S1) | dev_priv->display.dbuf.enabled_slices;
1114
1115	if (DISPLAY_VER(dev_priv) >= 14)
1116	intel_pmdemand_program_dbuf(i915: dev_priv, dbuf_slices: slices_mask);
1117
1118	/*
1119	* Just power up at least 1 slice, we will
1120	* figure out later which slices we have and what we need.
1121	*/
1122	gen9_dbuf_slices_update(dev_priv, req_slices: slices_mask);
1123	}
1124
1125	static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
1126	{
1127	gen9_dbuf_slices_update(dev_priv, req_slices: 0);
1128
1129	if (DISPLAY_VER(dev_priv) >= 14)
1130	intel_pmdemand_program_dbuf(i915: dev_priv, dbuf_slices: 0);
1131	}
1132
1133	static void gen12_dbuf_slices_config(struct drm_i915_private *dev_priv)
1134	{
1135	enum dbuf_slice slice;
1136
1137	if (IS_ALDERLAKE_P(dev_priv))
1138	return;
1139
1140	for_each_dbuf_slice(dev_priv, slice)
1141	intel_de_rmw(i915: dev_priv, DBUF_CTL_S(slice),
1142	DBUF_TRACKER_STATE_SERVICE_MASK,
1143	DBUF_TRACKER_STATE_SERVICE(8));
1144	}
1145
1146	static void icl_mbus_init(struct drm_i915_private *dev_priv)
1147	{
1148	unsigned long abox_regs = DISPLAY_INFO(dev_priv)->abox_mask;
1149	u32 mask, val, i;
1150
1151	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
1152	return;
1153
1154	mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
1155	MBUS_ABOX_BT_CREDIT_POOL2_MASK |
1156	MBUS_ABOX_B_CREDIT_MASK |
1157	MBUS_ABOX_BW_CREDIT_MASK;
1158	val = MBUS_ABOX_BT_CREDIT_POOL1(16) |
1159	MBUS_ABOX_BT_CREDIT_POOL2(16) |
1160	MBUS_ABOX_B_CREDIT(1) |
1161	MBUS_ABOX_BW_CREDIT(1);
1162
1163	/*
1164	* gen12 platforms that use abox1 and abox2 for pixel data reads still
1165	* expect us to program the abox_ctl0 register as well, even though
1166	* we don't have to program other instance-0 registers like BW_BUDDY.
1167	*/
1168	if (DISPLAY_VER(dev_priv) == 12)
1169	abox_regs |= BIT(0);
1170
1171	for_each_set_bit(i, &abox_regs, sizeof(abox_regs))
1172	intel_de_rmw(i915: dev_priv, MBUS_ABOX_CTL(i), clear: mask, set: val);
1173	}
1174
1175	static void hsw_assert_cdclk(struct drm_i915_private *dev_priv)
1176	{
1177	u32 val = intel_de_read(i915: dev_priv, LCPLL_CTL);
1178
1179	/*
1180	* The LCPLL register should be turned on by the BIOS. For now
1181	* let's just check its state and print errors in case
1182	* something is wrong. Don't even try to turn it on.
1183	*/
1184
1185	if (val & LCPLL_CD_SOURCE_FCLK)
1186	drm_err(&dev_priv->drm, "CDCLK source is not LCPLL\n");
1187
1188	if (val & LCPLL_PLL_DISABLE)
1189	drm_err(&dev_priv->drm, "LCPLL is disabled\n");
1190
1191	if ((val & LCPLL_REF_MASK) != LCPLL_REF_NON_SSC)
1192	drm_err(&dev_priv->drm, "LCPLL not using non-SSC reference\n");
1193	}
1194
1195	static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
1196	{
1197	struct intel_crtc *crtc;
1198
1199	for_each_intel_crtc(&dev_priv->drm, crtc)
1200	I915_STATE_WARN(dev_priv, crtc->active,
1201	"CRTC for pipe %c enabled\n",
1202	pipe_name(crtc->pipe));
1203
1204	I915_STATE_WARN(dev_priv, intel_de_read(dev_priv, HSW_PWR_WELL_CTL2),
1205	"Display power well on\n");
1206	I915_STATE_WARN(dev_priv,
1207	intel_de_read(dev_priv, SPLL_CTL) & SPLL_PLL_ENABLE,
1208	"SPLL enabled\n");
1209	I915_STATE_WARN(dev_priv,
1210	intel_de_read(dev_priv, WRPLL_CTL(0)) & WRPLL_PLL_ENABLE,
1211	"WRPLL1 enabled\n");
1212	I915_STATE_WARN(dev_priv,
1213	intel_de_read(dev_priv, WRPLL_CTL(1)) & WRPLL_PLL_ENABLE,
1214	"WRPLL2 enabled\n");
1215	I915_STATE_WARN(dev_priv,
1216	intel_de_read(dev_priv, PP_STATUS(0)) & PP_ON,
1217	"Panel power on\n");
1218	I915_STATE_WARN(dev_priv,
1219	intel_de_read(dev_priv, BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
1220	"CPU PWM1 enabled\n");
1221	if (IS_HASWELL(dev_priv))
1222	I915_STATE_WARN(dev_priv,
1223	intel_de_read(dev_priv, HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
1224	"CPU PWM2 enabled\n");
1225	I915_STATE_WARN(dev_priv,
1226	intel_de_read(dev_priv, BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
1227	"PCH PWM1 enabled\n");
1228	I915_STATE_WARN(dev_priv,
1229	(intel_de_read(dev_priv, UTIL_PIN_CTL) & (UTIL_PIN_ENABLE | UTIL_PIN_MODE_MASK)) == (UTIL_PIN_ENABLE | UTIL_PIN_MODE_PWM),
1230	"Utility pin enabled in PWM mode\n");
1231	I915_STATE_WARN(dev_priv,
1232	intel_de_read(dev_priv, PCH_GTC_CTL) & PCH_GTC_ENABLE,
1233	"PCH GTC enabled\n");
1234
1235	/*
1236	* In theory we can still leave IRQs enabled, as long as only the HPD
1237	* interrupts remain enabled. We used to check for that, but since it's
1238	* gen-specific and since we only disable LCPLL after we fully disable
1239	* the interrupts, the check below should be enough.
1240	*/
1241	I915_STATE_WARN(dev_priv, intel_irqs_enabled(dev_priv),
1242	"IRQs enabled\n");
1243	}
1244
1245	static u32 hsw_read_dcomp(struct drm_i915_private *dev_priv)
1246	{
1247	if (IS_HASWELL(dev_priv))
1248	return intel_de_read(i915: dev_priv, D_COMP_HSW);
1249	else
1250	return intel_de_read(i915: dev_priv, D_COMP_BDW);
1251	}
1252
1253	static void hsw_write_dcomp(struct drm_i915_private *dev_priv, u32 val)
1254	{
1255	if (IS_HASWELL(dev_priv)) {
1256	if (snb_pcode_write(&dev_priv->uncore, GEN6_PCODE_WRITE_D_COMP, val))
1257	drm_dbg_kms(&dev_priv->drm,
1258	"Failed to write to D_COMP\n");
1259	} else {
1260	intel_de_write(i915: dev_priv, D_COMP_BDW, val);
1261	intel_de_posting_read(i915: dev_priv, D_COMP_BDW);
1262	}
1263	}
1264
1265	/*
1266	* This function implements pieces of two sequences from BSpec:
1267	* - Sequence for display software to disable LCPLL
1268	* - Sequence for display software to allow package C8+
1269	* The steps implemented here are just the steps that actually touch the LCPLL
1270	* register. Callers should take care of disabling all the display engine
1271	* functions, doing the mode unset, fixing interrupts, etc.
1272	*/
1273	static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
1274	bool switch_to_fclk, bool allow_power_down)
1275	{
1276	u32 val;
1277
1278	assert_can_disable_lcpll(dev_priv);
1279
1280	val = intel_de_read(i915: dev_priv, LCPLL_CTL);
1281
1282	if (switch_to_fclk) {
1283	val |= LCPLL_CD_SOURCE_FCLK;
1284	intel_de_write(i915: dev_priv, LCPLL_CTL, val);
1285
1286	if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
1287	LCPLL_CD_SOURCE_FCLK_DONE, 1))
1288	drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
1289
1290	val = intel_de_read(i915: dev_priv, LCPLL_CTL);
1291	}
1292
1293	val |= LCPLL_PLL_DISABLE;
1294	intel_de_write(i915: dev_priv, LCPLL_CTL, val);
1295	intel_de_posting_read(i915: dev_priv, LCPLL_CTL);
1296
1297	if (intel_de_wait_for_clear(i915: dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, timeout: 1))
1298	drm_err(&dev_priv->drm, "LCPLL still locked\n");
1299
1300	val = hsw_read_dcomp(dev_priv);
1301	val |= D_COMP_COMP_DISABLE;
1302	hsw_write_dcomp(dev_priv, val);
1303	ndelay(100);
1304
1305	if (wait_for((hsw_read_dcomp(dev_priv) &
1306	D_COMP_RCOMP_IN_PROGRESS) == 0, 1))
1307	drm_err(&dev_priv->drm, "D_COMP RCOMP still in progress\n");
1308
1309	if (allow_power_down) {
1310	intel_de_rmw(i915: dev_priv, LCPLL_CTL, clear: 0, LCPLL_POWER_DOWN_ALLOW);
1311	intel_de_posting_read(i915: dev_priv, LCPLL_CTL);
1312	}
1313	}
1314
1315	/*
1316	* Fully restores LCPLL, disallowing power down and switching back to LCPLL
1317	* source.
1318	*/
1319	static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
1320	{
1321	u32 val;
1322
1323	val = intel_de_read(i915: dev_priv, LCPLL_CTL);
1324
1325	if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
1326	LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
1327	return;
1328
1329	/*
1330	* Make sure we're not on PC8 state before disabling PC8, otherwise
1331	* we'll hang the machine. To prevent PC8 state, just enable force_wake.
1332	*/
1333	intel_uncore_forcewake_get(uncore: &dev_priv->uncore, domains: FORCEWAKE_ALL);
1334
1335	if (val & LCPLL_POWER_DOWN_ALLOW) {
1336	val &= ~LCPLL_POWER_DOWN_ALLOW;
1337	intel_de_write(i915: dev_priv, LCPLL_CTL, val);
1338	intel_de_posting_read(i915: dev_priv, LCPLL_CTL);
1339	}
1340
1341	val = hsw_read_dcomp(dev_priv);
1342	val |= D_COMP_COMP_FORCE;
1343	val &= ~D_COMP_COMP_DISABLE;
1344	hsw_write_dcomp(dev_priv, val);
1345
1346	val = intel_de_read(i915: dev_priv, LCPLL_CTL);
1347	val &= ~LCPLL_PLL_DISABLE;
1348	intel_de_write(i915: dev_priv, LCPLL_CTL, val);
1349
1350	if (intel_de_wait_for_set(i915: dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, timeout: 5))
1351	drm_err(&dev_priv->drm, "LCPLL not locked yet\n");
1352
1353	if (val & LCPLL_CD_SOURCE_FCLK) {
1354	intel_de_rmw(i915: dev_priv, LCPLL_CTL, LCPLL_CD_SOURCE_FCLK, set: 0);
1355
1356	if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
1357	LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
1358	drm_err(&dev_priv->drm,
1359	"Switching back to LCPLL failed\n");
1360	}
1361
1362	intel_uncore_forcewake_put(uncore: &dev_priv->uncore, domains: FORCEWAKE_ALL);
1363
1364	intel_update_cdclk(dev_priv);
1365	intel_cdclk_dump_config(i915: dev_priv, cdclk_config: &dev_priv->display.cdclk.hw, context: "Current CDCLK");
1366	}
1367
1368	/*
1369	* Package states C8 and deeper are really deep PC states that can only be
1370	* reached when all the devices on the system allow it, so even if the graphics
1371	* device allows PC8+, it doesn't mean the system will actually get to these
1372	* states. Our driver only allows PC8+ when going into runtime PM.
1373	*
1374	* The requirements for PC8+ are that all the outputs are disabled, the power
1375	* well is disabled and most interrupts are disabled, and these are also
1376	* requirements for runtime PM. When these conditions are met, we manually do
1377	* the other conditions: disable the interrupts, clocks and switch LCPLL refclk
1378	* to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
1379	* hang the machine.
1380	*
1381	* When we really reach PC8 or deeper states (not just when we allow it) we lose
1382	* the state of some registers, so when we come back from PC8+ we need to
1383	* restore this state. We don't get into PC8+ if we're not in RC6, so we don't
1384	* need to take care of the registers kept by RC6. Notice that this happens even
1385	* if we don't put the device in PCI D3 state (which is what currently happens
1386	* because of the runtime PM support).
1387	*
1388	* For more, read "Display Sequences for Package C8" on the hardware
1389	* documentation.
1390	*/
1391	static void hsw_enable_pc8(struct drm_i915_private *dev_priv)
1392	{
1393	drm_dbg_kms(&dev_priv->drm, "Enabling package C8+\n");
1394
1395	if (HAS_PCH_LPT_LP(dev_priv))
1396	intel_de_rmw(i915: dev_priv, SOUTH_DSPCLK_GATE_D,
1397	PCH_LP_PARTITION_LEVEL_DISABLE, set: 0);
1398
1399	lpt_disable_clkout_dp(dev_priv);
1400	hsw_disable_lcpll(dev_priv, switch_to_fclk: true, allow_power_down: true);
1401	}
1402
1403	static void hsw_disable_pc8(struct drm_i915_private *dev_priv)
1404	{
1405	drm_dbg_kms(&dev_priv->drm, "Disabling package C8+\n");
1406
1407	hsw_restore_lcpll(dev_priv);
1408	intel_init_pch_refclk(dev_priv);
1409
1410	/* Many display registers don't survive PC8+ */
1411	intel_clock_gating_init(i915: dev_priv);
1412	}
1413
1414	static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
1415	bool enable)
1416	{
1417	i915_reg_t reg;
1418	u32 reset_bits;
1419
1420	if (IS_IVYBRIDGE(dev_priv)) {
1421	reg = GEN7_MSG_CTL;
1422	reset_bits = WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK;
1423	} else {
1424	reg = HSW_NDE_RSTWRN_OPT;
1425	reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
1426	}
1427
1428	if (DISPLAY_VER(dev_priv) >= 14)
1429	reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
1430
1431	intel_de_rmw(i915: dev_priv, reg, clear: reset_bits, set: enable ? reset_bits : 0);
1432	}
1433
1434	static void skl_display_core_init(struct drm_i915_private *dev_priv,
1435	bool resume)
1436	{
1437	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1438	struct i915_power_well *well;
1439
1440	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1441
1442	/* enable PCH reset handshake */
1443	intel_pch_reset_handshake(dev_priv, enable: !HAS_PCH_NOP(dev_priv));
1444
1445	if (!HAS_DISPLAY(dev_priv))
1446	return;
1447
1448	/* enable PG1 and Misc I/O */
1449	mutex_lock(&power_domains->lock);
1450
1451	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_1);
1452	intel_power_well_enable(i915: dev_priv, power_well: well);
1453
1454	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_MISC_IO);
1455	intel_power_well_enable(i915: dev_priv, power_well: well);
1456
1457	mutex_unlock(lock: &power_domains->lock);
1458
1459	intel_cdclk_init_hw(i915: dev_priv);
1460
1461	gen9_dbuf_enable(dev_priv);
1462
1463	if (resume)
1464	intel_dmc_load_program(i915: dev_priv);
1465	}
1466
1467	static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
1468	{
1469	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1470	struct i915_power_well *well;
1471
1472	if (!HAS_DISPLAY(dev_priv))
1473	return;
1474
1475	gen9_disable_dc_states(dev_priv);
1476	/* TODO: disable DMC program */
1477
1478	gen9_dbuf_disable(dev_priv);
1479
1480	intel_cdclk_uninit_hw(i915: dev_priv);
1481
1482	/* The spec doesn't call for removing the reset handshake flag */
1483	/* disable PG1 and Misc I/O */
1484
1485	mutex_lock(&power_domains->lock);
1486
1487	/*
1488	* BSpec says to keep the MISC IO power well enabled here, only
1489	* remove our request for power well 1.
1490	* Note that even though the driver's request is removed power well 1
1491	* may stay enabled after this due to DMC's own request on it.
1492	*/
1493	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_1);
1494	intel_power_well_disable(i915: dev_priv, power_well: well);
1495
1496	mutex_unlock(lock: &power_domains->lock);
1497
1498	usleep_range(min: 10, max: 30); /* 10 us delay per Bspec */
1499	}
1500
1501	static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
1502	{
1503	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1504	struct i915_power_well *well;
1505
1506	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1507
1508	/*
1509	* NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
1510	* or else the reset will hang because there is no PCH to respond.
1511	* Move the handshake programming to initialization sequence.
1512	* Previously was left up to BIOS.
1513	*/
1514	intel_pch_reset_handshake(dev_priv, enable: false);
1515
1516	if (!HAS_DISPLAY(dev_priv))
1517	return;
1518
1519	/* Enable PG1 */
1520	mutex_lock(&power_domains->lock);
1521
1522	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_1);
1523	intel_power_well_enable(i915: dev_priv, power_well: well);
1524
1525	mutex_unlock(lock: &power_domains->lock);
1526
1527	intel_cdclk_init_hw(i915: dev_priv);
1528
1529	gen9_dbuf_enable(dev_priv);
1530
1531	if (resume)
1532	intel_dmc_load_program(i915: dev_priv);
1533	}
1534
1535	static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
1536	{
1537	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1538	struct i915_power_well *well;
1539
1540	if (!HAS_DISPLAY(dev_priv))
1541	return;
1542
1543	gen9_disable_dc_states(dev_priv);
1544	/* TODO: disable DMC program */
1545
1546	gen9_dbuf_disable(dev_priv);
1547
1548	intel_cdclk_uninit_hw(i915: dev_priv);
1549
1550	/* The spec doesn't call for removing the reset handshake flag */
1551
1552	/*
1553	* Disable PW1 (PG1).
1554	* Note that even though the driver's request is removed power well 1
1555	* may stay enabled after this due to DMC's own request on it.
1556	*/
1557	mutex_lock(&power_domains->lock);
1558
1559	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_1);
1560	intel_power_well_disable(i915: dev_priv, power_well: well);
1561
1562	mutex_unlock(lock: &power_domains->lock);
1563
1564	usleep_range(min: 10, max: 30); /* 10 us delay per Bspec */
1565	}
1566
1567	struct buddy_page_mask {
1568	u32 page_mask;
1569	u8 type;
1570	u8 num_channels;
1571	};
1572
1573	static const struct buddy_page_mask tgl_buddy_page_masks[] = {
1574	{ .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0xF },
1575	{ .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0xF },
1576	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1C },
1577	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1C },
1578	{ .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x1F },
1579	{ .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x1E },
1580	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x38 },
1581	{ .num_channels = 4, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x38 },
1582	{}
1583	};
1584
1585	static const struct buddy_page_mask wa_1409767108_buddy_page_masks[] = {
1586	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x1 },
1587	{ .num_channels = 1, .type = INTEL_DRAM_DDR4, .page_mask = 0x1 },
1588	{ .num_channels = 1, .type = INTEL_DRAM_DDR5, .page_mask = 0x1 },
1589	{ .num_channels = 1, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x1 },
1590	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR4, .page_mask = 0x3 },
1591	{ .num_channels = 2, .type = INTEL_DRAM_DDR4, .page_mask = 0x3 },
1592	{ .num_channels = 2, .type = INTEL_DRAM_DDR5, .page_mask = 0x3 },
1593	{ .num_channels = 2, .type = INTEL_DRAM_LPDDR5, .page_mask = 0x3 },
1594	{}
1595	};
1596
1597	static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
1598	{
1599	enum intel_dram_type type = dev_priv->dram_info.type;
1600	u8 num_channels = dev_priv->dram_info.num_channels;
1601	const struct buddy_page_mask *table;
1602	unsigned long abox_mask = DISPLAY_INFO(dev_priv)->abox_mask;
1603	int config, i;
1604
1605	/* BW_BUDDY registers are not used on dgpu's beyond DG1 */
1606	if (IS_DGFX(dev_priv) && !IS_DG1(dev_priv))
1607	return;
1608
1609	if (IS_ALDERLAKE_S(dev_priv) ||
1610	(IS_ROCKETLAKE(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)))
1611	/* Wa_1409767108 */
1612	table = wa_1409767108_buddy_page_masks;
1613	else
1614	table = tgl_buddy_page_masks;
1615
1616	for (config = 0; table[config].page_mask != 0; config++)
1617	if (table[config].num_channels == num_channels &&
1618	table[config].type == type)
1619	break;
1620
1621	if (table[config].page_mask == 0) {
1622	drm_dbg(&dev_priv->drm,
1623	"Unknown memory configuration; disabling address buddy logic.\n");
1624	for_each_set_bit(i, &abox_mask, sizeof(abox_mask))
1625	intel_de_write(i915: dev_priv, BW_BUDDY_CTL(i),
1626	BW_BUDDY_DISABLE);
1627	} else {
1628	for_each_set_bit(i, &abox_mask, sizeof(abox_mask)) {
1629	intel_de_write(i915: dev_priv, BW_BUDDY_PAGE_MASK(i),
1630	val: table[config].page_mask);
1631
1632	/* Wa_22010178259:tgl,dg1,rkl,adl-s */
1633	if (DISPLAY_VER(dev_priv) == 12)
1634	intel_de_rmw(i915: dev_priv, BW_BUDDY_CTL(i),
1635	BW_BUDDY_TLB_REQ_TIMER_MASK,
1636	BW_BUDDY_TLB_REQ_TIMER(0x8));
1637	}
1638	}
1639	}
1640
1641	static void icl_display_core_init(struct drm_i915_private *dev_priv,
1642	bool resume)
1643	{
1644	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1645	struct i915_power_well *well;
1646
1647	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
1648
1649	/* Wa_14011294188:ehl,jsl,tgl,rkl,adl-s */
1650	if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP &&
1651	INTEL_PCH_TYPE(dev_priv) < PCH_DG1)
1652	intel_de_rmw(i915: dev_priv, SOUTH_DSPCLK_GATE_D, clear: 0,
1653	PCH_DPMGUNIT_CLOCK_GATE_DISABLE);
1654
1655	/* 1. Enable PCH reset handshake. */
1656	intel_pch_reset_handshake(dev_priv, enable: !HAS_PCH_NOP(dev_priv));
1657
1658	if (!HAS_DISPLAY(dev_priv))
1659	return;
1660
1661	/* 2. Initialize all combo phys */
1662	intel_combo_phy_init(dev_priv);
1663
1664	/*
1665	* 3. Enable Power Well 1 (PG1).
1666	* The AUX IO power wells will be enabled on demand.
1667	*/
1668	mutex_lock(&power_domains->lock);
1669	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_1);
1670	intel_power_well_enable(i915: dev_priv, power_well: well);
1671	mutex_unlock(lock: &power_domains->lock);
1672
1673	if (DISPLAY_VER(dev_priv) == 14)
1674	intel_de_rmw(i915: dev_priv, DC_STATE_EN,
1675	HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH, set: 0);
1676
1677	/* 4. Enable CDCLK. */
1678	intel_cdclk_init_hw(i915: dev_priv);
1679
1680	if (DISPLAY_VER(dev_priv) >= 12)
1681	gen12_dbuf_slices_config(dev_priv);
1682
1683	/* 5. Enable DBUF. */
1684	gen9_dbuf_enable(dev_priv);
1685
1686	/* 6. Setup MBUS. */
1687	icl_mbus_init(dev_priv);
1688
1689	/* 7. Program arbiter BW_BUDDY registers */
1690	if (DISPLAY_VER(dev_priv) >= 12)
1691	tgl_bw_buddy_init(dev_priv);
1692
1693	/* 8. Ensure PHYs have completed calibration and adaptation */
1694	if (IS_DG2(dev_priv))
1695	intel_snps_phy_wait_for_calibration(dev_priv);
1696
1697	if (resume)
1698	intel_dmc_load_program(i915: dev_priv);
1699
1700	/* Wa_14011508470:tgl,dg1,rkl,adl-s,adl-p,dg2 */
1701	if (IS_DISPLAY_IP_RANGE(dev_priv, IP_VER(12, 0), IP_VER(13, 0)))
1702	intel_de_rmw(i915: dev_priv, GEN11_CHICKEN_DCPR_2, clear: 0,
1703	DCPR_CLEAR_MEMSTAT_DIS | DCPR_SEND_RESP_IMM |
1704	DCPR_MASK_LPMODE | DCPR_MASK_MAXLATENCY_MEMUP_CLR);
1705
1706	/* Wa_14011503030:xelpd */
1707	if (DISPLAY_VER(dev_priv) == 13)
1708	intel_de_write(i915: dev_priv, XELPD_DISPLAY_ERR_FATAL_MASK, val: ~0);
1709	}
1710
1711	static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
1712	{
1713	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
1714	struct i915_power_well *well;
1715
1716	if (!HAS_DISPLAY(dev_priv))
1717	return;
1718
1719	gen9_disable_dc_states(dev_priv);
1720	intel_dmc_disable_program(i915: dev_priv);
1721
1722	/* 1. Disable all display engine functions -> aready done */
1723
1724	/* 2. Disable DBUF */
1725	gen9_dbuf_disable(dev_priv);
1726
1727	/* 3. Disable CD clock */
1728	intel_cdclk_uninit_hw(i915: dev_priv);
1729
1730	if (DISPLAY_VER(dev_priv) == 14)
1731	intel_de_rmw(i915: dev_priv, DC_STATE_EN, clear: 0,
1732	HOLD_PHY_PG1_LATCH | HOLD_PHY_CLKREQ_PG1_LATCH);
1733
1734	/*
1735	* 4. Disable Power Well 1 (PG1).
1736	* The AUX IO power wells are toggled on demand, so they are already
1737	* disabled at this point.
1738	*/
1739	mutex_lock(&power_domains->lock);
1740	well = lookup_power_well(i915: dev_priv, id: SKL_DISP_PW_1);
1741	intel_power_well_disable(i915: dev_priv, power_well: well);
1742	mutex_unlock(lock: &power_domains->lock);
1743
1744	/* 5. */
1745	intel_combo_phy_uninit(dev_priv);
1746	}
1747
1748	static void chv_phy_control_init(struct drm_i915_private *dev_priv)
1749	{
1750	struct i915_power_well *cmn_bc =
1751	lookup_power_well(i915: dev_priv, id: VLV_DISP_PW_DPIO_CMN_BC);
1752	struct i915_power_well *cmn_d =
1753	lookup_power_well(i915: dev_priv, id: CHV_DISP_PW_DPIO_CMN_D);
1754
1755	/*
1756	* DISPLAY_PHY_CONTROL can get corrupted if read. As a
1757	* workaround never ever read DISPLAY_PHY_CONTROL, and
1758	* instead maintain a shadow copy ourselves. Use the actual
1759	* power well state and lane status to reconstruct the
1760	* expected initial value.
1761	*/
1762	dev_priv->display.power.chv_phy_control =
1763	PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
1764	PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
1765	PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
1766	PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
1767	PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
1768
1769	/*
1770	* If all lanes are disabled we leave the override disabled
1771	* with all power down bits cleared to match the state we
1772	* would use after disabling the port. Otherwise enable the
1773	* override and set the lane powerdown bits accding to the
1774	* current lane status.
1775	*/
1776	if (intel_power_well_is_enabled(i915: dev_priv, power_well: cmn_bc)) {
1777	u32 status = intel_de_read(i915: dev_priv, DPLL(PIPE_A));
1778	unsigned int mask;
1779
1780	mask = status & DPLL_PORTB_READY_MASK;
1781	if (mask == 0xf)
1782	mask = 0x0;
1783	else
1784	dev_priv->display.power.chv_phy_control |=
1785	PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
1786
1787	dev_priv->display.power.chv_phy_control |=
1788	PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
1789
1790	mask = (status & DPLL_PORTC_READY_MASK) >> 4;
1791	if (mask == 0xf)
1792	mask = 0x0;
1793	else
1794	dev_priv->display.power.chv_phy_control |=
1795	PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
1796
1797	dev_priv->display.power.chv_phy_control |=
1798	PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
1799
1800	dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
1801
1802	dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false;
1803	} else {
1804	dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true;
1805	}
1806
1807	if (intel_power_well_is_enabled(i915: dev_priv, power_well: cmn_d)) {
1808	u32 status = intel_de_read(i915: dev_priv, DPIO_PHY_STATUS);
1809	unsigned int mask;
1810
1811	mask = status & DPLL_PORTD_READY_MASK;
1812
1813	if (mask == 0xf)
1814	mask = 0x0;
1815	else
1816	dev_priv->display.power.chv_phy_control |=
1817	PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
1818
1819	dev_priv->display.power.chv_phy_control |=
1820	PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
1821
1822	dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
1823
1824	dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false;
1825	} else {
1826	dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true;
1827	}
1828
1829	drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
1830	dev_priv->display.power.chv_phy_control);
1831
1832	/* Defer application of initial phy_control to enabling the powerwell */
1833	}
1834
1835	static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
1836	{
1837	struct i915_power_well *cmn =
1838	lookup_power_well(i915: dev_priv, id: VLV_DISP_PW_DPIO_CMN_BC);
1839	struct i915_power_well *disp2d =
1840	lookup_power_well(i915: dev_priv, id: VLV_DISP_PW_DISP2D);
1841
1842	/* If the display might be already active skip this */
1843	if (intel_power_well_is_enabled(i915: dev_priv, power_well: cmn) &&
1844	intel_power_well_is_enabled(i915: dev_priv, power_well: disp2d) &&
1845	intel_de_read(i915: dev_priv, DPIO_CTL) & DPIO_CMNRST)
1846	return;
1847
1848	drm_dbg_kms(&dev_priv->drm, "toggling display PHY side reset\n");
1849
1850	/* cmnlane needs DPLL registers */
1851	intel_power_well_enable(i915: dev_priv, power_well: disp2d);
1852
1853	/*
1854	* From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
1855	* Need to assert and de-assert PHY SB reset by gating the
1856	* common lane power, then un-gating it.
1857	* Simply ungating isn't enough to reset the PHY enough to get
1858	* ports and lanes running.
1859	*/
1860	intel_power_well_disable(i915: dev_priv, power_well: cmn);
1861	}
1862
1863	static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
1864	{
1865	bool ret;
1866
1867	vlv_punit_get(i915: dev_priv);
1868	ret = (vlv_punit_read(i915: dev_priv, addr: reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
1869	vlv_punit_put(i915: dev_priv);
1870
1871	return ret;
1872	}
1873
1874	static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
1875	{
1876	drm_WARN(&dev_priv->drm,
1877	!vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
1878	"VED not power gated\n");
1879	}
1880
1881	static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
1882	{
1883	static const struct pci_device_id isp_ids[] = {
1884	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
1885	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
1886	{}
1887	};
1888
1889	drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) &&
1890	!vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
1891	"ISP not power gated\n");
1892	}
1893
1894	static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
1895
1896	/**
1897	* intel_power_domains_init_hw - initialize hardware power domain state
1898	* @i915: i915 device instance
1899	* @resume: Called from resume code paths or not
1900	*
1901	* This function initializes the hardware power domain state and enables all
1902	* power wells belonging to the INIT power domain. Power wells in other
1903	* domains (and not in the INIT domain) are referenced or disabled by
1904	* intel_modeset_readout_hw_state(). After that the reference count of each
1905	* power well must match its HW enabled state, see
1906	* intel_power_domains_verify_state().
1907	*
1908	* It will return with power domains disabled (to be enabled later by
1909	* intel_power_domains_enable()) and must be paired with
1910	* intel_power_domains_driver_remove().
1911	*/
1912	void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
1913	{
1914	struct i915_power_domains *power_domains = &i915->display.power.domains;
1915
1916	power_domains->initializing = true;
1917
1918	if (DISPLAY_VER(i915) >= 11) {
1919	icl_display_core_init(dev_priv: i915, resume);
1920	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
1921	bxt_display_core_init(dev_priv: i915, resume);
1922	} else if (DISPLAY_VER(i915) == 9) {
1923	skl_display_core_init(dev_priv: i915, resume);
1924	} else if (IS_CHERRYVIEW(i915)) {
1925	mutex_lock(&power_domains->lock);
1926	chv_phy_control_init(dev_priv: i915);
1927	mutex_unlock(lock: &power_domains->lock);
1928	assert_isp_power_gated(dev_priv: i915);
1929	} else if (IS_VALLEYVIEW(i915)) {
1930	mutex_lock(&power_domains->lock);
1931	vlv_cmnlane_wa(dev_priv: i915);
1932	mutex_unlock(lock: &power_domains->lock);
1933	assert_ved_power_gated(dev_priv: i915);
1934	assert_isp_power_gated(dev_priv: i915);
1935	} else if (IS_BROADWELL(i915) || IS_HASWELL(i915)) {
1936	hsw_assert_cdclk(dev_priv: i915);
1937	intel_pch_reset_handshake(dev_priv: i915, enable: !HAS_PCH_NOP(i915));
1938	} else if (IS_IVYBRIDGE(i915)) {
1939	intel_pch_reset_handshake(dev_priv: i915, enable: !HAS_PCH_NOP(i915));
1940	}
1941
1942	/*
1943	* Keep all power wells enabled for any dependent HW access during
1944	* initialization and to make sure we keep BIOS enabled display HW
1945	* resources powered until display HW readout is complete. We drop
1946	* this reference in intel_power_domains_enable().
1947	*/
1948	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
1949	power_domains->init_wakeref =
1950	intel_display_power_get(dev_priv: i915, domain: POWER_DOMAIN_INIT);
1951
1952	/* Disable power support if the user asked so. */
1953	if (!i915->display.params.disable_power_well) {
1954	drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
1955	i915->display.power.domains.disable_wakeref = intel_display_power_get(dev_priv: i915,
1956	domain: POWER_DOMAIN_INIT);
1957	}
1958	intel_power_domains_sync_hw(dev_priv: i915);
1959
1960	power_domains->initializing = false;
1961	}
1962
1963	/**
1964	* intel_power_domains_driver_remove - deinitialize hw power domain state
1965	* @i915: i915 device instance
1966	*
1967	* De-initializes the display power domain HW state. It also ensures that the
1968	* device stays powered up so that the driver can be reloaded.
1969	*
1970	* It must be called with power domains already disabled (after a call to
1971	* intel_power_domains_disable()) and must be paired with
1972	* intel_power_domains_init_hw().
1973	*/
1974	void intel_power_domains_driver_remove(struct drm_i915_private *i915)
1975	{
1976	intel_wakeref_t wakeref __maybe_unused =
1977	fetch_and_zero(&i915->display.power.domains.init_wakeref);
1978
1979	/* Remove the refcount we took to keep power well support disabled. */
1980	if (!i915->display.params.disable_power_well)
1981	intel_display_power_put(dev_priv: i915, domain: POWER_DOMAIN_INIT,
1982	fetch_and_zero(&i915->display.power.domains.disable_wakeref));
1983
1984	intel_display_power_flush_work_sync(i915);
1985
1986	intel_power_domains_verify_state(dev_priv: i915);
1987
1988	/* Keep the power well enabled, but cancel its rpm wakeref. */
1989	intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref);
1990	}
1991
1992	/**
1993	* intel_power_domains_sanitize_state - sanitize power domains state
1994	* @i915: i915 device instance
1995	*
1996	* Sanitize the power domains state during driver loading and system resume.
1997	* The function will disable all display power wells that BIOS has enabled
1998	* without a user for it (any user for a power well has taken a reference
1999	* on it by the time this function is called, after the state of all the
2000	* pipe, encoder, etc. HW resources have been sanitized).
2001	*/
2002	void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
2003	{
2004	struct i915_power_domains *power_domains = &i915->display.power.domains;
2005	struct i915_power_well *power_well;
2006
2007	mutex_lock(&power_domains->lock);
2008
2009	for_each_power_well_reverse(i915, power_well) {
2010	if (power_well->desc->always_on || power_well->count ||
2011	!intel_power_well_is_enabled(i915, power_well))
2012	continue;
2013
2014	drm_dbg_kms(&i915->drm,
2015	"BIOS left unused %s power well enabled, disabling it\n",
2016	intel_power_well_name(power_well));
2017	intel_power_well_disable(i915, power_well);
2018	}
2019
2020	mutex_unlock(lock: &power_domains->lock);
2021	}
2022
2023	/**
2024	* intel_power_domains_enable - enable toggling of display power wells
2025	* @i915: i915 device instance
2026	*
2027	* Enable the ondemand enabling/disabling of the display power wells. Note that
2028	* power wells not belonging to POWER_DOMAIN_INIT are allowed to be toggled
2029	* only at specific points of the display modeset sequence, thus they are not
2030	* affected by the intel_power_domains_enable()/disable() calls. The purpose
2031	* of these function is to keep the rest of power wells enabled until the end
2032	* of display HW readout (which will acquire the power references reflecting
2033	* the current HW state).
2034	*/
2035	void intel_power_domains_enable(struct drm_i915_private *i915)
2036	{
2037	intel_wakeref_t wakeref __maybe_unused =
2038	fetch_and_zero(&i915->display.power.domains.init_wakeref);
2039
2040	intel_display_power_put(dev_priv: i915, domain: POWER_DOMAIN_INIT, wakeref);
2041	intel_power_domains_verify_state(dev_priv: i915);
2042	}
2043
2044	/**
2045	* intel_power_domains_disable - disable toggling of display power wells
2046	* @i915: i915 device instance
2047	*
2048	* Disable the ondemand enabling/disabling of the display power wells. See
2049	* intel_power_domains_enable() for which power wells this call controls.
2050	*/
2051	void intel_power_domains_disable(struct drm_i915_private *i915)
2052	{
2053	struct i915_power_domains *power_domains = &i915->display.power.domains;
2054
2055	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2056	power_domains->init_wakeref =
2057	intel_display_power_get(dev_priv: i915, domain: POWER_DOMAIN_INIT);
2058
2059	intel_power_domains_verify_state(dev_priv: i915);
2060	}
2061
2062	/**
2063	* intel_power_domains_suspend - suspend power domain state
2064	* @i915: i915 device instance
2065	* @s2idle: specifies whether we go to idle, or deeper sleep
2066	*
2067	* This function prepares the hardware power domain state before entering
2068	* system suspend.
2069	*
2070	* It must be called with power domains already disabled (after a call to
2071	* intel_power_domains_disable()) and paired with intel_power_domains_resume().
2072	*/
2073	void intel_power_domains_suspend(struct drm_i915_private *i915, bool s2idle)
2074	{
2075	struct i915_power_domains *power_domains = &i915->display.power.domains;
2076	intel_wakeref_t wakeref __maybe_unused =
2077	fetch_and_zero(&power_domains->init_wakeref);
2078
2079	intel_display_power_put(dev_priv: i915, domain: POWER_DOMAIN_INIT, wakeref);
2080
2081	/*
2082	* In case of suspend-to-idle (aka S0ix) on a DMC platform without DC9
2083	* support don't manually deinit the power domains. This also means the
2084	* DMC firmware will stay active, it will power down any HW
2085	* resources as required and also enable deeper system power states
2086	* that would be blocked if the firmware was inactive.
2087	*/
2088	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC9) && s2idle &&
2089	intel_dmc_has_payload(i915)) {
2090	intel_display_power_flush_work(i915);
2091	intel_power_domains_verify_state(dev_priv: i915);
2092	return;
2093	}
2094
2095	/*
2096	* Even if power well support was disabled we still want to disable
2097	* power wells if power domains must be deinitialized for suspend.
2098	*/
2099	if (!i915->display.params.disable_power_well)
2100	intel_display_power_put(dev_priv: i915, domain: POWER_DOMAIN_INIT,
2101	fetch_and_zero(&i915->display.power.domains.disable_wakeref));
2102
2103	intel_display_power_flush_work(i915);
2104	intel_power_domains_verify_state(dev_priv: i915);
2105
2106	if (DISPLAY_VER(i915) >= 11)
2107	icl_display_core_uninit(dev_priv: i915);
2108	else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
2109	bxt_display_core_uninit(dev_priv: i915);
2110	else if (DISPLAY_VER(i915) == 9)
2111	skl_display_core_uninit(dev_priv: i915);
2112
2113	power_domains->display_core_suspended = true;
2114	}
2115
2116	/**
2117	* intel_power_domains_resume - resume power domain state
2118	* @i915: i915 device instance
2119	*
2120	* This function resume the hardware power domain state during system resume.
2121	*
2122	* It will return with power domain support disabled (to be enabled later by
2123	* intel_power_domains_enable()) and must be paired with
2124	* intel_power_domains_suspend().
2125	*/
2126	void intel_power_domains_resume(struct drm_i915_private *i915)
2127	{
2128	struct i915_power_domains *power_domains = &i915->display.power.domains;
2129
2130	if (power_domains->display_core_suspended) {
2131	intel_power_domains_init_hw(i915, resume: true);
2132	power_domains->display_core_suspended = false;
2133	} else {
2134	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
2135	power_domains->init_wakeref =
2136	intel_display_power_get(dev_priv: i915, domain: POWER_DOMAIN_INIT);
2137	}
2138
2139	intel_power_domains_verify_state(dev_priv: i915);
2140	}
2141
2142	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM)
2143
2144	static void intel_power_domains_dump_info(struct drm_i915_private *i915)
2145	{
2146	struct i915_power_domains *power_domains = &i915->display.power.domains;
2147	struct i915_power_well *power_well;
2148
2149	for_each_power_well(i915, power_well) {
2150	enum intel_display_power_domain domain;
2151
2152	drm_dbg(&i915->drm, "%-25s %d\n",
2153	intel_power_well_name(power_well), intel_power_well_refcount(power_well));
2154
2155	for_each_power_domain(domain, intel_power_well_domains(power_well))
2156	drm_dbg(&i915->drm, " %-23s %d\n",
2157	intel_display_power_domain_str(domain),
2158	power_domains->domain_use_count[domain]);
2159	}
2160	}
2161
2162	/**
2163	* intel_power_domains_verify_state - verify the HW/SW state for all power wells
2164	* @i915: i915 device instance
2165	*
2166	* Verify if the reference count of each power well matches its HW enabled
2167	* state and the total refcount of the domains it belongs to. This must be
2168	* called after modeset HW state sanitization, which is responsible for
2169	* acquiring reference counts for any power wells in use and disabling the
2170	* ones left on by BIOS but not required by any active output.
2171	*/
2172	static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2173	{
2174	struct i915_power_domains *power_domains = &i915->display.power.domains;
2175	struct i915_power_well *power_well;
2176	bool dump_domain_info;
2177
2178	mutex_lock(&power_domains->lock);
2179
2180	verify_async_put_domains_state(power_domains);
2181
2182	dump_domain_info = false;
2183	for_each_power_well(i915, power_well) {
2184	enum intel_display_power_domain domain;
2185	int domains_count;
2186	bool enabled;
2187
2188	enabled = intel_power_well_is_enabled(i915, power_well);
2189	if ((intel_power_well_refcount(power_well) ||
2190	intel_power_well_is_always_on(power_well)) !=
2191	enabled)
2192	drm_err(&i915->drm,
2193	"power well %s state mismatch (refcount %d/enabled %d)",
2194	intel_power_well_name(power_well),
2195	intel_power_well_refcount(power_well), enabled);
2196
2197	domains_count = 0;
2198	for_each_power_domain(domain, intel_power_well_domains(power_well))
2199	domains_count += power_domains->domain_use_count[domain];
2200
2201	if (intel_power_well_refcount(power_well) != domains_count) {
2202	drm_err(&i915->drm,
2203	"power well %s refcount/domain refcount mismatch "
2204	"(refcount %d/domains refcount %d)\n",
2205	intel_power_well_name(power_well),
2206	intel_power_well_refcount(power_well),
2207	domains_count);
2208	dump_domain_info = true;
2209	}
2210	}
2211
2212	if (dump_domain_info) {
2213	static bool dumped;
2214
2215	if (!dumped) {
2216	intel_power_domains_dump_info(i915);
2217	dumped = true;
2218	}
2219	}
2220
2221	mutex_unlock(lock: &power_domains->lock);
2222	}
2223
2224	#else
2225
2226	static void intel_power_domains_verify_state(struct drm_i915_private *i915)
2227	{
2228	}
2229
2230	#endif
2231
2232	void intel_display_power_suspend_late(struct drm_i915_private *i915)
2233	{
2234	if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2235	IS_BROXTON(i915)) {
2236	bxt_enable_dc9(dev_priv: i915);
2237	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2238	hsw_enable_pc8(dev_priv: i915);
2239	}
2240
2241	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2242	if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2243	intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS);
2244	}
2245
2246	void intel_display_power_resume_early(struct drm_i915_private *i915)
2247	{
2248	if (DISPLAY_VER(i915) >= 11 || IS_GEMINILAKE(i915) ||
2249	IS_BROXTON(i915)) {
2250	gen9_sanitize_dc_state(dev_priv: i915);
2251	bxt_disable_dc9(dev_priv: i915);
2252	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2253	hsw_disable_pc8(dev_priv: i915);
2254	}
2255
2256	/* Tweaked Wa_14010685332:cnp,icp,jsp,mcc,tgp,adp */
2257	if (INTEL_PCH_TYPE(i915) >= PCH_CNP && INTEL_PCH_TYPE(i915) < PCH_DG1)
2258	intel_de_rmw(i915, SOUTH_CHICKEN1, SBCLK_RUN_REFCLK_DIS, set: 0);
2259	}
2260
2261	void intel_display_power_suspend(struct drm_i915_private *i915)
2262	{
2263	if (DISPLAY_VER(i915) >= 11) {
2264	icl_display_core_uninit(dev_priv: i915);
2265	bxt_enable_dc9(dev_priv: i915);
2266	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2267	bxt_display_core_uninit(dev_priv: i915);
2268	bxt_enable_dc9(dev_priv: i915);
2269	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2270	hsw_enable_pc8(dev_priv: i915);
2271	}
2272	}
2273
2274	void intel_display_power_resume(struct drm_i915_private *i915)
2275	{
2276	struct i915_power_domains *power_domains = &i915->display.power.domains;
2277
2278	if (DISPLAY_VER(i915) >= 11) {
2279	bxt_disable_dc9(dev_priv: i915);
2280	icl_display_core_init(dev_priv: i915, resume: true);
2281	if (intel_dmc_has_payload(i915)) {
2282	if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
2283	skl_enable_dc6(dev_priv: i915);
2284	else if (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
2285	gen9_enable_dc5(dev_priv: i915);
2286	}
2287	} else if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) {
2288	bxt_disable_dc9(dev_priv: i915);
2289	bxt_display_core_init(dev_priv: i915, resume: true);
2290	if (intel_dmc_has_payload(i915) &&
2291	(power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
2292	gen9_enable_dc5(dev_priv: i915);
2293	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2294	hsw_disable_pc8(dev_priv: i915);
2295	}
2296	}
2297
2298	void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m)
2299	{
2300	struct i915_power_domains *power_domains = &i915->display.power.domains;
2301	int i;
2302
2303	mutex_lock(&power_domains->lock);
2304
2305	seq_printf(m, fmt: "%-25s %s\n", "Power well/domain", "Use count");
2306	for (i = 0; i < power_domains->power_well_count; i++) {
2307	struct i915_power_well *power_well;
2308	enum intel_display_power_domain power_domain;
2309
2310	power_well = &power_domains->power_wells[i];
2311	seq_printf(m, fmt: "%-25s %d\n", intel_power_well_name(power_well),
2312	intel_power_well_refcount(power_well));
2313
2314	for_each_power_domain(power_domain, intel_power_well_domains(power_well))
2315	seq_printf(m, fmt: " %-23s %d\n",
2316	intel_display_power_domain_str(domain: power_domain),
2317	power_domains->domain_use_count[power_domain]);
2318	}
2319
2320	mutex_unlock(lock: &power_domains->lock);
2321	}
2322
2323	struct intel_ddi_port_domains {
2324	enum port port_start;
2325	enum port port_end;
2326	enum aux_ch aux_ch_start;
2327	enum aux_ch aux_ch_end;
2328
2329	enum intel_display_power_domain ddi_lanes;
2330	enum intel_display_power_domain ddi_io;
2331	enum intel_display_power_domain aux_io;
2332	enum intel_display_power_domain aux_legacy_usbc;
2333	enum intel_display_power_domain aux_tbt;
2334	};
2335
2336	static const struct intel_ddi_port_domains
2337	i9xx_port_domains[] = {
2338	{
2339	.port_start = PORT_A,
2340	.port_end = PORT_F,
2341	.aux_ch_start = AUX_CH_A,
2342	.aux_ch_end = AUX_CH_F,
2343
2344	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2345	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2346	.aux_io = POWER_DOMAIN_AUX_IO_A,
2347	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2348	.aux_tbt = POWER_DOMAIN_INVALID,
2349	},
2350	};
2351
2352	static const struct intel_ddi_port_domains
2353	d11_port_domains[] = {
2354	{
2355	.port_start = PORT_A,
2356	.port_end = PORT_B,
2357	.aux_ch_start = AUX_CH_A,
2358	.aux_ch_end = AUX_CH_B,
2359
2360	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2361	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2362	.aux_io = POWER_DOMAIN_AUX_IO_A,
2363	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2364	.aux_tbt = POWER_DOMAIN_INVALID,
2365	}, {
2366	.port_start = PORT_C,
2367	.port_end = PORT_F,
2368	.aux_ch_start = AUX_CH_C,
2369	.aux_ch_end = AUX_CH_F,
2370
2371	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_C,
2372	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_C,
2373	.aux_io = POWER_DOMAIN_AUX_IO_C,
2374	.aux_legacy_usbc = POWER_DOMAIN_AUX_C,
2375	.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2376	},
2377	};
2378
2379	static const struct intel_ddi_port_domains
2380	d12_port_domains[] = {
2381	{
2382	.port_start = PORT_A,
2383	.port_end = PORT_C,
2384	.aux_ch_start = AUX_CH_A,
2385	.aux_ch_end = AUX_CH_C,
2386
2387	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2388	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2389	.aux_io = POWER_DOMAIN_AUX_IO_A,
2390	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2391	.aux_tbt = POWER_DOMAIN_INVALID,
2392	}, {
2393	.port_start = PORT_TC1,
2394	.port_end = PORT_TC6,
2395	.aux_ch_start = AUX_CH_USBC1,
2396	.aux_ch_end = AUX_CH_USBC6,
2397
2398	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2399	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2400	.aux_io = POWER_DOMAIN_INVALID,
2401	.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2402	.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2403	},
2404	};
2405
2406	static const struct intel_ddi_port_domains
2407	d13_port_domains[] = {
2408	{
2409	.port_start = PORT_A,
2410	.port_end = PORT_C,
2411	.aux_ch_start = AUX_CH_A,
2412	.aux_ch_end = AUX_CH_C,
2413
2414	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_A,
2415	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_A,
2416	.aux_io = POWER_DOMAIN_AUX_IO_A,
2417	.aux_legacy_usbc = POWER_DOMAIN_AUX_A,
2418	.aux_tbt = POWER_DOMAIN_INVALID,
2419	}, {
2420	.port_start = PORT_TC1,
2421	.port_end = PORT_TC4,
2422	.aux_ch_start = AUX_CH_USBC1,
2423	.aux_ch_end = AUX_CH_USBC4,
2424
2425	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_TC1,
2426	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_TC1,
2427	.aux_io = POWER_DOMAIN_INVALID,
2428	.aux_legacy_usbc = POWER_DOMAIN_AUX_USBC1,
2429	.aux_tbt = POWER_DOMAIN_AUX_TBT1,
2430	}, {
2431	.port_start = PORT_D_XELPD,
2432	.port_end = PORT_E_XELPD,
2433	.aux_ch_start = AUX_CH_D_XELPD,
2434	.aux_ch_end = AUX_CH_E_XELPD,
2435
2436	.ddi_lanes = POWER_DOMAIN_PORT_DDI_LANES_D,
2437	.ddi_io = POWER_DOMAIN_PORT_DDI_IO_D,
2438	.aux_io = POWER_DOMAIN_AUX_IO_D,
2439	.aux_legacy_usbc = POWER_DOMAIN_AUX_D,
2440	.aux_tbt = POWER_DOMAIN_INVALID,
2441	},
2442	};
2443
2444	static void
2445	intel_port_domains_for_platform(struct drm_i915_private *i915,
2446	const struct intel_ddi_port_domains **domains,
2447	int *domains_size)
2448	{
2449	if (DISPLAY_VER(i915) >= 13) {
2450	*domains = d13_port_domains;
2451	*domains_size = ARRAY_SIZE(d13_port_domains);
2452	} else if (DISPLAY_VER(i915) >= 12) {
2453	*domains = d12_port_domains;
2454	*domains_size = ARRAY_SIZE(d12_port_domains);
2455	} else if (DISPLAY_VER(i915) >= 11) {
2456	*domains = d11_port_domains;
2457	*domains_size = ARRAY_SIZE(d11_port_domains);
2458	} else {
2459	*domains = i9xx_port_domains;
2460	*domains_size = ARRAY_SIZE(i9xx_port_domains);
2461	}
2462	}
2463
2464	static const struct intel_ddi_port_domains *
2465	intel_port_domains_for_port(struct drm_i915_private *i915, enum port port)
2466	{
2467	const struct intel_ddi_port_domains *domains;
2468	int domains_size;
2469	int i;
2470
2471	intel_port_domains_for_platform(i915, domains: &domains, domains_size: &domains_size);
2472	for (i = 0; i < domains_size; i++)
2473	if (port >= domains[i].port_start && port <= domains[i].port_end)
2474	return &domains[i];
2475
2476	return NULL;
2477	}
2478
2479	enum intel_display_power_domain
2480	intel_display_power_ddi_io_domain(struct drm_i915_private *i915, enum port port)
2481	{
2482	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2483
2484	if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_io == POWER_DOMAIN_INVALID))
2485	return POWER_DOMAIN_PORT_DDI_IO_A;
2486
2487	return domains->ddi_io + (int)(port - domains->port_start);
2488	}
2489
2490	enum intel_display_power_domain
2491	intel_display_power_ddi_lanes_domain(struct drm_i915_private *i915, enum port port)
2492	{
2493	const struct intel_ddi_port_domains *domains = intel_port_domains_for_port(i915, port);
2494
2495	if (drm_WARN_ON(&i915->drm, !domains || domains->ddi_lanes == POWER_DOMAIN_INVALID))
2496	return POWER_DOMAIN_PORT_DDI_LANES_A;
2497
2498	return domains->ddi_lanes + (int)(port - domains->port_start);
2499	}
2500
2501	static const struct intel_ddi_port_domains *
2502	intel_port_domains_for_aux_ch(struct drm_i915_private *i915, enum aux_ch aux_ch)
2503	{
2504	const struct intel_ddi_port_domains *domains;
2505	int domains_size;
2506	int i;
2507
2508	intel_port_domains_for_platform(i915, domains: &domains, domains_size: &domains_size);
2509	for (i = 0; i < domains_size; i++)
2510	if (aux_ch >= domains[i].aux_ch_start && aux_ch <= domains[i].aux_ch_end)
2511	return &domains[i];
2512
2513	return NULL;
2514	}
2515
2516	enum intel_display_power_domain
2517	intel_display_power_aux_io_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2518	{
2519	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2520
2521	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_io == POWER_DOMAIN_INVALID))
2522	return POWER_DOMAIN_AUX_IO_A;
2523
2524	return domains->aux_io + (int)(aux_ch - domains->aux_ch_start);
2525	}
2526
2527	enum intel_display_power_domain
2528	intel_display_power_legacy_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2529	{
2530	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2531
2532	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_legacy_usbc == POWER_DOMAIN_INVALID))
2533	return POWER_DOMAIN_AUX_A;
2534
2535	return domains->aux_legacy_usbc + (int)(aux_ch - domains->aux_ch_start);
2536	}
2537
2538	enum intel_display_power_domain
2539	intel_display_power_tbt_aux_domain(struct drm_i915_private *i915, enum aux_ch aux_ch)
2540	{
2541	const struct intel_ddi_port_domains *domains = intel_port_domains_for_aux_ch(i915, aux_ch);
2542
2543	if (drm_WARN_ON(&i915->drm, !domains || domains->aux_tbt == POWER_DOMAIN_INVALID))
2544	return POWER_DOMAIN_AUX_TBT1;
2545
2546	return domains->aux_tbt + (int)(aux_ch - domains->aux_ch_start);
2547	}
2548