1	/*
2	* Copyright © 2013 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*/
23
24	#include <drm/drm_managed.h>
25	#include <linux/pm_runtime.h>
26
27	#include "gt/intel_engine_regs.h"
28	#include "gt/intel_gt_regs.h"
29
30	#include "i915_drv.h"
31	#include "i915_iosf_mbi.h"
32	#include "i915_reg.h"
33	#include "i915_trace.h"
34	#include "i915_vgpu.h"
35
36	#define FORCEWAKE_ACK_TIMEOUT_MS 50
37	#define GT_FIFO_TIMEOUT_MS 10
38
39	#define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__))
40
41	static void
42	fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
43	{
44	uncore->fw_get_funcs->force_wake_get(uncore, fw_domains);
45	}
46
47	void
48	intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915)
49	{
50	spin_lock_init(&i915->mmio_debug.lock);
51	i915->mmio_debug.unclaimed_mmio_check = 1;
52
53	i915->uncore.debug = &i915->mmio_debug;
54	}
55
56	static void mmio_debug_suspend(struct intel_uncore *uncore)
57	{
58	if (!uncore->debug)
59	return;
60
61	spin_lock(lock: &uncore->debug->lock);
62
63	/* Save and disable mmio debugging for the user bypass */
64	if (!uncore->debug->suspend_count++) {
65	uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check;
66	uncore->debug->unclaimed_mmio_check = 0;
67	}
68
69	spin_unlock(lock: &uncore->debug->lock);
70	}
71
72	static bool check_for_unclaimed_mmio(struct intel_uncore *uncore);
73
74	static void mmio_debug_resume(struct intel_uncore *uncore)
75	{
76	if (!uncore->debug)
77	return;
78
79	spin_lock(lock: &uncore->debug->lock);
80
81	if (!--uncore->debug->suspend_count)
82	uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check;
83
84	if (check_for_unclaimed_mmio(uncore))
85	drm_info(&uncore->i915->drm,
86	"Invalid mmio detected during user access\n");
87
88	spin_unlock(lock: &uncore->debug->lock);
89	}
90
91	static const char * const forcewake_domain_names[] = {
92	"render",
93	"gt",
94	"media",
95	"vdbox0",
96	"vdbox1",
97	"vdbox2",
98	"vdbox3",
99	"vdbox4",
100	"vdbox5",
101	"vdbox6",
102	"vdbox7",
103	"vebox0",
104	"vebox1",
105	"vebox2",
106	"vebox3",
107	"gsc",
108	};
109
110	const char *
111	intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
112	{
113	BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
114
115	if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
116	return forcewake_domain_names[id];
117
118	WARN_ON(id);
119
120	return "unknown";
121	}
122
123	#define fw_ack(d) readl((d)->reg_ack)
124	#define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
125	#define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
126
127	static inline void
128	fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
129	{
130	/*
131	* We don't really know if the powerwell for the forcewake domain we are
132	* trying to reset here does exist at this point (engines could be fused
133	* off in ICL+), so no waiting for acks
134	*/
135	/* WaRsClearFWBitsAtReset */
136	if (GRAPHICS_VER(d->uncore->i915) >= 12)
137	fw_clear(d, 0xefff);
138	else
139	fw_clear(d, 0xffff);
140	}
141
142	static inline void
143	fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
144	{
145	GEM_BUG_ON(d->uncore->fw_domains_timer & d->mask);
146	d->uncore->fw_domains_timer |= d->mask;
147	d->wake_count++;
148	hrtimer_start_range_ns(timer: &d->timer,
149	NSEC_PER_MSEC,
150	NSEC_PER_MSEC,
151	mode: HRTIMER_MODE_REL);
152	}
153
154	static inline int
155	__wait_for_ack(const struct intel_uncore_forcewake_domain *d,
156	const u32 ack,
157	const u32 value)
158	{
159	return wait_for_atomic((fw_ack(d) & ack) == value,
160	FORCEWAKE_ACK_TIMEOUT_MS);
161	}
162
163	static inline int
164	wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
165	const u32 ack)
166	{
167	return __wait_for_ack(d, ack, value: 0);
168	}
169
170	static inline int
171	wait_ack_set(const struct intel_uncore_forcewake_domain *d,
172	const u32 ack)
173	{
174	return __wait_for_ack(d, ack, value: ack);
175	}
176
177	static inline void
178	fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
179	{
180	if (!wait_ack_clear(d, FORCEWAKE_KERNEL))
181	return;
182
183	if (fw_ack(d) == ~0)
184	drm_err(&d->uncore->i915->drm,
185	"%s: MMIO unreliable (forcewake register returns 0xFFFFFFFF)!\n",
186	intel_uncore_forcewake_domain_to_str(d->id));
187	else
188	drm_err(&d->uncore->i915->drm,
189	"%s: timed out waiting for forcewake ack to clear.\n",
190	intel_uncore_forcewake_domain_to_str(d->id));
191
192	add_taint_for_CI(i915: d->uncore->i915, TAINT_WARN); /* CI now unreliable */
193	}
194
195	enum ack_type {
196	ACK_CLEAR = 0,
197	ACK_SET
198	};
199
200	static int
201	fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
202	const enum ack_type type)
203	{
204	const u32 ack_bit = FORCEWAKE_KERNEL;
205	const u32 value = type == ACK_SET ? ack_bit : 0;
206	unsigned int pass;
207	bool ack_detected;
208
209	/*
210	* There is a possibility of driver's wake request colliding
211	* with hardware's own wake requests and that can cause
212	* hardware to not deliver the driver's ack message.
213	*
214	* Use a fallback bit toggle to kick the gpu state machine
215	* in the hope that the original ack will be delivered along with
216	* the fallback ack.
217	*
218	* This workaround is described in HSDES #1604254524 and it's known as:
219	* WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
220	* although the name is a bit misleading.
221	*/
222
223	pass = 1;
224	do {
225	wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
226
227	fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
228	/* Give gt some time to relax before the polling frenzy */
229	udelay(10 * pass);
230	wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
231
232	ack_detected = (fw_ack(d) & ack_bit) == value;
233
234	fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
235	} while (!ack_detected && pass++ < 10);
236
237	drm_dbg(&d->uncore->i915->drm,
238	"%s had to use fallback to %s ack, 0x%x (passes %u)\n",
239	intel_uncore_forcewake_domain_to_str(d->id),
240	type == ACK_SET ? "set" : "clear",
241	fw_ack(d),
242	pass);
243
244	return ack_detected ? 0 : -ETIMEDOUT;
245	}
246
247	static inline void
248	fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
249	{
250	if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
251	return;
252
253	if (fw_domain_wait_ack_with_fallback(d, type: ACK_CLEAR))
254	fw_domain_wait_ack_clear(d);
255	}
256
257	static inline void
258	fw_domain_get(const struct intel_uncore_forcewake_domain *d)
259	{
260	fw_set(d, FORCEWAKE_KERNEL);
261	}
262
263	static inline void
264	fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
265	{
266	if (wait_ack_set(d, FORCEWAKE_KERNEL)) {
267	drm_err(&d->uncore->i915->drm,
268	"%s: timed out waiting for forcewake ack request.\n",
269	intel_uncore_forcewake_domain_to_str(d->id));
270	add_taint_for_CI(i915: d->uncore->i915, TAINT_WARN); /* CI now unreliable */
271	}
272	}
273
274	static inline void
275	fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
276	{
277	if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
278	return;
279
280	if (fw_domain_wait_ack_with_fallback(d, type: ACK_SET))
281	fw_domain_wait_ack_set(d);
282	}
283
284	static inline void
285	fw_domain_put(const struct intel_uncore_forcewake_domain *d)
286	{
287	fw_clear(d, FORCEWAKE_KERNEL);
288	}
289
290	static void
291	fw_domains_get_normal(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
292	{
293	struct intel_uncore_forcewake_domain *d;
294	unsigned int tmp;
295
296	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
297
298	for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
299	fw_domain_wait_ack_clear(d);
300	fw_domain_get(d);
301	}
302
303	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
304	fw_domain_wait_ack_set(d);
305
306	uncore->fw_domains_active |= fw_domains;
307	}
308
309	static void
310	fw_domains_get_with_fallback(struct intel_uncore *uncore,
311	enum forcewake_domains fw_domains)
312	{
313	struct intel_uncore_forcewake_domain *d;
314	unsigned int tmp;
315
316	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
317
318	for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
319	fw_domain_wait_ack_clear_fallback(d);
320	fw_domain_get(d);
321	}
322
323	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
324	fw_domain_wait_ack_set_fallback(d);
325
326	uncore->fw_domains_active |= fw_domains;
327	}
328
329	static void
330	fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
331	{
332	struct intel_uncore_forcewake_domain *d;
333	unsigned int tmp;
334
335	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
336
337	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
338	fw_domain_put(d);
339
340	uncore->fw_domains_active &= ~fw_domains;
341	}
342
343	static void
344	fw_domains_reset(struct intel_uncore *uncore,
345	enum forcewake_domains fw_domains)
346	{
347	struct intel_uncore_forcewake_domain *d;
348	unsigned int tmp;
349
350	if (!fw_domains)
351	return;
352
353	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
354
355	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
356	fw_domain_reset(d);
357	}
358
359	static inline u32 gt_thread_status(struct intel_uncore *uncore)
360	{
361	u32 val;
362
363	val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG);
364	val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
365
366	return val;
367	}
368
369	static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore)
370	{
371	/*
372	* w/a for a sporadic read returning 0 by waiting for the GT
373	* thread to wake up.
374	*/
375	drm_WARN_ONCE(&uncore->i915->drm,
376	wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),
377	"GT thread status wait timed out\n");
378	}
379
380	static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
381	enum forcewake_domains fw_domains)
382	{
383	fw_domains_get_normal(uncore, fw_domains);
384
385	/* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
386	__gen6_gt_wait_for_thread_c0(uncore);
387	}
388
389	static inline u32 fifo_free_entries(struct intel_uncore *uncore)
390	{
391	u32 count = __raw_uncore_read32(uncore, GTFIFOCTL);
392
393	return count & GT_FIFO_FREE_ENTRIES_MASK;
394	}
395
396	static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore)
397	{
398	u32 n;
399
400	/* On VLV, FIFO will be shared by both SW and HW.
401	* So, we need to read the FREE_ENTRIES everytime */
402	if (IS_VALLEYVIEW(uncore->i915))
403	n = fifo_free_entries(uncore);
404	else
405	n = uncore->fifo_count;
406
407	if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
408	if (wait_for_atomic((n = fifo_free_entries(uncore)) >
409	GT_FIFO_NUM_RESERVED_ENTRIES,
410	GT_FIFO_TIMEOUT_MS)) {
411	drm_dbg(&uncore->i915->drm,
412	"GT_FIFO timeout, entries: %u\n", n);
413	return;
414	}
415	}
416
417	uncore->fifo_count = n - 1;
418	}
419
420	static enum hrtimer_restart
421	intel_uncore_fw_release_timer(struct hrtimer *timer)
422	{
423	struct intel_uncore_forcewake_domain *domain =
424	container_of(timer, struct intel_uncore_forcewake_domain, timer);
425	struct intel_uncore *uncore = domain->uncore;
426	unsigned long irqflags;
427
428	assert_rpm_device_not_suspended(rpm: uncore->rpm);
429
430	if (xchg(&domain->active, false))
431	return HRTIMER_RESTART;
432
433	spin_lock_irqsave(&uncore->lock, irqflags);
434
435	uncore->fw_domains_timer &= ~domain->mask;
436
437	GEM_BUG_ON(!domain->wake_count);
438	if (--domain->wake_count == 0)
439	fw_domains_put(uncore, fw_domains: domain->mask);
440
441	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
442
443	return HRTIMER_NORESTART;
444	}
445
446	/* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
447	static unsigned int
448	intel_uncore_forcewake_reset(struct intel_uncore *uncore)
449	{
450	unsigned long irqflags;
451	struct intel_uncore_forcewake_domain *domain;
452	int retry_count = 100;
453	enum forcewake_domains fw, active_domains;
454
455	iosf_mbi_assert_punit_acquired();
456
457	/* Hold uncore.lock across reset to prevent any register access
458	* with forcewake not set correctly. Wait until all pending
459	* timers are run before holding.
460	*/
461	while (1) {
462	unsigned int tmp;
463
464	active_domains = 0;
465
466	for_each_fw_domain(domain, uncore, tmp) {
467	smp_store_mb(domain->active, false);
468	if (hrtimer_cancel(timer: &domain->timer) == 0)
469	continue;
470
471	intel_uncore_fw_release_timer(timer: &domain->timer);
472	}
473
474	spin_lock_irqsave(&uncore->lock, irqflags);
475
476	for_each_fw_domain(domain, uncore, tmp) {
477	if (hrtimer_active(timer: &domain->timer))
478	active_domains |= domain->mask;
479	}
480
481	if (active_domains == 0)
482	break;
483
484	if (--retry_count == 0) {
485	drm_err(&uncore->i915->drm, "Timed out waiting for forcewake timers to finish\n");
486	break;
487	}
488
489	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
490	cond_resched();
491	}
492
493	drm_WARN_ON(&uncore->i915->drm, active_domains);
494
495	fw = uncore->fw_domains_active;
496	if (fw)
497	fw_domains_put(uncore, fw_domains: fw);
498
499	fw_domains_reset(uncore, fw_domains: uncore->fw_domains);
500	assert_forcewakes_inactive(uncore);
501
502	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
503
504	return fw; /* track the lost user forcewake domains */
505	}
506
507	static bool
508	fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore)
509	{
510	u32 dbg;
511
512	dbg = __raw_uncore_read32(uncore, FPGA_DBG);
513	if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
514	return false;
515
516	/*
517	* Bugs in PCI programming (or failing hardware) can occasionally cause
518	* us to lose access to the MMIO BAR. When this happens, register
519	* reads will come back with 0xFFFFFFFF for every register and things
520	* go bad very quickly. Let's try to detect that special case and at
521	* least try to print a more informative message about what has
522	* happened.
523	*
524	* During normal operation the FPGA_DBG register has several unused
525	* bits that will always read back as 0's so we can use them as canaries
526	* to recognize when MMIO accesses are just busted.
527	*/
528	if (unlikely(dbg == ~0))
529	drm_err(&uncore->i915->drm,
530	"Lost access to MMIO BAR; all registers now read back as 0xFFFFFFFF!\n");
531
532	__raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
533
534	return true;
535	}
536
537	static bool
538	vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore)
539	{
540	u32 cer;
541
542	cer = __raw_uncore_read32(uncore, CLAIM_ER);
543	if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
544	return false;
545
546	__raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR);
547
548	return true;
549	}
550
551	static bool
552	gen6_check_for_fifo_debug(struct intel_uncore *uncore)
553	{
554	u32 fifodbg;
555
556	fifodbg = __raw_uncore_read32(uncore, GTFIFODBG);
557
558	if (unlikely(fifodbg)) {
559	drm_dbg(&uncore->i915->drm, "GTFIFODBG = 0x08%x\n", fifodbg);
560	__raw_uncore_write32(uncore, GTFIFODBG, val: fifodbg);
561	}
562
563	return fifodbg;
564	}
565
566	static bool
567	check_for_unclaimed_mmio(struct intel_uncore *uncore)
568	{
569	bool ret = false;
570
571	lockdep_assert_held(&uncore->debug->lock);
572
573	if (uncore->debug->suspend_count)
574	return false;
575
576	if (intel_uncore_has_fpga_dbg_unclaimed(uncore))
577	ret |= fpga_check_for_unclaimed_mmio(uncore);
578
579	if (intel_uncore_has_dbg_unclaimed(uncore))
580	ret |= vlv_check_for_unclaimed_mmio(uncore);
581
582	if (intel_uncore_has_fifo(uncore))
583	ret |= gen6_check_for_fifo_debug(uncore);
584
585	return ret;
586	}
587
588	static void forcewake_early_sanitize(struct intel_uncore *uncore,
589	unsigned int restore_forcewake)
590	{
591	GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
592
593	/* WaDisableShadowRegForCpd:chv */
594	if (IS_CHERRYVIEW(uncore->i915)) {
595	__raw_uncore_write32(uncore, GTFIFOCTL,
596	val: __raw_uncore_read32(uncore, GTFIFOCTL) |
597	GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
598	GT_FIFO_CTL_RC6_POLICY_STALL);
599	}
600
601	iosf_mbi_punit_acquire();
602	intel_uncore_forcewake_reset(uncore);
603	if (restore_forcewake) {
604	spin_lock_irq(lock: &uncore->lock);
605	fw_domains_get(uncore, fw_domains: restore_forcewake);
606
607	if (intel_uncore_has_fifo(uncore))
608	uncore->fifo_count = fifo_free_entries(uncore);
609	spin_unlock_irq(lock: &uncore->lock);
610	}
611	iosf_mbi_punit_release();
612	}
613
614	void intel_uncore_suspend(struct intel_uncore *uncore)
615	{
616	if (!intel_uncore_has_forcewake(uncore))
617	return;
618
619	iosf_mbi_punit_acquire();
620	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
621	nb: &uncore->pmic_bus_access_nb);
622	uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore);
623	iosf_mbi_punit_release();
624	}
625
626	void intel_uncore_resume_early(struct intel_uncore *uncore)
627	{
628	unsigned int restore_forcewake;
629
630	if (intel_uncore_unclaimed_mmio(uncore))
631	drm_dbg(&uncore->i915->drm, "unclaimed mmio detected on resume, clearing\n");
632
633	if (!intel_uncore_has_forcewake(uncore))
634	return;
635
636	restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved);
637	forcewake_early_sanitize(uncore, restore_forcewake);
638
639	iosf_mbi_register_pmic_bus_access_notifier(nb: &uncore->pmic_bus_access_nb);
640	}
641
642	void intel_uncore_runtime_resume(struct intel_uncore *uncore)
643	{
644	if (!intel_uncore_has_forcewake(uncore))
645	return;
646
647	iosf_mbi_register_pmic_bus_access_notifier(nb: &uncore->pmic_bus_access_nb);
648	}
649
650	static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
651	enum forcewake_domains fw_domains)
652	{
653	struct intel_uncore_forcewake_domain *domain;
654	unsigned int tmp;
655
656	fw_domains &= uncore->fw_domains;
657
658	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
659	if (domain->wake_count++) {
660	fw_domains &= ~domain->mask;
661	domain->active = true;
662	}
663	}
664
665	if (fw_domains)
666	fw_domains_get(uncore, fw_domains);
667	}
668
669	/**
670	* intel_uncore_forcewake_get - grab forcewake domain references
671	* @uncore: the intel_uncore structure
672	* @fw_domains: forcewake domains to get reference on
673	*
674	* This function can be used get GT's forcewake domain references.
675	* Normal register access will handle the forcewake domains automatically.
676	* However if some sequence requires the GT to not power down a particular
677	* forcewake domains this function should be called at the beginning of the
678	* sequence. And subsequently the reference should be dropped by symmetric
679	* call to intel_unforce_forcewake_put(). Usually caller wants all the domains
680	* to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
681	*/
682	void intel_uncore_forcewake_get(struct intel_uncore *uncore,
683	enum forcewake_domains fw_domains)
684	{
685	unsigned long irqflags;
686
687	if (!uncore->fw_get_funcs)
688	return;
689
690	assert_rpm_wakelock_held(rpm: uncore->rpm);
691
692	spin_lock_irqsave(&uncore->lock, irqflags);
693	__intel_uncore_forcewake_get(uncore, fw_domains);
694	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
695	}
696
697	/**
698	* intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
699	* @uncore: the intel_uncore structure
700	*
701	* This function is a wrapper around intel_uncore_forcewake_get() to acquire
702	* the GT powerwell and in the process disable our debugging for the
703	* duration of userspace's bypass.
704	*/
705	void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
706	{
707	spin_lock_irq(lock: &uncore->lock);
708	if (!uncore->user_forcewake_count++) {
709	intel_uncore_forcewake_get__locked(uncore, domains: FORCEWAKE_ALL);
710	mmio_debug_suspend(uncore);
711	}
712	spin_unlock_irq(lock: &uncore->lock);
713	}
714
715	/**
716	* intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
717	* @uncore: the intel_uncore structure
718	*
719	* This function complements intel_uncore_forcewake_user_get() and releases
720	* the GT powerwell taken on behalf of the userspace bypass.
721	*/
722	void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
723	{
724	spin_lock_irq(lock: &uncore->lock);
725	if (!--uncore->user_forcewake_count) {
726	mmio_debug_resume(uncore);
727	intel_uncore_forcewake_put__locked(uncore, domains: FORCEWAKE_ALL);
728	}
729	spin_unlock_irq(lock: &uncore->lock);
730	}
731
732	/**
733	* intel_uncore_forcewake_get__locked - grab forcewake domain references
734	* @uncore: the intel_uncore structure
735	* @fw_domains: forcewake domains to get reference on
736	*
737	* See intel_uncore_forcewake_get(). This variant places the onus
738	* on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
739	*/
740	void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
741	enum forcewake_domains fw_domains)
742	{
743	lockdep_assert_held(&uncore->lock);
744
745	if (!uncore->fw_get_funcs)
746	return;
747
748	__intel_uncore_forcewake_get(uncore, fw_domains);
749	}
750
751	static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
752	enum forcewake_domains fw_domains,
753	bool delayed)
754	{
755	struct intel_uncore_forcewake_domain *domain;
756	unsigned int tmp;
757
758	fw_domains &= uncore->fw_domains;
759
760	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
761	GEM_BUG_ON(!domain->wake_count);
762
763	if (--domain->wake_count) {
764	domain->active = true;
765	continue;
766	}
767
768	if (delayed &&
769	!(domain->uncore->fw_domains_timer & domain->mask))
770	fw_domain_arm_timer(d: domain);
771	else
772	fw_domains_put(uncore, fw_domains: domain->mask);
773	}
774	}
775
776	/**
777	* intel_uncore_forcewake_put - release a forcewake domain reference
778	* @uncore: the intel_uncore structure
779	* @fw_domains: forcewake domains to put references
780	*
781	* This function drops the device-level forcewakes for specified
782	* domains obtained by intel_uncore_forcewake_get().
783	*/
784	void intel_uncore_forcewake_put(struct intel_uncore *uncore,
785	enum forcewake_domains fw_domains)
786	{
787	unsigned long irqflags;
788
789	if (!uncore->fw_get_funcs)
790	return;
791
792	spin_lock_irqsave(&uncore->lock, irqflags);
793	__intel_uncore_forcewake_put(uncore, fw_domains, delayed: false);
794	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
795	}
796
797	void intel_uncore_forcewake_put_delayed(struct intel_uncore *uncore,
798	enum forcewake_domains fw_domains)
799	{
800	unsigned long irqflags;
801
802	if (!uncore->fw_get_funcs)
803	return;
804
805	spin_lock_irqsave(&uncore->lock, irqflags);
806	__intel_uncore_forcewake_put(uncore, fw_domains, delayed: true);
807	spin_unlock_irqrestore(lock: &uncore->lock, flags: irqflags);
808	}
809
810	/**
811	* intel_uncore_forcewake_flush - flush the delayed release
812	* @uncore: the intel_uncore structure
813	* @fw_domains: forcewake domains to flush
814	*/
815	void intel_uncore_forcewake_flush(struct intel_uncore *uncore,
816	enum forcewake_domains fw_domains)
817	{
818	struct intel_uncore_forcewake_domain *domain;
819	unsigned int tmp;
820
821	if (!uncore->fw_get_funcs)
822	return;
823
824	fw_domains &= uncore->fw_domains;
825	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
826	WRITE_ONCE(domain->active, false);
827	if (hrtimer_cancel(timer: &domain->timer))
828	intel_uncore_fw_release_timer(timer: &domain->timer);
829	}
830	}
831
832	/**
833	* intel_uncore_forcewake_put__locked - release forcewake domain references
834	* @uncore: the intel_uncore structure
835	* @fw_domains: forcewake domains to put references
836	*
837	* See intel_uncore_forcewake_put(). This variant places the onus
838	* on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
839	*/
840	void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
841	enum forcewake_domains fw_domains)
842	{
843	lockdep_assert_held(&uncore->lock);
844
845	if (!uncore->fw_get_funcs)
846	return;
847
848	__intel_uncore_forcewake_put(uncore, fw_domains, delayed: false);
849	}
850
851	void assert_forcewakes_inactive(struct intel_uncore *uncore)
852	{
853	if (!uncore->fw_get_funcs)
854	return;
855
856	drm_WARN(&uncore->i915->drm, uncore->fw_domains_active,
857	"Expected all fw_domains to be inactive, but %08x are still on\n",
858	uncore->fw_domains_active);
859	}
860
861	void assert_forcewakes_active(struct intel_uncore *uncore,
862	enum forcewake_domains fw_domains)
863	{
864	struct intel_uncore_forcewake_domain *domain;
865	unsigned int tmp;
866
867	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM))
868	return;
869
870	if (!uncore->fw_get_funcs)
871	return;
872
873	spin_lock_irq(lock: &uncore->lock);
874
875	assert_rpm_wakelock_held(rpm: uncore->rpm);
876
877	fw_domains &= uncore->fw_domains;
878	drm_WARN(&uncore->i915->drm, fw_domains & ~uncore->fw_domains_active,
879	"Expected %08x fw_domains to be active, but %08x are off\n",
880	fw_domains, fw_domains & ~uncore->fw_domains_active);
881
882	/*
883	* Check that the caller has an explicit wakeref and we don't mistake
884	* it for the auto wakeref.
885	*/
886	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
887	unsigned int actual = READ_ONCE(domain->wake_count);
888	unsigned int expect = 1;
889
890	if (uncore->fw_domains_timer & domain->mask)
891	expect++; /* pending automatic release */
892
893	if (drm_WARN(&uncore->i915->drm, actual < expect,
894	"Expected domain %d to be held awake by caller, count=%d\n",
895	domain->id, actual))
896	break;
897	}
898
899	spin_unlock_irq(lock: &uncore->lock);
900	}
901
902	/*
903	* We give fast paths for the really cool registers. The second range includes
904	* media domains (and the GSC starting from Xe_LPM+)
905	*/
906	#define NEEDS_FORCE_WAKE(reg) ({ \
907	u32 __reg = (reg); \
908	__reg < 0x40000 || __reg >= 0x116000; \
909	})
910
911	static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
912	{
913	if (offset < entry->start)
914	return -1;
915	else if (offset > entry->end)
916	return 1;
917	else
918	return 0;
919	}
920
921	/* Copied and "macroized" from lib/bsearch.c */
922	#define BSEARCH(key, base, num, cmp) ({ \
923	unsigned int start__ = 0, end__ = (num); \
924	typeof(base) result__ = NULL; \
925	while (start__ < end__) { \
926	unsigned int mid__ = start__ + (end__ - start__) / 2; \
927	int ret__ = (cmp)((key), (base) + mid__); \
928	if (ret__ < 0) { \
929	end__ = mid__; \
930	} else if (ret__ > 0) { \
931	start__ = mid__ + 1; \
932	} else { \
933	result__ = (base) + mid__; \
934	break; \
935	} \
936	} \
937	result__; \
938	})
939
940	static enum forcewake_domains
941	find_fw_domain(struct intel_uncore *uncore, u32 offset)
942	{
943	const struct intel_forcewake_range *entry;
944
945	if (IS_GSI_REG(offset))
946	offset += uncore->gsi_offset;
947
948	entry = BSEARCH(offset,
949	uncore->fw_domains_table,
950	uncore->fw_domains_table_entries,
951	fw_range_cmp);
952
953	if (!entry)
954	return 0;
955
956	/*
957	* The list of FW domains depends on the SKU in gen11+ so we
958	* can't determine it statically. We use FORCEWAKE_ALL and
959	* translate it here to the list of available domains.
960	*/
961	if (entry->domains == FORCEWAKE_ALL)
962	return uncore->fw_domains;
963
964	drm_WARN(&uncore->i915->drm, entry->domains & ~uncore->fw_domains,
965	"Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
966	entry->domains & ~uncore->fw_domains, offset);
967
968	return entry->domains;
969	}
970
971	/*
972	* Shadowed register tables describe special register ranges that i915 is
973	* allowed to write to without acquiring forcewake. If these registers' power
974	* wells are down, the hardware will save values written by i915 to a shadow
975	* copy and automatically transfer them into the real register the next time
976	* the power well is woken up. Shadowing only applies to writes; forcewake
977	* must still be acquired when reading from registers in these ranges.
978	*
979	* The documentation for shadowed registers is somewhat spotty on older
980	* platforms. However missing registers from these lists is non-fatal; it just
981	* means we'll wake up the hardware for some register accesses where we didn't
982	* really need to.
983	*
984	* The ranges listed in these tables must be sorted by offset.
985	*
986	* When adding new tables here, please also add them to
987	* intel_shadow_table_check() in selftests/intel_uncore.c so that they will be
988	* scanned for obvious mistakes or typos by the selftests.
989	*/
990
991	static const struct i915_range gen8_shadowed_regs[] = {
992	{ .start = 0x2030, .end = 0x2030 },
993	{ .start = 0xA008, .end = 0xA00C },
994	{ .start = 0x12030, .end = 0x12030 },
995	{ .start = 0x1a030, .end = 0x1a030 },
996	{ .start = 0x22030, .end = 0x22030 },
997	};
998
999	static const struct i915_range gen11_shadowed_regs[] = {
1000	{ .start = 0x2030, .end = 0x2030 },
1001	{ .start = 0x2550, .end = 0x2550 },
1002	{ .start = 0xA008, .end = 0xA00C },
1003	{ .start = 0x22030, .end = 0x22030 },
1004	{ .start = 0x22230, .end = 0x22230 },
1005	{ .start = 0x22510, .end = 0x22550 },
1006	{ .start = 0x1C0030, .end = 0x1C0030 },
1007	{ .start = 0x1C0230, .end = 0x1C0230 },
1008	{ .start = 0x1C0510, .end = 0x1C0550 },
1009	{ .start = 0x1C4030, .end = 0x1C4030 },
1010	{ .start = 0x1C4230, .end = 0x1C4230 },
1011	{ .start = 0x1C4510, .end = 0x1C4550 },
1012	{ .start = 0x1C8030, .end = 0x1C8030 },
1013	{ .start = 0x1C8230, .end = 0x1C8230 },
1014	{ .start = 0x1C8510, .end = 0x1C8550 },
1015	{ .start = 0x1D0030, .end = 0x1D0030 },
1016	{ .start = 0x1D0230, .end = 0x1D0230 },
1017	{ .start = 0x1D0510, .end = 0x1D0550 },
1018	{ .start = 0x1D4030, .end = 0x1D4030 },
1019	{ .start = 0x1D4230, .end = 0x1D4230 },
1020	{ .start = 0x1D4510, .end = 0x1D4550 },
1021	{ .start = 0x1D8030, .end = 0x1D8030 },
1022	{ .start = 0x1D8230, .end = 0x1D8230 },
1023	{ .start = 0x1D8510, .end = 0x1D8550 },
1024	};
1025
1026	static const struct i915_range gen12_shadowed_regs[] = {
1027	{ .start = 0x2030, .end = 0x2030 },
1028	{ .start = 0x2510, .end = 0x2550 },
1029	{ .start = 0xA008, .end = 0xA00C },
1030	{ .start = 0xA188, .end = 0xA188 },
1031	{ .start = 0xA278, .end = 0xA278 },
1032	{ .start = 0xA540, .end = 0xA56C },
1033	{ .start = 0xC4C8, .end = 0xC4C8 },
1034	{ .start = 0xC4D4, .end = 0xC4D4 },
1035	{ .start = 0xC600, .end = 0xC600 },
1036	{ .start = 0x22030, .end = 0x22030 },
1037	{ .start = 0x22510, .end = 0x22550 },
1038	{ .start = 0x1C0030, .end = 0x1C0030 },
1039	{ .start = 0x1C0510, .end = 0x1C0550 },
1040	{ .start = 0x1C4030, .end = 0x1C4030 },
1041	{ .start = 0x1C4510, .end = 0x1C4550 },
1042	{ .start = 0x1C8030, .end = 0x1C8030 },
1043	{ .start = 0x1C8510, .end = 0x1C8550 },
1044	{ .start = 0x1D0030, .end = 0x1D0030 },
1045	{ .start = 0x1D0510, .end = 0x1D0550 },
1046	{ .start = 0x1D4030, .end = 0x1D4030 },
1047	{ .start = 0x1D4510, .end = 0x1D4550 },
1048	{ .start = 0x1D8030, .end = 0x1D8030 },
1049	{ .start = 0x1D8510, .end = 0x1D8550 },
1050
1051	/*
1052	* The rest of these ranges are specific to Xe_HP and beyond, but
1053	* are reserved/unused ranges on earlier gen12 platforms, so they can
1054	* be safely added to the gen12 table.
1055	*/
1056	{ .start = 0x1E0030, .end = 0x1E0030 },
1057	{ .start = 0x1E0510, .end = 0x1E0550 },
1058	{ .start = 0x1E4030, .end = 0x1E4030 },
1059	{ .start = 0x1E4510, .end = 0x1E4550 },
1060	{ .start = 0x1E8030, .end = 0x1E8030 },
1061	{ .start = 0x1E8510, .end = 0x1E8550 },
1062	{ .start = 0x1F0030, .end = 0x1F0030 },
1063	{ .start = 0x1F0510, .end = 0x1F0550 },
1064	{ .start = 0x1F4030, .end = 0x1F4030 },
1065	{ .start = 0x1F4510, .end = 0x1F4550 },
1066	{ .start = 0x1F8030, .end = 0x1F8030 },
1067	{ .start = 0x1F8510, .end = 0x1F8550 },
1068	};
1069
1070	static const struct i915_range dg2_shadowed_regs[] = {
1071	{ .start = 0x2030, .end = 0x2030 },
1072	{ .start = 0x2510, .end = 0x2550 },
1073	{ .start = 0xA008, .end = 0xA00C },
1074	{ .start = 0xA188, .end = 0xA188 },
1075	{ .start = 0xA278, .end = 0xA278 },
1076	{ .start = 0xA540, .end = 0xA56C },
1077	{ .start = 0xC4C8, .end = 0xC4C8 },
1078	{ .start = 0xC4E0, .end = 0xC4E0 },
1079	{ .start = 0xC600, .end = 0xC600 },
1080	{ .start = 0xC658, .end = 0xC658 },
1081	{ .start = 0x22030, .end = 0x22030 },
1082	{ .start = 0x22510, .end = 0x22550 },
1083	{ .start = 0x1C0030, .end = 0x1C0030 },
1084	{ .start = 0x1C0510, .end = 0x1C0550 },
1085	{ .start = 0x1C4030, .end = 0x1C4030 },
1086	{ .start = 0x1C4510, .end = 0x1C4550 },
1087	{ .start = 0x1C8030, .end = 0x1C8030 },
1088	{ .start = 0x1C8510, .end = 0x1C8550 },
1089	{ .start = 0x1D0030, .end = 0x1D0030 },
1090	{ .start = 0x1D0510, .end = 0x1D0550 },
1091	{ .start = 0x1D4030, .end = 0x1D4030 },
1092	{ .start = 0x1D4510, .end = 0x1D4550 },
1093	{ .start = 0x1D8030, .end = 0x1D8030 },
1094	{ .start = 0x1D8510, .end = 0x1D8550 },
1095	{ .start = 0x1E0030, .end = 0x1E0030 },
1096	{ .start = 0x1E0510, .end = 0x1E0550 },
1097	{ .start = 0x1E4030, .end = 0x1E4030 },
1098	{ .start = 0x1E4510, .end = 0x1E4550 },
1099	{ .start = 0x1E8030, .end = 0x1E8030 },
1100	{ .start = 0x1E8510, .end = 0x1E8550 },
1101	{ .start = 0x1F0030, .end = 0x1F0030 },
1102	{ .start = 0x1F0510, .end = 0x1F0550 },
1103	{ .start = 0x1F4030, .end = 0x1F4030 },
1104	{ .start = 0x1F4510, .end = 0x1F4550 },
1105	{ .start = 0x1F8030, .end = 0x1F8030 },
1106	{ .start = 0x1F8510, .end = 0x1F8550 },
1107	};
1108
1109	static const struct i915_range pvc_shadowed_regs[] = {
1110	{ .start = 0x2030, .end = 0x2030 },
1111	{ .start = 0x2510, .end = 0x2550 },
1112	{ .start = 0xA008, .end = 0xA00C },
1113	{ .start = 0xA188, .end = 0xA188 },
1114	{ .start = 0xA278, .end = 0xA278 },
1115	{ .start = 0xA540, .end = 0xA56C },
1116	{ .start = 0xC4C8, .end = 0xC4C8 },
1117	{ .start = 0xC4E0, .end = 0xC4E0 },
1118	{ .start = 0xC600, .end = 0xC600 },
1119	{ .start = 0xC658, .end = 0xC658 },
1120	{ .start = 0x22030, .end = 0x22030 },
1121	{ .start = 0x22510, .end = 0x22550 },
1122	{ .start = 0x1C0030, .end = 0x1C0030 },
1123	{ .start = 0x1C0510, .end = 0x1C0550 },
1124	{ .start = 0x1C4030, .end = 0x1C4030 },
1125	{ .start = 0x1C4510, .end = 0x1C4550 },
1126	{ .start = 0x1C8030, .end = 0x1C8030 },
1127	{ .start = 0x1C8510, .end = 0x1C8550 },
1128	{ .start = 0x1D0030, .end = 0x1D0030 },
1129	{ .start = 0x1D0510, .end = 0x1D0550 },
1130	{ .start = 0x1D4030, .end = 0x1D4030 },
1131	{ .start = 0x1D4510, .end = 0x1D4550 },
1132	{ .start = 0x1D8030, .end = 0x1D8030 },
1133	{ .start = 0x1D8510, .end = 0x1D8550 },
1134	{ .start = 0x1E0030, .end = 0x1E0030 },
1135	{ .start = 0x1E0510, .end = 0x1E0550 },
1136	{ .start = 0x1E4030, .end = 0x1E4030 },
1137	{ .start = 0x1E4510, .end = 0x1E4550 },
1138	{ .start = 0x1E8030, .end = 0x1E8030 },
1139	{ .start = 0x1E8510, .end = 0x1E8550 },
1140	{ .start = 0x1F0030, .end = 0x1F0030 },
1141	{ .start = 0x1F0510, .end = 0x1F0550 },
1142	{ .start = 0x1F4030, .end = 0x1F4030 },
1143	{ .start = 0x1F4510, .end = 0x1F4550 },
1144	{ .start = 0x1F8030, .end = 0x1F8030 },
1145	{ .start = 0x1F8510, .end = 0x1F8550 },
1146	};
1147
1148	static const struct i915_range mtl_shadowed_regs[] = {
1149	{ .start = 0x2030, .end = 0x2030 },
1150	{ .start = 0x2510, .end = 0x2550 },
1151	{ .start = 0xA008, .end = 0xA00C },
1152	{ .start = 0xA188, .end = 0xA188 },
1153	{ .start = 0xA278, .end = 0xA278 },
1154	{ .start = 0xA540, .end = 0xA56C },
1155	{ .start = 0xC050, .end = 0xC050 },
1156	{ .start = 0xC340, .end = 0xC340 },
1157	{ .start = 0xC4C8, .end = 0xC4C8 },
1158	{ .start = 0xC4E0, .end = 0xC4E0 },
1159	{ .start = 0xC600, .end = 0xC600 },
1160	{ .start = 0xC658, .end = 0xC658 },
1161	{ .start = 0xCFD4, .end = 0xCFDC },
1162	{ .start = 0x22030, .end = 0x22030 },
1163	{ .start = 0x22510, .end = 0x22550 },
1164	};
1165
1166	static const struct i915_range xelpmp_shadowed_regs[] = {
1167	{ .start = 0x1C0030, .end = 0x1C0030 },
1168	{ .start = 0x1C0510, .end = 0x1C0550 },
1169	{ .start = 0x1C8030, .end = 0x1C8030 },
1170	{ .start = 0x1C8510, .end = 0x1C8550 },
1171	{ .start = 0x1D0030, .end = 0x1D0030 },
1172	{ .start = 0x1D0510, .end = 0x1D0550 },
1173	{ .start = 0x38A008, .end = 0x38A00C },
1174	{ .start = 0x38A188, .end = 0x38A188 },
1175	{ .start = 0x38A278, .end = 0x38A278 },
1176	{ .start = 0x38A540, .end = 0x38A56C },
1177	{ .start = 0x38A618, .end = 0x38A618 },
1178	{ .start = 0x38C050, .end = 0x38C050 },
1179	{ .start = 0x38C340, .end = 0x38C340 },
1180	{ .start = 0x38C4C8, .end = 0x38C4C8 },
1181	{ .start = 0x38C4E0, .end = 0x38C4E4 },
1182	{ .start = 0x38C600, .end = 0x38C600 },
1183	{ .start = 0x38C658, .end = 0x38C658 },
1184	{ .start = 0x38CFD4, .end = 0x38CFDC },
1185	};
1186
1187	static int mmio_range_cmp(u32 key, const struct i915_range *range)
1188	{
1189	if (key < range->start)
1190	return -1;
1191	else if (key > range->end)
1192	return 1;
1193	else
1194	return 0;
1195	}
1196
1197	static bool is_shadowed(struct intel_uncore *uncore, u32 offset)
1198	{
1199	if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table))
1200	return false;
1201
1202	if (IS_GSI_REG(offset))
1203	offset += uncore->gsi_offset;
1204
1205	return BSEARCH(offset,
1206	uncore->shadowed_reg_table,
1207	uncore->shadowed_reg_table_entries,
1208	mmio_range_cmp);
1209	}
1210
1211	static enum forcewake_domains
1212	gen6_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
1213	{
1214	return FORCEWAKE_RENDER;
1215	}
1216
1217	#define __fwtable_reg_read_fw_domains(uncore, offset) \
1218	({ \
1219	enum forcewake_domains __fwd = 0; \
1220	if (NEEDS_FORCE_WAKE((offset))) \
1221	__fwd = find_fw_domain(uncore, offset); \
1222	__fwd; \
1223	})
1224
1225	#define __fwtable_reg_write_fw_domains(uncore, offset) \
1226	({ \
1227	enum forcewake_domains __fwd = 0; \
1228	const u32 __offset = (offset); \
1229	if (NEEDS_FORCE_WAKE((__offset)) && !is_shadowed(uncore, __offset)) \
1230	__fwd = find_fw_domain(uncore, __offset); \
1231	__fwd; \
1232	})
1233
1234	#define GEN_FW_RANGE(s, e, d) \
1235	{ .start = (s), .end = (e), .domains = (d) }
1236
1237	/*
1238	* All platforms' forcewake tables below must be sorted by offset ranges.
1239	* Furthermore, new forcewake tables added should be "watertight" and have
1240	* no gaps between ranges.
1241	*
1242	* When there are multiple consecutive ranges listed in the bspec with
1243	* the same forcewake domain, it is customary to combine them into a single
1244	* row in the tables below to keep the tables small and lookups fast.
1245	* Likewise, reserved/unused ranges may be combined with the preceding and/or
1246	* following ranges since the driver will never be making MMIO accesses in
1247	* those ranges.
1248	*
1249	* For example, if the bspec were to list:
1250	*
1251	* ...
1252	* 0x1000 - 0x1fff: GT
1253	* 0x2000 - 0x2cff: GT
1254	* 0x2d00 - 0x2fff: unused/reserved
1255	* 0x3000 - 0xffff: GT
1256	* ...
1257	*
1258	* these could all be represented by a single line in the code:
1259	*
1260	* GEN_FW_RANGE(0x1000, 0xffff, FORCEWAKE_GT)
1261	*
1262	* When adding new forcewake tables here, please also add them to
1263	* intel_uncore_mock_selftests in selftests/intel_uncore.c so that they will be
1264	* scanned for obvious mistakes or typos by the selftests.
1265	*/
1266
1267	static const struct intel_forcewake_range __gen6_fw_ranges[] = {
1268	GEN_FW_RANGE(0x0, 0x3ffff, FORCEWAKE_RENDER),
1269	};
1270
1271	static const struct intel_forcewake_range __vlv_fw_ranges[] = {
1272	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1273	GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
1274	GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
1275	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1276	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
1277	GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
1278	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1279	};
1280
1281	static const struct intel_forcewake_range __chv_fw_ranges[] = {
1282	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
1283	GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1284	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1285	GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1286	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1287	GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1288	GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
1289	GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1290	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1291	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1292	GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
1293	GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1294	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1295	GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
1296	GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
1297	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
1298	};
1299
1300	static const struct intel_forcewake_range __gen9_fw_ranges[] = {
1301	GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_GT),
1302	GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1303	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1304	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1305	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1306	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1307	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1308	GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_GT),
1309	GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
1310	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1311	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1312	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1313	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1314	GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
1315	GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_GT),
1316	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1317	GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_GT),
1318	GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1319	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1320	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1321	GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_GT),
1322	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1323	GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_GT),
1324	GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1325	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT),
1326	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1327	GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_GT),
1328	GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
1329	GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_GT),
1330	GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1331	GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_GT),
1332	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1333	};
1334
1335	static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1336	GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */
1337	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1338	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1339	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1340	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT),
1341	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1342	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1343	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1344	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_GT),
1345	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1346	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_GT),
1347	GEN_FW_RANGE(0x8800, 0x8bff, 0),
1348	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1349	GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_GT),
1350	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1351	GEN_FW_RANGE(0x9560, 0x95ff, 0),
1352	GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_GT),
1353	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1354	GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_GT),
1355	GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER),
1356	GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_GT),
1357	GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER),
1358	GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_GT),
1359	GEN_FW_RANGE(0x24000, 0x2407f, 0),
1360	GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_GT),
1361	GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER),
1362	GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_GT),
1363	GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER),
1364	GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_GT),
1365	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1366	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1367	GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1368	GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0),
1369	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1370	GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0)
1371	};
1372
1373	static const struct intel_forcewake_range __gen12_fw_ranges[] = {
1374	GEN_FW_RANGE(0x0, 0x1fff, 0), /*
1375	0x0 - 0xaff: reserved
1376	0xb00 - 0x1fff: always on */
1377	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1378	GEN_FW_RANGE(0x2700, 0x27ff, FORCEWAKE_GT),
1379	GEN_FW_RANGE(0x2800, 0x2aff, FORCEWAKE_RENDER),
1380	GEN_FW_RANGE(0x2b00, 0x2fff, FORCEWAKE_GT),
1381	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1382	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1383	0x4000 - 0x48ff: gt
1384	0x4900 - 0x51ff: reserved */
1385	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1386	0x5200 - 0x53ff: render
1387	0x5400 - 0x54ff: reserved
1388	0x5500 - 0x7fff: render */
1389	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1390	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1391	GEN_FW_RANGE(0x8160, 0x81ff, 0), /*
1392	0x8160 - 0x817f: reserved
1393	0x8180 - 0x81ff: always on */
1394	GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT),
1395	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1396	GEN_FW_RANGE(0x8500, 0x94cf, FORCEWAKE_GT), /*
1397	0x8500 - 0x87ff: gt
1398	0x8800 - 0x8fff: reserved
1399	0x9000 - 0x947f: gt
1400	0x9480 - 0x94cf: reserved */
1401	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1402	GEN_FW_RANGE(0x9560, 0x97ff, 0), /*
1403	0x9560 - 0x95ff: always on
1404	0x9600 - 0x97ff: reserved */
1405	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_GT),
1406	GEN_FW_RANGE(0xb000, 0xb3ff, FORCEWAKE_RENDER),
1407	GEN_FW_RANGE(0xb400, 0xcfff, FORCEWAKE_GT), /*
1408	0xb400 - 0xbf7f: gt
1409	0xb480 - 0xbfff: reserved
1410	0xc000 - 0xcfff: gt */
1411	GEN_FW_RANGE(0xd000, 0xd7ff, 0),
1412	GEN_FW_RANGE(0xd800, 0xd8ff, FORCEWAKE_RENDER),
1413	GEN_FW_RANGE(0xd900, 0xdbff, FORCEWAKE_GT),
1414	GEN_FW_RANGE(0xdc00, 0xefff, FORCEWAKE_RENDER), /*
1415	0xdc00 - 0xddff: render
1416	0xde00 - 0xde7f: reserved
1417	0xde80 - 0xe8ff: render
1418	0xe900 - 0xefff: reserved */
1419	GEN_FW_RANGE(0xf000, 0x147ff, FORCEWAKE_GT), /*
1420	0xf000 - 0xffff: gt
1421	0x10000 - 0x147ff: reserved */
1422	GEN_FW_RANGE(0x14800, 0x1ffff, FORCEWAKE_RENDER), /*
1423	0x14800 - 0x14fff: render
1424	0x15000 - 0x16dff: reserved
1425	0x16e00 - 0x1bfff: render
1426	0x1c000 - 0x1ffff: reserved */
1427	GEN_FW_RANGE(0x20000, 0x20fff, FORCEWAKE_MEDIA_VDBOX0),
1428	GEN_FW_RANGE(0x21000, 0x21fff, FORCEWAKE_MEDIA_VDBOX2),
1429	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1430	GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1431	0x24000 - 0x2407f: always on
1432	0x24080 - 0x2417f: reserved */
1433	GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /*
1434	0x24180 - 0x241ff: gt
1435	0x24200 - 0x249ff: reserved */
1436	GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /*
1437	0x24a00 - 0x24a7f: render
1438	0x24a80 - 0x251ff: reserved */
1439	GEN_FW_RANGE(0x25200, 0x255ff, FORCEWAKE_GT), /*
1440	0x25200 - 0x252ff: gt
1441	0x25300 - 0x255ff: reserved */
1442	GEN_FW_RANGE(0x25600, 0x2567f, FORCEWAKE_MEDIA_VDBOX0),
1443	GEN_FW_RANGE(0x25680, 0x259ff, FORCEWAKE_MEDIA_VDBOX2), /*
1444	0x25680 - 0x256ff: VD2
1445	0x25700 - 0x259ff: reserved */
1446	GEN_FW_RANGE(0x25a00, 0x25a7f, FORCEWAKE_MEDIA_VDBOX0),
1447	GEN_FW_RANGE(0x25a80, 0x2ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1448	0x25a80 - 0x25aff: VD2
1449	0x25b00 - 0x2ffff: reserved */
1450	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1451	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1452	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1453	0x1c0000 - 0x1c2bff: VD0
1454	0x1c2c00 - 0x1c2cff: reserved
1455	0x1c2d00 - 0x1c2dff: VD0
1456	0x1c2e00 - 0x1c3eff: reserved
1457	0x1c3f00 - 0x1c3fff: VD0 */
1458	GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0),
1459	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1460	0x1c8000 - 0x1ca0ff: VE0
1461	0x1ca100 - 0x1cbeff: reserved
1462	0x1cbf00 - 0x1cbfff: VE0 */
1463	GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1464	0x1cc000 - 0x1ccfff: VD0
1465	0x1cd000 - 0x1cffff: reserved */
1466	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /*
1467	0x1d0000 - 0x1d2bff: VD2
1468	0x1d2c00 - 0x1d2cff: reserved
1469	0x1d2d00 - 0x1d2dff: VD2
1470	0x1d2e00 - 0x1d3eff: reserved
1471	0x1d3f00 - 0x1d3fff: VD2 */
1472	};
1473
1474	/*
1475	* Graphics IP version 12.55 brings a slight change to the 0xd800 range,
1476	* switching it from the GT domain to the render domain.
1477	*/
1478	#define XEHP_FWRANGES(FW_RANGE_D800) \
1479	GEN_FW_RANGE(0x0, 0x1fff, 0), /* \
1480	0x0 - 0xaff: reserved \
1481	0xb00 - 0x1fff: always on */ \
1482	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), \
1483	GEN_FW_RANGE(0x2700, 0x4aff, FORCEWAKE_GT), \
1484	GEN_FW_RANGE(0x4b00, 0x51ff, 0), /* \
1485	0x4b00 - 0x4fff: reserved \
1486	0x5000 - 0x51ff: always on */ \
1487	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), \
1488	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT), \
1489	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), \
1490	GEN_FW_RANGE(0x8160, 0x81ff, 0), /* \
1491	0x8160 - 0x817f: reserved \
1492	0x8180 - 0x81ff: always on */ \
1493	GEN_FW_RANGE(0x8200, 0x82ff, FORCEWAKE_GT), \
1494	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), \
1495	GEN_FW_RANGE(0x8500, 0x8cff, FORCEWAKE_GT), /* \
1496	0x8500 - 0x87ff: gt \
1497	0x8800 - 0x8c7f: reserved \
1498	0x8c80 - 0x8cff: gt (DG2 only) */ \
1499	GEN_FW_RANGE(0x8d00, 0x8fff, FORCEWAKE_RENDER), /* \
1500	0x8d00 - 0x8dff: render (DG2 only) \
1501	0x8e00 - 0x8fff: reserved */ \
1502	GEN_FW_RANGE(0x9000, 0x94cf, FORCEWAKE_GT), /* \
1503	0x9000 - 0x947f: gt \
1504	0x9480 - 0x94cf: reserved */ \
1505	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), \
1506	GEN_FW_RANGE(0x9560, 0x967f, 0), /* \
1507	0x9560 - 0x95ff: always on \
1508	0x9600 - 0x967f: reserved */ \
1509	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /* \
1510	0x9680 - 0x96ff: render (DG2 only) \
1511	0x9700 - 0x97ff: reserved */ \
1512	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /* \
1513	0x9800 - 0xb4ff: gt \
1514	0xb500 - 0xbfff: reserved \
1515	0xc000 - 0xcfff: gt */ \
1516	GEN_FW_RANGE(0xd000, 0xd7ff, 0), \
1517	GEN_FW_RANGE(0xd800, 0xd87f, FW_RANGE_D800), \
1518	GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT), \
1519	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER), \
1520	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /* \
1521	0xdd00 - 0xddff: gt \
1522	0xde00 - 0xde7f: reserved */ \
1523	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /* \
1524	0xde80 - 0xdfff: render \
1525	0xe000 - 0xe0ff: reserved \
1526	0xe100 - 0xe8ff: render */ \
1527	GEN_FW_RANGE(0xe900, 0xffff, FORCEWAKE_GT), /* \
1528	0xe900 - 0xe9ff: gt \
1529	0xea00 - 0xefff: reserved \
1530	0xf000 - 0xffff: gt */ \
1531	GEN_FW_RANGE(0x10000, 0x12fff, 0), /* \
1532	0x10000 - 0x11fff: reserved \
1533	0x12000 - 0x127ff: always on \
1534	0x12800 - 0x12fff: reserved */ \
1535	GEN_FW_RANGE(0x13000, 0x131ff, FORCEWAKE_MEDIA_VDBOX0), /* DG2 only */ \
1536	GEN_FW_RANGE(0x13200, 0x13fff, FORCEWAKE_MEDIA_VDBOX2), /* \
1537	0x13200 - 0x133ff: VD2 (DG2 only) \
1538	0x13400 - 0x13fff: reserved */ \
1539	GEN_FW_RANGE(0x14000, 0x141ff, FORCEWAKE_MEDIA_VDBOX0), /* XEHPSDV only */ \
1540	GEN_FW_RANGE(0x14200, 0x143ff, FORCEWAKE_MEDIA_VDBOX2), /* XEHPSDV only */ \
1541	GEN_FW_RANGE(0x14400, 0x145ff, FORCEWAKE_MEDIA_VDBOX4), /* XEHPSDV only */ \
1542	GEN_FW_RANGE(0x14600, 0x147ff, FORCEWAKE_MEDIA_VDBOX6), /* XEHPSDV only */ \
1543	GEN_FW_RANGE(0x14800, 0x14fff, FORCEWAKE_RENDER), \
1544	GEN_FW_RANGE(0x15000, 0x16dff, FORCEWAKE_GT), /* \
1545	0x15000 - 0x15fff: gt (DG2 only) \
1546	0x16000 - 0x16dff: reserved */ \
1547	GEN_FW_RANGE(0x16e00, 0x1ffff, FORCEWAKE_RENDER), \
1548	GEN_FW_RANGE(0x20000, 0x21fff, FORCEWAKE_MEDIA_VDBOX0), /* \
1549	0x20000 - 0x20fff: VD0 (XEHPSDV only) \
1550	0x21000 - 0x21fff: reserved */ \
1551	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT), \
1552	GEN_FW_RANGE(0x24000, 0x2417f, 0), /* \
1553	0x24000 - 0x2407f: always on \
1554	0x24080 - 0x2417f: reserved */ \
1555	GEN_FW_RANGE(0x24180, 0x249ff, FORCEWAKE_GT), /* \
1556	0x24180 - 0x241ff: gt \
1557	0x24200 - 0x249ff: reserved */ \
1558	GEN_FW_RANGE(0x24a00, 0x251ff, FORCEWAKE_RENDER), /* \
1559	0x24a00 - 0x24a7f: render \
1560	0x24a80 - 0x251ff: reserved */ \
1561	GEN_FW_RANGE(0x25200, 0x25fff, FORCEWAKE_GT), /* \
1562	0x25200 - 0x252ff: gt \
1563	0x25300 - 0x25fff: reserved */ \
1564	GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /* \
1565	0x26000 - 0x27fff: render \
1566	0x28000 - 0x29fff: reserved \
1567	0x2a000 - 0x2ffff: undocumented */ \
1568	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT), \
1569	GEN_FW_RANGE(0x40000, 0x1bffff, 0), \
1570	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /* \
1571	0x1c0000 - 0x1c2bff: VD0 \
1572	0x1c2c00 - 0x1c2cff: reserved \
1573	0x1c2d00 - 0x1c2dff: VD0 \
1574	0x1c2e00 - 0x1c3eff: VD0 (DG2 only) \
1575	0x1c3f00 - 0x1c3fff: VD0 */ \
1576	GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), /* \
1577	0x1c4000 - 0x1c6bff: VD1 \
1578	0x1c6c00 - 0x1c6cff: reserved \
1579	0x1c6d00 - 0x1c6dff: VD1 \
1580	0x1c6e00 - 0x1c7fff: reserved */ \
1581	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /* \
1582	0x1c8000 - 0x1ca0ff: VE0 \
1583	0x1ca100 - 0x1cbfff: reserved */ \
1584	GEN_FW_RANGE(0x1cc000, 0x1ccfff, FORCEWAKE_MEDIA_VDBOX0), \
1585	GEN_FW_RANGE(0x1cd000, 0x1cdfff, FORCEWAKE_MEDIA_VDBOX2), \
1586	GEN_FW_RANGE(0x1ce000, 0x1cefff, FORCEWAKE_MEDIA_VDBOX4), \
1587	GEN_FW_RANGE(0x1cf000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX6), \
1588	GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), /* \
1589	0x1d0000 - 0x1d2bff: VD2 \
1590	0x1d2c00 - 0x1d2cff: reserved \
1591	0x1d2d00 - 0x1d2dff: VD2 \
1592	0x1d2e00 - 0x1d3dff: VD2 (DG2 only) \
1593	0x1d3e00 - 0x1d3eff: reserved \
1594	0x1d3f00 - 0x1d3fff: VD2 */ \
1595	GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), /* \
1596	0x1d4000 - 0x1d6bff: VD3 \
1597	0x1d6c00 - 0x1d6cff: reserved \
1598	0x1d6d00 - 0x1d6dff: VD3 \
1599	0x1d6e00 - 0x1d7fff: reserved */ \
1600	GEN_FW_RANGE(0x1d8000, 0x1dffff, FORCEWAKE_MEDIA_VEBOX1), /* \
1601	0x1d8000 - 0x1da0ff: VE1 \
1602	0x1da100 - 0x1dffff: reserved */ \
1603	GEN_FW_RANGE(0x1e0000, 0x1e3fff, FORCEWAKE_MEDIA_VDBOX4), /* \
1604	0x1e0000 - 0x1e2bff: VD4 \
1605	0x1e2c00 - 0x1e2cff: reserved \
1606	0x1e2d00 - 0x1e2dff: VD4 \
1607	0x1e2e00 - 0x1e3eff: reserved \
1608	0x1e3f00 - 0x1e3fff: VD4 */ \
1609	GEN_FW_RANGE(0x1e4000, 0x1e7fff, FORCEWAKE_MEDIA_VDBOX5), /* \
1610	0x1e4000 - 0x1e6bff: VD5 \
1611	0x1e6c00 - 0x1e6cff: reserved \
1612	0x1e6d00 - 0x1e6dff: VD5 \
1613	0x1e6e00 - 0x1e7fff: reserved */ \
1614	GEN_FW_RANGE(0x1e8000, 0x1effff, FORCEWAKE_MEDIA_VEBOX2), /* \
1615	0x1e8000 - 0x1ea0ff: VE2 \
1616	0x1ea100 - 0x1effff: reserved */ \
1617	GEN_FW_RANGE(0x1f0000, 0x1f3fff, FORCEWAKE_MEDIA_VDBOX6), /* \
1618	0x1f0000 - 0x1f2bff: VD6 \
1619	0x1f2c00 - 0x1f2cff: reserved \
1620	0x1f2d00 - 0x1f2dff: VD6 \
1621	0x1f2e00 - 0x1f3eff: reserved \
1622	0x1f3f00 - 0x1f3fff: VD6 */ \
1623	GEN_FW_RANGE(0x1f4000, 0x1f7fff, FORCEWAKE_MEDIA_VDBOX7), /* \
1624	0x1f4000 - 0x1f6bff: VD7 \
1625	0x1f6c00 - 0x1f6cff: reserved \
1626	0x1f6d00 - 0x1f6dff: VD7 \
1627	0x1f6e00 - 0x1f7fff: reserved */ \
1628	GEN_FW_RANGE(0x1f8000, 0x1fa0ff, FORCEWAKE_MEDIA_VEBOX3),
1629
1630	static const struct intel_forcewake_range __xehp_fw_ranges[] = {
1631	XEHP_FWRANGES(FORCEWAKE_GT)
1632	};
1633
1634	static const struct intel_forcewake_range __dg2_fw_ranges[] = {
1635	XEHP_FWRANGES(FORCEWAKE_RENDER)
1636	};
1637
1638	static const struct intel_forcewake_range __pvc_fw_ranges[] = {
1639	GEN_FW_RANGE(0x0, 0xaff, 0),
1640	GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1641	GEN_FW_RANGE(0xc00, 0xfff, 0),
1642	GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1643	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1644	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1645	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1646	GEN_FW_RANGE(0x4000, 0x813f, FORCEWAKE_GT), /*
1647	0x4000 - 0x4aff: gt
1648	0x4b00 - 0x4fff: reserved
1649	0x5000 - 0x51ff: gt
1650	0x5200 - 0x52ff: reserved
1651	0x5300 - 0x53ff: gt
1652	0x5400 - 0x7fff: reserved
1653	0x8000 - 0x813f: gt */
1654	GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER),
1655	GEN_FW_RANGE(0x8180, 0x81ff, 0),
1656	GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1657	0x8200 - 0x82ff: gt
1658	0x8300 - 0x84ff: reserved
1659	0x8500 - 0x887f: gt
1660	0x8880 - 0x8a7f: reserved
1661	0x8a80 - 0x8aff: gt
1662	0x8b00 - 0x8fff: reserved
1663	0x9000 - 0x947f: gt
1664	0x9480 - 0x94cf: reserved */
1665	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1666	GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1667	0x9560 - 0x95ff: always on
1668	0x9600 - 0x967f: reserved */
1669	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1670	0x9680 - 0x96ff: render
1671	0x9700 - 0x97ff: reserved */
1672	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1673	0x9800 - 0xb4ff: gt
1674	0xb500 - 0xbfff: reserved
1675	0xc000 - 0xcfff: gt */
1676	GEN_FW_RANGE(0xd000, 0xd3ff, 0),
1677	GEN_FW_RANGE(0xd400, 0xdbff, FORCEWAKE_GT),
1678	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1679	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1680	0xdd00 - 0xddff: gt
1681	0xde00 - 0xde7f: reserved */
1682	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1683	0xde80 - 0xdeff: render
1684	0xdf00 - 0xe1ff: reserved
1685	0xe200 - 0xe7ff: render
1686	0xe800 - 0xe8ff: reserved */
1687	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_GT), /*
1688	0xe900 - 0xe9ff: gt
1689	0xea00 - 0xebff: reserved
1690	0xec00 - 0xffff: gt
1691	0x10000 - 0x11fff: reserved */
1692	GEN_FW_RANGE(0x12000, 0x12fff, 0), /*
1693	0x12000 - 0x127ff: always on
1694	0x12800 - 0x12fff: reserved */
1695	GEN_FW_RANGE(0x13000, 0x19fff, FORCEWAKE_GT), /*
1696	0x13000 - 0x135ff: gt
1697	0x13600 - 0x147ff: reserved
1698	0x14800 - 0x153ff: gt
1699	0x15400 - 0x19fff: reserved */
1700	GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1701	0x1a000 - 0x1ffff: render
1702	0x20000 - 0x21fff: reserved */
1703	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1704	GEN_FW_RANGE(0x24000, 0x2417f, 0), /*
1705	24000 - 0x2407f: always on
1706	24080 - 0x2417f: reserved */
1707	GEN_FW_RANGE(0x24180, 0x25fff, FORCEWAKE_GT), /*
1708	0x24180 - 0x241ff: gt
1709	0x24200 - 0x251ff: reserved
1710	0x25200 - 0x252ff: gt
1711	0x25300 - 0x25fff: reserved */
1712	GEN_FW_RANGE(0x26000, 0x2ffff, FORCEWAKE_RENDER), /*
1713	0x26000 - 0x27fff: render
1714	0x28000 - 0x2ffff: reserved */
1715	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT),
1716	GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1717	GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), /*
1718	0x1c0000 - 0x1c2bff: VD0
1719	0x1c2c00 - 0x1c2cff: reserved
1720	0x1c2d00 - 0x1c2dff: VD0
1721	0x1c2e00 - 0x1c3eff: reserved
1722	0x1c3f00 - 0x1c3fff: VD0 */
1723	GEN_FW_RANGE(0x1c4000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX1), /*
1724	0x1c4000 - 0x1c6aff: VD1
1725	0x1c6b00 - 0x1c7eff: reserved
1726	0x1c7f00 - 0x1c7fff: VD1
1727	0x1c8000 - 0x1cffff: reserved */
1728	GEN_FW_RANGE(0x1d0000, 0x23ffff, FORCEWAKE_MEDIA_VDBOX2), /*
1729	0x1d0000 - 0x1d2aff: VD2
1730	0x1d2b00 - 0x1d3eff: reserved
1731	0x1d3f00 - 0x1d3fff: VD2
1732	0x1d4000 - 0x23ffff: reserved */
1733	GEN_FW_RANGE(0x240000, 0x3dffff, 0),
1734	GEN_FW_RANGE(0x3e0000, 0x3effff, FORCEWAKE_GT),
1735	};
1736
1737	static const struct intel_forcewake_range __mtl_fw_ranges[] = {
1738	GEN_FW_RANGE(0x0, 0xaff, 0),
1739	GEN_FW_RANGE(0xb00, 0xbff, FORCEWAKE_GT),
1740	GEN_FW_RANGE(0xc00, 0xfff, 0),
1741	GEN_FW_RANGE(0x1000, 0x1fff, FORCEWAKE_GT),
1742	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1743	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_GT),
1744	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1745	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_GT), /*
1746	0x4000 - 0x48ff: render
1747	0x4900 - 0x51ff: reserved */
1748	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER), /*
1749	0x5200 - 0x53ff: render
1750	0x5400 - 0x54ff: reserved
1751	0x5500 - 0x7fff: render */
1752	GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_GT),
1753	GEN_FW_RANGE(0x8140, 0x817f, FORCEWAKE_RENDER), /*
1754	0x8140 - 0x815f: render
1755	0x8160 - 0x817f: reserved */
1756	GEN_FW_RANGE(0x8180, 0x81ff, 0),
1757	GEN_FW_RANGE(0x8200, 0x94cf, FORCEWAKE_GT), /*
1758	0x8200 - 0x87ff: gt
1759	0x8800 - 0x8dff: reserved
1760	0x8e00 - 0x8f7f: gt
1761	0x8f80 - 0x8fff: reserved
1762	0x9000 - 0x947f: gt
1763	0x9480 - 0x94cf: reserved */
1764	GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER),
1765	GEN_FW_RANGE(0x9560, 0x967f, 0), /*
1766	0x9560 - 0x95ff: always on
1767	0x9600 - 0x967f: reserved */
1768	GEN_FW_RANGE(0x9680, 0x97ff, FORCEWAKE_RENDER), /*
1769	0x9680 - 0x96ff: render
1770	0x9700 - 0x97ff: reserved */
1771	GEN_FW_RANGE(0x9800, 0xcfff, FORCEWAKE_GT), /*
1772	0x9800 - 0xb4ff: gt
1773	0xb500 - 0xbfff: reserved
1774	0xc000 - 0xcfff: gt */
1775	GEN_FW_RANGE(0xd000, 0xd7ff, 0), /*
1776	0xd000 - 0xd3ff: always on
1777	0xd400 - 0xd7ff: reserved */
1778	GEN_FW_RANGE(0xd800, 0xd87f, FORCEWAKE_RENDER),
1779	GEN_FW_RANGE(0xd880, 0xdbff, FORCEWAKE_GT),
1780	GEN_FW_RANGE(0xdc00, 0xdcff, FORCEWAKE_RENDER),
1781	GEN_FW_RANGE(0xdd00, 0xde7f, FORCEWAKE_GT), /*
1782	0xdd00 - 0xddff: gt
1783	0xde00 - 0xde7f: reserved */
1784	GEN_FW_RANGE(0xde80, 0xe8ff, FORCEWAKE_RENDER), /*
1785	0xde80 - 0xdfff: render
1786	0xe000 - 0xe0ff: reserved
1787	0xe100 - 0xe8ff: render */
1788	GEN_FW_RANGE(0xe900, 0xe9ff, FORCEWAKE_GT),
1789	GEN_FW_RANGE(0xea00, 0x147ff, 0), /*
1790	0xea00 - 0x11fff: reserved
1791	0x12000 - 0x127ff: always on
1792	0x12800 - 0x147ff: reserved */
1793	GEN_FW_RANGE(0x14800, 0x19fff, FORCEWAKE_GT), /*
1794	0x14800 - 0x153ff: gt
1795	0x15400 - 0x19fff: reserved */
1796	GEN_FW_RANGE(0x1a000, 0x21fff, FORCEWAKE_RENDER), /*
1797	0x1a000 - 0x1bfff: render
1798	0x1c000 - 0x21fff: reserved */
1799	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_GT),
1800	GEN_FW_RANGE(0x24000, 0x2ffff, 0), /*
1801	0x24000 - 0x2407f: always on
1802	0x24080 - 0x2ffff: reserved */
1803	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_GT)
1804	};
1805
1806	/*
1807	* Note that the register ranges here are the final offsets after
1808	* translation of the GSI block to the 0x380000 offset.
1809	*
1810	* NOTE: There are a couple MCR ranges near the bottom of this table
1811	* that need to power up either VD0 or VD2 depending on which replicated
1812	* instance of the register we're trying to access. Our forcewake logic
1813	* at the moment doesn't have a good way to take steering into consideration,
1814	* and the driver doesn't even access any registers in those ranges today,
1815	* so for now we just mark those ranges as FORCEWAKE_ALL. That will ensure
1816	* proper operation if we do start using the ranges in the future, and we
1817	* can determine at that time whether it's worth adding extra complexity to
1818	* the forcewake handling to take steering into consideration.
1819	*/
1820	static const struct intel_forcewake_range __xelpmp_fw_ranges[] = {
1821	GEN_FW_RANGE(0x0, 0x115fff, 0), /* render GT range */
1822	GEN_FW_RANGE(0x116000, 0x11ffff, FORCEWAKE_GSC), /*
1823	0x116000 - 0x117fff: gsc
1824	0x118000 - 0x119fff: reserved
1825	0x11a000 - 0x11efff: gsc
1826	0x11f000 - 0x11ffff: reserved */
1827	GEN_FW_RANGE(0x120000, 0x1bffff, 0), /* non-GT range */
1828	GEN_FW_RANGE(0x1c0000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX0), /*
1829	0x1c0000 - 0x1c3dff: VD0
1830	0x1c3e00 - 0x1c3eff: reserved
1831	0x1c3f00 - 0x1c3fff: VD0
1832	0x1c4000 - 0x1c7fff: reserved */
1833	GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), /*
1834	0x1c8000 - 0x1ca0ff: VE0
1835	0x1ca100 - 0x1cbfff: reserved */
1836	GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_MEDIA_VDBOX0), /*
1837	0x1cc000 - 0x1cdfff: VD0
1838	0x1ce000 - 0x1cffff: reserved */
1839	GEN_FW_RANGE(0x1d0000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX2), /*
1840	0x1d0000 - 0x1d3dff: VD2
1841	0x1d3e00 - 0x1d3eff: reserved
1842	0x1d4000 - 0x1d7fff: VD2 */
1843	GEN_FW_RANGE(0x1d8000, 0x1da0ff, FORCEWAKE_MEDIA_VEBOX1),
1844	GEN_FW_RANGE(0x1da100, 0x380aff, 0), /*
1845	0x1da100 - 0x23ffff: reserved
1846	0x240000 - 0x37ffff: non-GT range
1847	0x380000 - 0x380aff: reserved */
1848	GEN_FW_RANGE(0x380b00, 0x380bff, FORCEWAKE_GT),
1849	GEN_FW_RANGE(0x380c00, 0x380fff, 0),
1850	GEN_FW_RANGE(0x381000, 0x38817f, FORCEWAKE_GT), /*
1851	0x381000 - 0x381fff: gt
1852	0x382000 - 0x383fff: reserved
1853	0x384000 - 0x384aff: gt
1854	0x384b00 - 0x3851ff: reserved
1855	0x385200 - 0x3871ff: gt
1856	0x387200 - 0x387fff: reserved
1857	0x388000 - 0x38813f: gt
1858	0x388140 - 0x38817f: reserved */
1859	GEN_FW_RANGE(0x388180, 0x3882ff, 0), /*
1860	0x388180 - 0x3881ff: always on
1861	0x388200 - 0x3882ff: reserved */
1862	GEN_FW_RANGE(0x388300, 0x38955f, FORCEWAKE_GT), /*
1863	0x388300 - 0x38887f: gt
1864	0x388880 - 0x388fff: reserved
1865	0x389000 - 0x38947f: gt
1866	0x389480 - 0x38955f: reserved */
1867	GEN_FW_RANGE(0x389560, 0x389fff, 0), /*
1868	0x389560 - 0x3895ff: always on
1869	0x389600 - 0x389fff: reserved */
1870	GEN_FW_RANGE(0x38a000, 0x38cfff, FORCEWAKE_GT), /*
1871	0x38a000 - 0x38afff: gt
1872	0x38b000 - 0x38bfff: reserved
1873	0x38c000 - 0x38cfff: gt */
1874	GEN_FW_RANGE(0x38d000, 0x38d11f, 0),
1875	GEN_FW_RANGE(0x38d120, 0x391fff, FORCEWAKE_GT), /*
1876	0x38d120 - 0x38dfff: gt
1877	0x38e000 - 0x38efff: reserved
1878	0x38f000 - 0x38ffff: gt
1879	0x389000 - 0x391fff: reserved */
1880	GEN_FW_RANGE(0x392000, 0x392fff, 0), /*
1881	0x392000 - 0x3927ff: always on
1882	0x392800 - 0x292fff: reserved */
1883	GEN_FW_RANGE(0x393000, 0x3931ff, FORCEWAKE_GT),
1884	GEN_FW_RANGE(0x393200, 0x39323f, FORCEWAKE_ALL), /* instance-based, see note above */
1885	GEN_FW_RANGE(0x393240, 0x3933ff, FORCEWAKE_GT),
1886	GEN_FW_RANGE(0x393400, 0x3934ff, FORCEWAKE_ALL), /* instance-based, see note above */
1887	GEN_FW_RANGE(0x393500, 0x393c7f, 0), /*
1888	0x393500 - 0x393bff: reserved
1889	0x393c00 - 0x393c7f: always on */
1890	GEN_FW_RANGE(0x393c80, 0x393dff, FORCEWAKE_GT),
1891	};
1892
1893	static void
1894	ilk_dummy_write(struct intel_uncore *uncore)
1895	{
1896	/* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1897	* the chip from rc6 before touching it for real. MI_MODE is masked,
1898	* hence harmless to write 0 into. */
1899	__raw_uncore_write32(uncore, RING_MI_MODE(RENDER_RING_BASE), val: 0);
1900	}
1901
1902	static void
1903	__unclaimed_reg_debug(struct intel_uncore *uncore,
1904	const i915_reg_t reg,
1905	const bool read)
1906	{
1907	if (drm_WARN(&uncore->i915->drm,
1908	check_for_unclaimed_mmio(uncore),
1909	"Unclaimed %s register 0x%x\n",
1910	read ? "read from" : "write to",
1911	i915_mmio_reg_offset(reg)))
1912	/* Only report the first N failures */
1913	uncore->i915->params.mmio_debug--;
1914	}
1915
1916	static void
1917	__unclaimed_previous_reg_debug(struct intel_uncore *uncore,
1918	const i915_reg_t reg,
1919	const bool read)
1920	{
1921	if (check_for_unclaimed_mmio(uncore))
1922	drm_dbg(&uncore->i915->drm,
1923	"Unclaimed access detected before %s register 0x%x\n",
1924	read ? "read from" : "write to",
1925	i915_mmio_reg_offset(reg));
1926	}
1927
1928	static inline bool __must_check
1929	unclaimed_reg_debug_header(struct intel_uncore *uncore,
1930	const i915_reg_t reg, const bool read)
1931	{
1932	if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug)
1933	return false;
1934
1935	/* interrupts are disabled and re-enabled around uncore->lock usage */
1936	lockdep_assert_held(&uncore->lock);
1937
1938	spin_lock(lock: &uncore->debug->lock);
1939	__unclaimed_previous_reg_debug(uncore, reg, read);
1940
1941	return true;
1942	}
1943
1944	static inline void
1945	unclaimed_reg_debug_footer(struct intel_uncore *uncore,
1946	const i915_reg_t reg, const bool read)
1947	{
1948	/* interrupts are disabled and re-enabled around uncore->lock usage */
1949	lockdep_assert_held(&uncore->lock);
1950
1951	__unclaimed_reg_debug(uncore, reg, read);
1952	spin_unlock(lock: &uncore->debug->lock);
1953	}
1954
1955	#define __vgpu_read(x) \
1956	static u##x \
1957	vgpu_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1958	u##x val = __raw_uncore_read##x(uncore, reg); \
1959	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1960	return val; \
1961	}
1962	__vgpu_read(8)
1963	__vgpu_read(16)
1964	__vgpu_read(32)
1965	__vgpu_read(64)
1966
1967	#define GEN2_READ_HEADER(x) \
1968	u##x val = 0; \
1969	assert_rpm_wakelock_held(uncore->rpm);
1970
1971	#define GEN2_READ_FOOTER \
1972	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1973	return val
1974
1975	#define __gen2_read(x) \
1976	static u##x \
1977	gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1978	GEN2_READ_HEADER(x); \
1979	val = __raw_uncore_read##x(uncore, reg); \
1980	GEN2_READ_FOOTER; \
1981	}
1982
1983	#define __gen5_read(x) \
1984	static u##x \
1985	gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
1986	GEN2_READ_HEADER(x); \
1987	ilk_dummy_write(uncore); \
1988	val = __raw_uncore_read##x(uncore, reg); \
1989	GEN2_READ_FOOTER; \
1990	}
1991
1992	__gen5_read(8)
1993	__gen5_read(16)
1994	__gen5_read(32)
1995	__gen5_read(64)
1996	__gen2_read(8)
1997	__gen2_read(16)
1998	__gen2_read(32)
1999	__gen2_read(64)
2000
2001	#undef __gen5_read
2002	#undef __gen2_read
2003
2004	#undef GEN2_READ_FOOTER
2005	#undef GEN2_READ_HEADER
2006
2007	#define GEN6_READ_HEADER(x) \
2008	u32 offset = i915_mmio_reg_offset(reg); \
2009	unsigned long irqflags; \
2010	bool unclaimed_reg_debug; \
2011	u##x val = 0; \
2012	assert_rpm_wakelock_held(uncore->rpm); \
2013	spin_lock_irqsave(&uncore->lock, irqflags); \
2014	unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, true)
2015
2016	#define GEN6_READ_FOOTER \
2017	if (unclaimed_reg_debug) \
2018	unclaimed_reg_debug_footer(uncore, reg, true); \
2019	spin_unlock_irqrestore(&uncore->lock, irqflags); \
2020	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
2021	return val
2022
2023	static noinline void ___force_wake_auto(struct intel_uncore *uncore,
2024	enum forcewake_domains fw_domains)
2025	{
2026	struct intel_uncore_forcewake_domain *domain;
2027	unsigned int tmp;
2028
2029	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
2030
2031	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
2032	fw_domain_arm_timer(d: domain);
2033
2034	fw_domains_get(uncore, fw_domains);
2035	}
2036
2037	static inline void __force_wake_auto(struct intel_uncore *uncore,
2038	enum forcewake_domains fw_domains)
2039	{
2040	GEM_BUG_ON(!fw_domains);
2041
2042	/* Turn on all requested but inactive supported forcewake domains. */
2043	fw_domains &= uncore->fw_domains;
2044	fw_domains &= ~uncore->fw_domains_active;
2045
2046	if (fw_domains)
2047	___force_wake_auto(uncore, fw_domains);
2048	}
2049
2050	#define __gen_fwtable_read(x) \
2051	static u##x \
2052	fwtable_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) \
2053	{ \
2054	enum forcewake_domains fw_engine; \
2055	GEN6_READ_HEADER(x); \
2056	fw_engine = __fwtable_reg_read_fw_domains(uncore, offset); \
2057	if (fw_engine) \
2058	__force_wake_auto(uncore, fw_engine); \
2059	val = __raw_uncore_read##x(uncore, reg); \
2060	GEN6_READ_FOOTER; \
2061	}
2062
2063	static enum forcewake_domains
2064	fwtable_reg_read_fw_domains(struct intel_uncore *uncore, i915_reg_t reg) {
2065	return __fwtable_reg_read_fw_domains(uncore, i915_mmio_reg_offset(reg));
2066	}
2067
2068	__gen_fwtable_read(8)
2069	__gen_fwtable_read(16)
2070	__gen_fwtable_read(32)
2071	__gen_fwtable_read(64)
2072
2073	#undef __gen_fwtable_read
2074	#undef GEN6_READ_FOOTER
2075	#undef GEN6_READ_HEADER
2076
2077	#define GEN2_WRITE_HEADER \
2078	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2079	assert_rpm_wakelock_held(uncore->rpm); \
2080
2081	#define GEN2_WRITE_FOOTER
2082
2083	#define __gen2_write(x) \
2084	static void \
2085	gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2086	GEN2_WRITE_HEADER; \
2087	__raw_uncore_write##x(uncore, reg, val); \
2088	GEN2_WRITE_FOOTER; \
2089	}
2090
2091	#define __gen5_write(x) \
2092	static void \
2093	gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2094	GEN2_WRITE_HEADER; \
2095	ilk_dummy_write(uncore); \
2096	__raw_uncore_write##x(uncore, reg, val); \
2097	GEN2_WRITE_FOOTER; \
2098	}
2099
2100	__gen5_write(8)
2101	__gen5_write(16)
2102	__gen5_write(32)
2103	__gen2_write(8)
2104	__gen2_write(16)
2105	__gen2_write(32)
2106
2107	#undef __gen5_write
2108	#undef __gen2_write
2109
2110	#undef GEN2_WRITE_FOOTER
2111	#undef GEN2_WRITE_HEADER
2112
2113	#define GEN6_WRITE_HEADER \
2114	u32 offset = i915_mmio_reg_offset(reg); \
2115	unsigned long irqflags; \
2116	bool unclaimed_reg_debug; \
2117	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2118	assert_rpm_wakelock_held(uncore->rpm); \
2119	spin_lock_irqsave(&uncore->lock, irqflags); \
2120	unclaimed_reg_debug = unclaimed_reg_debug_header(uncore, reg, false)
2121
2122	#define GEN6_WRITE_FOOTER \
2123	if (unclaimed_reg_debug) \
2124	unclaimed_reg_debug_footer(uncore, reg, false); \
2125	spin_unlock_irqrestore(&uncore->lock, irqflags)
2126
2127	#define __gen6_write(x) \
2128	static void \
2129	gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2130	GEN6_WRITE_HEADER; \
2131	if (NEEDS_FORCE_WAKE(offset)) \
2132	__gen6_gt_wait_for_fifo(uncore); \
2133	__raw_uncore_write##x(uncore, reg, val); \
2134	GEN6_WRITE_FOOTER; \
2135	}
2136	__gen6_write(8)
2137	__gen6_write(16)
2138	__gen6_write(32)
2139
2140	#define __gen_fwtable_write(x) \
2141	static void \
2142	fwtable_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2143	enum forcewake_domains fw_engine; \
2144	GEN6_WRITE_HEADER; \
2145	fw_engine = __fwtable_reg_write_fw_domains(uncore, offset); \
2146	if (fw_engine) \
2147	__force_wake_auto(uncore, fw_engine); \
2148	__raw_uncore_write##x(uncore, reg, val); \
2149	GEN6_WRITE_FOOTER; \
2150	}
2151
2152	static enum forcewake_domains
2153	fwtable_reg_write_fw_domains(struct intel_uncore *uncore, i915_reg_t reg)
2154	{
2155	return __fwtable_reg_write_fw_domains(uncore, i915_mmio_reg_offset(reg));
2156	}
2157
2158	__gen_fwtable_write(8)
2159	__gen_fwtable_write(16)
2160	__gen_fwtable_write(32)
2161
2162	#undef __gen_fwtable_write
2163	#undef GEN6_WRITE_FOOTER
2164	#undef GEN6_WRITE_HEADER
2165
2166	#define __vgpu_write(x) \
2167	static void \
2168	vgpu_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
2169	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
2170	__raw_uncore_write##x(uncore, reg, val); \
2171	}
2172	__vgpu_write(8)
2173	__vgpu_write(16)
2174	__vgpu_write(32)
2175
2176	#define ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, x) \
2177	do { \
2178	(uncore)->funcs.mmio_writeb = x##_write8; \
2179	(uncore)->funcs.mmio_writew = x##_write16; \
2180	(uncore)->funcs.mmio_writel = x##_write32; \
2181	} while (0)
2182
2183	#define ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x) \
2184	do { \
2185	(uncore)->funcs.mmio_readb = x##_read8; \
2186	(uncore)->funcs.mmio_readw = x##_read16; \
2187	(uncore)->funcs.mmio_readl = x##_read32; \
2188	(uncore)->funcs.mmio_readq = x##_read64; \
2189	} while (0)
2190
2191	#define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \
2192	do { \
2193	ASSIGN_RAW_WRITE_MMIO_VFUNCS((uncore), x); \
2194	(uncore)->funcs.write_fw_domains = x##_reg_write_fw_domains; \
2195	} while (0)
2196
2197	#define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \
2198	do { \
2199	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, x); \
2200	(uncore)->funcs.read_fw_domains = x##_reg_read_fw_domains; \
2201	} while (0)
2202
2203	static int __fw_domain_init(struct intel_uncore *uncore,
2204	enum forcewake_domain_id domain_id,
2205	i915_reg_t reg_set,
2206	i915_reg_t reg_ack)
2207	{
2208	struct intel_uncore_forcewake_domain *d;
2209
2210	GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2211	GEM_BUG_ON(uncore->fw_domain[domain_id]);
2212
2213	if (i915_inject_probe_failure(uncore->i915))
2214	return -ENOMEM;
2215
2216	d = kzalloc(size: sizeof(*d), GFP_KERNEL);
2217	if (!d)
2218	return -ENOMEM;
2219
2220	drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_set));
2221	drm_WARN_ON(&uncore->i915->drm, !i915_mmio_reg_valid(reg_ack));
2222
2223	d->uncore = uncore;
2224	d->wake_count = 0;
2225	d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset;
2226	d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset;
2227
2228	d->id = domain_id;
2229
2230	BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
2231	BUILD_BUG_ON(FORCEWAKE_GT != (1 << FW_DOMAIN_ID_GT));
2232	BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
2233	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
2234	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
2235	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
2236	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
2237	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX4 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX4));
2238	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX5 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX5));
2239	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX6 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX6));
2240	BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX7 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX7));
2241	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
2242	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
2243	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX2));
2244	BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX3));
2245	BUILD_BUG_ON(FORCEWAKE_GSC != (1 << FW_DOMAIN_ID_GSC));
2246
2247	d->mask = BIT(domain_id);
2248
2249	hrtimer_init(timer: &d->timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
2250	d->timer.function = intel_uncore_fw_release_timer;
2251
2252	uncore->fw_domains |= BIT(domain_id);
2253
2254	fw_domain_reset(d);
2255
2256	uncore->fw_domain[domain_id] = d;
2257
2258	return 0;
2259	}
2260
2261	static void fw_domain_fini(struct intel_uncore *uncore,
2262	enum forcewake_domain_id domain_id)
2263	{
2264	struct intel_uncore_forcewake_domain *d;
2265
2266	GEM_BUG_ON(domain_id >= FW_DOMAIN_ID_COUNT);
2267
2268	d = fetch_and_zero(&uncore->fw_domain[domain_id]);
2269	if (!d)
2270	return;
2271
2272	uncore->fw_domains &= ~BIT(domain_id);
2273	drm_WARN_ON(&uncore->i915->drm, d->wake_count);
2274	drm_WARN_ON(&uncore->i915->drm, hrtimer_cancel(&d->timer));
2275	kfree(objp: d);
2276	}
2277
2278	static void intel_uncore_fw_domains_fini(struct intel_uncore *uncore)
2279	{
2280	struct intel_uncore_forcewake_domain *d;
2281	int tmp;
2282
2283	for_each_fw_domain(d, uncore, tmp)
2284	fw_domain_fini(uncore, domain_id: d->id);
2285	}
2286
2287	static const struct intel_uncore_fw_get uncore_get_fallback = {
2288	.force_wake_get = fw_domains_get_with_fallback
2289	};
2290
2291	static const struct intel_uncore_fw_get uncore_get_normal = {
2292	.force_wake_get = fw_domains_get_normal,
2293	};
2294
2295	static const struct intel_uncore_fw_get uncore_get_thread_status = {
2296	.force_wake_get = fw_domains_get_with_thread_status
2297	};
2298
2299	static int intel_uncore_fw_domains_init(struct intel_uncore *uncore)
2300	{
2301	struct drm_i915_private *i915 = uncore->i915;
2302	int ret = 0;
2303
2304	GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2305
2306	#define fw_domain_init(uncore__, id__, set__, ack__) \
2307	(ret ?: (ret = __fw_domain_init((uncore__), (id__), (set__), (ack__))))
2308
2309	if (GRAPHICS_VER(i915) >= 11) {
2310	intel_engine_mask_t emask;
2311	int i;
2312
2313	/* we'll prune the domains of missing engines later */
2314	emask = uncore->gt->info.engine_mask;
2315
2316	uncore->fw_get_funcs = &uncore_get_fallback;
2317	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2318	fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2319	FORCEWAKE_GT_GEN9,
2320	FORCEWAKE_ACK_GT_MTL);
2321	else
2322	fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2323	FORCEWAKE_GT_GEN9,
2324	FORCEWAKE_ACK_GT_GEN9);
2325
2326	if (RCS_MASK(uncore->gt) || CCS_MASK(uncore->gt))
2327	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2328	FORCEWAKE_RENDER_GEN9,
2329	FORCEWAKE_ACK_RENDER_GEN9);
2330
2331	for (i = 0; i < I915_MAX_VCS; i++) {
2332	if (!__HAS_ENGINE(emask, _VCS(i)))
2333	continue;
2334
2335	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
2336	FORCEWAKE_MEDIA_VDBOX_GEN11(i),
2337	FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
2338	}
2339	for (i = 0; i < I915_MAX_VECS; i++) {
2340	if (!__HAS_ENGINE(emask, _VECS(i)))
2341	continue;
2342
2343	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
2344	FORCEWAKE_MEDIA_VEBOX_GEN11(i),
2345	FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
2346	}
2347
2348	if (uncore->gt->type == GT_MEDIA)
2349	fw_domain_init(uncore, FW_DOMAIN_ID_GSC,
2350	FORCEWAKE_REQ_GSC, FORCEWAKE_ACK_GSC);
2351	} else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2352	uncore->fw_get_funcs = &uncore_get_fallback;
2353	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2354	FORCEWAKE_RENDER_GEN9,
2355	FORCEWAKE_ACK_RENDER_GEN9);
2356	fw_domain_init(uncore, FW_DOMAIN_ID_GT,
2357	FORCEWAKE_GT_GEN9,
2358	FORCEWAKE_ACK_GT_GEN9);
2359	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2360	FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
2361	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2362	uncore->fw_get_funcs = &uncore_get_normal;
2363	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2364	FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
2365	fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
2366	FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
2367	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
2368	uncore->fw_get_funcs = &uncore_get_thread_status;
2369	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2370	FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
2371	} else if (IS_IVYBRIDGE(i915)) {
2372	u32 ecobus;
2373
2374	/* IVB configs may use multi-threaded forcewake */
2375
2376	/* A small trick here - if the bios hasn't configured
2377	* MT forcewake, and if the device is in RC6, then
2378	* force_wake_mt_get will not wake the device and the
2379	* ECOBUS read will return zero. Which will be
2380	* (correctly) interpreted by the test below as MT
2381	* forcewake being disabled.
2382	*/
2383	uncore->fw_get_funcs = &uncore_get_thread_status;
2384
2385	/* We need to init first for ECOBUS access and then
2386	* determine later if we want to reinit, in case of MT access is
2387	* not working. In this stage we don't know which flavour this
2388	* ivb is, so it is better to reset also the gen6 fw registers
2389	* before the ecobus check.
2390	*/
2391
2392	__raw_uncore_write32(uncore, FORCEWAKE, val: 0);
2393	__raw_posting_read(uncore, ECOBUS);
2394
2395	ret = __fw_domain_init(uncore, domain_id: FW_DOMAIN_ID_RENDER,
2396	FORCEWAKE_MT, FORCEWAKE_MT_ACK);
2397	if (ret)
2398	goto out;
2399
2400	spin_lock_irq(lock: &uncore->lock);
2401	fw_domains_get_with_thread_status(uncore, fw_domains: FORCEWAKE_RENDER);
2402	ecobus = __raw_uncore_read32(uncore, ECOBUS);
2403	fw_domains_put(uncore, fw_domains: FORCEWAKE_RENDER);
2404	spin_unlock_irq(lock: &uncore->lock);
2405
2406	if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
2407	drm_info(&i915->drm, "No MT forcewake available on Ivybridge, this can result in issues\n");
2408	drm_info(&i915->drm, "when using vblank-synced partial screen updates.\n");
2409	fw_domain_fini(uncore, domain_id: FW_DOMAIN_ID_RENDER);
2410	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2411	FORCEWAKE, FORCEWAKE_ACK);
2412	}
2413	} else if (GRAPHICS_VER(i915) == 6) {
2414	uncore->fw_get_funcs = &uncore_get_thread_status;
2415	fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
2416	FORCEWAKE, FORCEWAKE_ACK);
2417	}
2418
2419	#undef fw_domain_init
2420
2421	/* All future platforms are expected to require complex power gating */
2422	drm_WARN_ON(&i915->drm, !ret && uncore->fw_domains == 0);
2423
2424	out:
2425	if (ret)
2426	intel_uncore_fw_domains_fini(uncore);
2427
2428	return ret;
2429	}
2430
2431	#define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \
2432	{ \
2433	(uncore)->fw_domains_table = \
2434	(struct intel_forcewake_range *)(d); \
2435	(uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \
2436	}
2437
2438	#define ASSIGN_SHADOW_TABLE(uncore, d) \
2439	{ \
2440	(uncore)->shadowed_reg_table = d; \
2441	(uncore)->shadowed_reg_table_entries = ARRAY_SIZE((d)); \
2442	}
2443
2444	static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
2445	unsigned long action, void *data)
2446	{
2447	struct intel_uncore *uncore = container_of(nb,
2448	struct intel_uncore, pmic_bus_access_nb);
2449
2450	switch (action) {
2451	case MBI_PMIC_BUS_ACCESS_BEGIN:
2452	/*
2453	* forcewake all now to make sure that we don't need to do a
2454	* forcewake later which on systems where this notifier gets
2455	* called requires the punit to access to the shared pmic i2c
2456	* bus, which will be busy after this notification, leading to:
2457	* "render: timed out waiting for forcewake ack request."
2458	* errors.
2459	*
2460	* The notifier is unregistered during intel_runtime_suspend(),
2461	* so it's ok to access the HW here without holding a RPM
2462	* wake reference -> disable wakeref asserts for the time of
2463	* the access.
2464	*/
2465	disable_rpm_wakeref_asserts(rpm: uncore->rpm);
2466	intel_uncore_forcewake_get(uncore, fw_domains: FORCEWAKE_ALL);
2467	enable_rpm_wakeref_asserts(rpm: uncore->rpm);
2468	break;
2469	case MBI_PMIC_BUS_ACCESS_END:
2470	intel_uncore_forcewake_put(uncore, fw_domains: FORCEWAKE_ALL);
2471	break;
2472	}
2473
2474	return NOTIFY_OK;
2475	}
2476
2477	static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
2478	{
2479	iounmap(addr: (void __iomem *)regs);
2480	}
2481
2482	int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
2483	{
2484	struct drm_i915_private *i915 = uncore->i915;
2485	int mmio_size;
2486
2487	/*
2488	* Before gen4, the registers and the GTT are behind different BARs.
2489	* However, from gen4 onwards, the registers and the GTT are shared
2490	* in the same BAR, so we want to restrict this ioremap from
2491	* clobbering the GTT which we want ioremap_wc instead. Fortunately,
2492	* the register BAR remains the same size for all the earlier
2493	* generations up to Ironlake.
2494	* For dgfx chips register range is expanded to 4MB, and this larger
2495	* range is also used for integrated gpus beginning with Meteor Lake.
2496	*/
2497	if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2498	mmio_size = 4 * 1024 * 1024;
2499	else if (GRAPHICS_VER(i915) >= 5)
2500	mmio_size = 2 * 1024 * 1024;
2501	else
2502	mmio_size = 512 * 1024;
2503
2504	uncore->regs = ioremap(offset: phys_addr, size: mmio_size);
2505	if (uncore->regs == NULL) {
2506	drm_err(&i915->drm, "failed to map registers\n");
2507	return -EIO;
2508	}
2509
2510	return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio,
2511	(void __force *)uncore->regs);
2512	}
2513
2514	void intel_uncore_init_early(struct intel_uncore *uncore,
2515	struct intel_gt *gt)
2516	{
2517	spin_lock_init(&uncore->lock);
2518	uncore->i915 = gt->i915;
2519	uncore->gt = gt;
2520	uncore->rpm = &gt->i915->runtime_pm;
2521	}
2522
2523	static void uncore_raw_init(struct intel_uncore *uncore)
2524	{
2525	GEM_BUG_ON(intel_uncore_has_forcewake(uncore));
2526
2527	if (intel_vgpu_active(i915: uncore->i915)) {
2528	ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, vgpu);
2529	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, vgpu);
2530	} else if (GRAPHICS_VER(uncore->i915) == 5) {
2531	ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen5);
2532	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen5);
2533	} else {
2534	ASSIGN_RAW_WRITE_MMIO_VFUNCS(uncore, gen2);
2535	ASSIGN_RAW_READ_MMIO_VFUNCS(uncore, gen2);
2536	}
2537	}
2538
2539	static int uncore_media_forcewake_init(struct intel_uncore *uncore)
2540	{
2541	struct drm_i915_private *i915 = uncore->i915;
2542
2543	if (MEDIA_VER(i915) >= 13) {
2544	ASSIGN_FW_DOMAINS_TABLE(uncore, __xelpmp_fw_ranges);
2545	ASSIGN_SHADOW_TABLE(uncore, xelpmp_shadowed_regs);
2546	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2547	} else {
2548	MISSING_CASE(MEDIA_VER(i915));
2549	return -ENODEV;
2550	}
2551
2552	return 0;
2553	}
2554
2555	static int uncore_forcewake_init(struct intel_uncore *uncore)
2556	{
2557	struct drm_i915_private *i915 = uncore->i915;
2558	int ret;
2559
2560	GEM_BUG_ON(!intel_uncore_has_forcewake(uncore));
2561
2562	ret = intel_uncore_fw_domains_init(uncore);
2563	if (ret)
2564	return ret;
2565	forcewake_early_sanitize(uncore, restore_forcewake: 0);
2566
2567	ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
2568
2569	if (uncore->gt->type == GT_MEDIA)
2570	return uncore_media_forcewake_init(uncore);
2571
2572	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2573	ASSIGN_FW_DOMAINS_TABLE(uncore, __mtl_fw_ranges);
2574	ASSIGN_SHADOW_TABLE(uncore, mtl_shadowed_regs);
2575	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2576	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60)) {
2577	ASSIGN_FW_DOMAINS_TABLE(uncore, __pvc_fw_ranges);
2578	ASSIGN_SHADOW_TABLE(uncore, pvc_shadowed_regs);
2579	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2580	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
2581	ASSIGN_FW_DOMAINS_TABLE(uncore, __dg2_fw_ranges);
2582	ASSIGN_SHADOW_TABLE(uncore, dg2_shadowed_regs);
2583	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2584	} else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
2585	ASSIGN_FW_DOMAINS_TABLE(uncore, __xehp_fw_ranges);
2586	ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2587	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2588	} else if (GRAPHICS_VER(i915) >= 12) {
2589	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen12_fw_ranges);
2590	ASSIGN_SHADOW_TABLE(uncore, gen12_shadowed_regs);
2591	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2592	} else if (GRAPHICS_VER(i915) == 11) {
2593	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges);
2594	ASSIGN_SHADOW_TABLE(uncore, gen11_shadowed_regs);
2595	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2596	} else if (IS_GRAPHICS_VER(i915, 9, 10)) {
2597	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges);
2598	ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2599	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2600	} else if (IS_CHERRYVIEW(i915)) {
2601	ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges);
2602	ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2603	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2604	} else if (GRAPHICS_VER(i915) == 8) {
2605	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2606	ASSIGN_SHADOW_TABLE(uncore, gen8_shadowed_regs);
2607	ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
2608	} else if (IS_VALLEYVIEW(i915)) {
2609	ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges);
2610	ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2611	} else if (IS_GRAPHICS_VER(i915, 6, 7)) {
2612	ASSIGN_FW_DOMAINS_TABLE(uncore, __gen6_fw_ranges);
2613	ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
2614	}
2615
2616	uncore->pmic_bus_access_nb.notifier_call = i915_pmic_bus_access_notifier;
2617	iosf_mbi_register_pmic_bus_access_notifier(nb: &uncore->pmic_bus_access_nb);
2618
2619	return 0;
2620	}
2621
2622	static int sanity_check_mmio_access(struct intel_uncore *uncore)
2623	{
2624	struct drm_i915_private *i915 = uncore->i915;
2625
2626	if (GRAPHICS_VER(i915) < 8)
2627	return 0;
2628
2629	/*
2630	* Sanitycheck that MMIO access to the device is working properly. If
2631	* the CPU is unable to communcate with a PCI device, BAR reads will
2632	* return 0xFFFFFFFF. Let's make sure the device isn't in this state
2633	* before we start trying to access registers.
2634	*
2635	* We use the primary GT's forcewake register as our guinea pig since
2636	* it's been around since HSW and it's a masked register so the upper
2637	* 16 bits can never read back as 1's if device access is operating
2638	* properly.
2639	*
2640	* If MMIO isn't working, we'll wait up to 2 seconds to see if it
2641	* recovers, then give up.
2642	*/
2643	#define COND (__raw_uncore_read32(uncore, FORCEWAKE_MT) != ~0)
2644	if (wait_for(COND, 2000) == -ETIMEDOUT) {
2645	drm_err(&i915->drm, "Device is non-operational; MMIO access returns 0xFFFFFFFF!\n");
2646	return -EIO;
2647	}
2648
2649	return 0;
2650	}
2651
2652	int intel_uncore_init_mmio(struct intel_uncore *uncore)
2653	{
2654	struct drm_i915_private *i915 = uncore->i915;
2655	int ret;
2656
2657	ret = sanity_check_mmio_access(uncore);
2658	if (ret)
2659	return ret;
2660
2661	/*
2662	* The boot firmware initializes local memory and assesses its health.
2663	* If memory training fails, the punit will have been instructed to
2664	* keep the GT powered down; we won't be able to communicate with it
2665	* and we should not continue with driver initialization.
2666	*/
2667	if (IS_DGFX(i915) &&
2668	!(__raw_uncore_read32(uncore, GU_CNTL) & LMEM_INIT)) {
2669	drm_err(&i915->drm, "LMEM not initialized by firmware\n");
2670	return -ENODEV;
2671	}
2672
2673	if (GRAPHICS_VER(i915) > 5 && !intel_vgpu_active(i915))
2674	uncore->flags |= UNCORE_HAS_FORCEWAKE;
2675
2676	if (!intel_uncore_has_forcewake(uncore)) {
2677	uncore_raw_init(uncore);
2678	} else {
2679	ret = uncore_forcewake_init(uncore);
2680	if (ret)
2681	return ret;
2682	}
2683
2684	/* make sure fw funcs are set if and only if we have fw*/
2685	GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->fw_get_funcs);
2686	GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.read_fw_domains);
2687	GEM_BUG_ON(intel_uncore_has_forcewake(uncore) != !!uncore->funcs.write_fw_domains);
2688
2689	if (HAS_FPGA_DBG_UNCLAIMED(i915))
2690	uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED;
2691
2692	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2693	uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED;
2694
2695	if (IS_GRAPHICS_VER(i915, 6, 7))
2696	uncore->flags |= UNCORE_HAS_FIFO;
2697
2698	/* clear out unclaimed reg detection bit */
2699	if (intel_uncore_unclaimed_mmio(uncore))
2700	drm_dbg(&i915->drm, "unclaimed mmio detected on uncore init, clearing\n");
2701
2702	return 0;
2703	}
2704
2705	/*
2706	* We might have detected that some engines are fused off after we initialized
2707	* the forcewake domains. Prune them, to make sure they only reference existing
2708	* engines.
2709	*/
2710	void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
2711	struct intel_gt *gt)
2712	{
2713	enum forcewake_domains fw_domains = uncore->fw_domains;
2714	enum forcewake_domain_id domain_id;
2715	int i;
2716
2717	if (!intel_uncore_has_forcewake(uncore) || GRAPHICS_VER(uncore->i915) < 11)
2718	return;
2719
2720	for (i = 0; i < I915_MAX_VCS; i++) {
2721	domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
2722
2723	if (HAS_ENGINE(gt, _VCS(i)))
2724	continue;
2725
2726	/*
2727	* Starting with XeHP, the power well for an even-numbered
2728	* VDBOX is also used for shared units within the
2729	* media slice such as SFC. So even if the engine
2730	* itself is fused off, we still need to initialize
2731	* the forcewake domain if any of the other engines
2732	* in the same media slice are present.
2733	*/
2734	if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 50) && i % 2 == 0) {
2735	if ((i + 1 < I915_MAX_VCS) && HAS_ENGINE(gt, _VCS(i + 1)))
2736	continue;
2737
2738	if (HAS_ENGINE(gt, _VECS(i / 2)))
2739	continue;
2740	}
2741
2742	if (fw_domains & BIT(domain_id))
2743	fw_domain_fini(uncore, domain_id);
2744	}
2745
2746	for (i = 0; i < I915_MAX_VECS; i++) {
2747	domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
2748
2749	if (HAS_ENGINE(gt, _VECS(i)))
2750	continue;
2751
2752	if (fw_domains & BIT(domain_id))
2753	fw_domain_fini(uncore, domain_id);
2754	}
2755
2756	if ((fw_domains & BIT(FW_DOMAIN_ID_GSC)) && !HAS_ENGINE(gt, GSC0))
2757	fw_domain_fini(uncore, domain_id: FW_DOMAIN_ID_GSC);
2758	}
2759
2760	/*
2761	* The driver-initiated FLR is the highest level of reset that we can trigger
2762	* from within the driver. It is different from the PCI FLR in that it doesn't
2763	* fully reset the SGUnit and doesn't modify the PCI config space and therefore
2764	* it doesn't require a re-enumeration of the PCI BARs. However, the
2765	* driver-initiated FLR does still cause a reset of both GT and display and a
2766	* memory wipe of local and stolen memory, so recovery would require a full HW
2767	* re-init and saving/restoring (or re-populating) the wiped memory. Since we
2768	* perform the FLR as the very last action before releasing access to the HW
2769	* during the driver release flow, we don't attempt recovery at all, because
2770	* if/when a new instance of i915 is bound to the device it will do a full
2771	* re-init anyway.
2772	*/
2773	static void driver_initiated_flr(struct intel_uncore *uncore)
2774	{
2775	struct drm_i915_private *i915 = uncore->i915;
2776	const unsigned int flr_timeout_ms = 3000; /* specs recommend a 3s wait */
2777	int ret;
2778
2779	drm_dbg(&i915->drm, "Triggering Driver-FLR\n");
2780
2781	/*
2782	* Make sure any pending FLR requests have cleared by waiting for the
2783	* FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
2784	* to make sure it's not still set from a prior attempt (it's a write to
2785	* clear bit).
2786	* Note that we should never be in a situation where a previous attempt
2787	* is still pending (unless the HW is totally dead), but better to be
2788	* safe in case something unexpected happens
2789	*/
2790	ret = intel_wait_for_register_fw(uncore, GU_CNTL, DRIVERFLR, value: 0, timeout_ms: flr_timeout_ms);
2791	if (ret) {
2792	drm_err(&i915->drm,
2793	"Failed to wait for Driver-FLR bit to clear! %d\n",
2794	ret);
2795	return;
2796	}
2797	intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2798
2799	/* Trigger the actual Driver-FLR */
2800	intel_uncore_rmw_fw(uncore, GU_CNTL, clear: 0, DRIVERFLR);
2801
2802	/* Wait for hardware teardown to complete */
2803	ret = intel_wait_for_register_fw(uncore, GU_CNTL,
2804	DRIVERFLR, value: 0,
2805	timeout_ms: flr_timeout_ms);
2806	if (ret) {
2807	drm_err(&i915->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
2808	return;
2809	}
2810
2811	/* Wait for hardware/firmware re-init to complete */
2812	ret = intel_wait_for_register_fw(uncore, GU_DEBUG,
2813	DRIVERFLR_STATUS, DRIVERFLR_STATUS,
2814	timeout_ms: flr_timeout_ms);
2815	if (ret) {
2816	drm_err(&i915->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
2817	return;
2818	}
2819
2820	/* Clear sticky completion status */
2821	intel_uncore_write_fw(uncore, GU_DEBUG, DRIVERFLR_STATUS);
2822	}
2823
2824	/* Called via drm-managed action */
2825	void intel_uncore_fini_mmio(struct drm_device *dev, void *data)
2826	{
2827	struct intel_uncore *uncore = data;
2828
2829	if (intel_uncore_has_forcewake(uncore)) {
2830	iosf_mbi_punit_acquire();
2831	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
2832	nb: &uncore->pmic_bus_access_nb);
2833	intel_uncore_forcewake_reset(uncore);
2834	intel_uncore_fw_domains_fini(uncore);
2835	iosf_mbi_punit_release();
2836	}
2837
2838	if (intel_uncore_needs_flr_on_fini(uncore))
2839	driver_initiated_flr(uncore);
2840	}
2841
2842	/**
2843	* __intel_wait_for_register_fw - wait until register matches expected state
2844	* @uncore: the struct intel_uncore
2845	* @reg: the register to read
2846	* @mask: mask to apply to register value
2847	* @value: expected value
2848	* @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2849	* @slow_timeout_ms: slow timeout in millisecond
2850	* @out_value: optional placeholder to hold registry value
2851	*
2852	* This routine waits until the target register @reg contains the expected
2853	* @value after applying the @mask, i.e. it waits until ::
2854	*
2855	* (intel_uncore_read_fw(uncore, reg) & mask) == value
2856	*
2857	* Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
2858	* For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
2859	* must be not larger than 20,0000 microseconds.
2860	*
2861	* Note that this routine assumes the caller holds forcewake asserted, it is
2862	* not suitable for very long waits. See intel_wait_for_register() if you
2863	* wish to wait without holding forcewake for the duration (i.e. you expect
2864	* the wait to be slow).
2865	*
2866	* Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2867	*/
2868	int __intel_wait_for_register_fw(struct intel_uncore *uncore,
2869	i915_reg_t reg,
2870	u32 mask,
2871	u32 value,
2872	unsigned int fast_timeout_us,
2873	unsigned int slow_timeout_ms,
2874	u32 *out_value)
2875	{
2876	u32 reg_value = 0;
2877	#define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value)
2878	int ret;
2879
2880	/* Catch any overuse of this function */
2881	might_sleep_if(slow_timeout_ms);
2882	GEM_BUG_ON(fast_timeout_us > 20000);
2883	GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms);
2884
2885	ret = -ETIMEDOUT;
2886	if (fast_timeout_us && fast_timeout_us <= 20000)
2887	ret = _wait_for_atomic(done, fast_timeout_us, 0);
2888	if (ret && slow_timeout_ms)
2889	ret = wait_for(done, slow_timeout_ms);
2890
2891	if (out_value)
2892	*out_value = reg_value;
2893
2894	return ret;
2895	#undef done
2896	}
2897
2898	/**
2899	* __intel_wait_for_register - wait until register matches expected state
2900	* @uncore: the struct intel_uncore
2901	* @reg: the register to read
2902	* @mask: mask to apply to register value
2903	* @value: expected value
2904	* @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2905	* @slow_timeout_ms: slow timeout in millisecond
2906	* @out_value: optional placeholder to hold registry value
2907	*
2908	* This routine waits until the target register @reg contains the expected
2909	* @value after applying the @mask, i.e. it waits until ::
2910	*
2911	* (intel_uncore_read(uncore, reg) & mask) == value
2912	*
2913	* Otherwise, the wait will timeout after @timeout_ms milliseconds.
2914	*
2915	* Return: 0 if the register matches the desired condition, or -ETIMEDOUT.
2916	*/
2917	int __intel_wait_for_register(struct intel_uncore *uncore,
2918	i915_reg_t reg,
2919	u32 mask,
2920	u32 value,
2921	unsigned int fast_timeout_us,
2922	unsigned int slow_timeout_ms,
2923	u32 *out_value)
2924	{
2925	unsigned fw =
2926	intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
2927	u32 reg_value;
2928	int ret;
2929
2930	might_sleep_if(slow_timeout_ms);
2931
2932	spin_lock_irq(lock: &uncore->lock);
2933	intel_uncore_forcewake_get__locked(uncore, fw_domains: fw);
2934
2935	ret = __intel_wait_for_register_fw(uncore,
2936	reg, mask, value,
2937	fast_timeout_us, slow_timeout_ms: 0, out_value: &reg_value);
2938
2939	intel_uncore_forcewake_put__locked(uncore, fw_domains: fw);
2940	spin_unlock_irq(lock: &uncore->lock);
2941
2942	if (ret && slow_timeout_ms)
2943	ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,
2944	reg),
2945	(reg_value & mask) == value,
2946	slow_timeout_ms * 1000, 10, 1000);
2947
2948	/* just trace the final value */
2949	trace_i915_reg_rw(write: false, reg, val: reg_value, len: sizeof(reg_value), trace: true);
2950
2951	if (out_value)
2952	*out_value = reg_value;
2953
2954	return ret;
2955	}
2956
2957	bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
2958	{
2959	bool ret;
2960
2961	if (!uncore->debug)
2962	return false;
2963
2964	spin_lock_irq(lock: &uncore->debug->lock);
2965	ret = check_for_unclaimed_mmio(uncore);
2966	spin_unlock_irq(lock: &uncore->debug->lock);
2967
2968	return ret;
2969	}
2970
2971	bool
2972	intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
2973	{
2974	bool ret = false;
2975
2976	if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug))
2977	return false;
2978
2979	spin_lock_irq(lock: &uncore->debug->lock);
2980
2981	if (unlikely(uncore->debug->unclaimed_mmio_check <= 0))
2982	goto out;
2983
2984	if (unlikely(check_for_unclaimed_mmio(uncore))) {
2985	if (!uncore->i915->params.mmio_debug) {
2986	drm_dbg(&uncore->i915->drm,
2987	"Unclaimed register detected, "
2988	"enabling oneshot unclaimed register reporting. "
2989	"Please use i915.mmio_debug=N for more information.\n");
2990	uncore->i915->params.mmio_debug++;
2991	}
2992	uncore->debug->unclaimed_mmio_check--;
2993	ret = true;
2994	}
2995
2996	out:
2997	spin_unlock_irq(lock: &uncore->debug->lock);
2998
2999	return ret;
3000	}
3001
3002	/**
3003	* intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
3004	* a register
3005	* @uncore: pointer to struct intel_uncore
3006	* @reg: register in question
3007	* @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
3008	*
3009	* Returns a set of forcewake domains required to be taken with for example
3010	* intel_uncore_forcewake_get for the specified register to be accessible in the
3011	* specified mode (read, write or read/write) with raw mmio accessors.
3012	*
3013	* NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
3014	* callers to do FIFO management on their own or risk losing writes.
3015	*/
3016	enum forcewake_domains
3017	intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
3018	i915_reg_t reg, unsigned int op)
3019	{
3020	enum forcewake_domains fw_domains = 0;
3021
3022	drm_WARN_ON(&uncore->i915->drm, !op);
3023
3024	if (!intel_uncore_has_forcewake(uncore))
3025	return 0;
3026
3027	if (op & FW_REG_READ)
3028	fw_domains = uncore->funcs.read_fw_domains(uncore, reg);
3029
3030	if (op & FW_REG_WRITE)
3031	fw_domains |= uncore->funcs.write_fw_domains(uncore, reg);
3032
3033	drm_WARN_ON(&uncore->i915->drm, fw_domains & ~uncore->fw_domains);
3034
3035	return fw_domains;
3036	}
3037
3038	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3039	#include "selftests/mock_uncore.c"
3040	#include "selftests/intel_uncore.c"
3041	#endif
3042