1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2014-2018 Intel Corporation
4	*/
5
6	#include "i915_drv.h"
7	#include "i915_reg.h"
8	#include "intel_context.h"
9	#include "intel_engine_pm.h"
10	#include "intel_engine_regs.h"
11	#include "intel_gpu_commands.h"
12	#include "intel_gt.h"
13	#include "intel_gt_ccs_mode.h"
14	#include "intel_gt_mcr.h"
15	#include "intel_gt_print.h"
16	#include "intel_gt_regs.h"
17	#include "intel_ring.h"
18	#include "intel_workarounds.h"
19
20	/**
21	* DOC: Hardware workarounds
22	*
23	* Hardware workarounds are register programming documented to be executed in
24	* the driver that fall outside of the normal programming sequences for a
25	* platform. There are some basic categories of workarounds, depending on
26	* how/when they are applied:
27	*
28	* - Context workarounds: workarounds that touch registers that are
29	* saved/restored to/from the HW context image. The list is emitted (via Load
30	* Register Immediate commands) once when initializing the device and saved in
31	* the default context. That default context is then used on every context
32	* creation to have a "primed golden context", i.e. a context image that
33	* already contains the changes needed to all the registers.
34	*
35	* Context workarounds should be implemented in the \*_ctx_workarounds_init()
36	* variants respective to the targeted platforms.
37	*
38	* - Engine workarounds: the list of these WAs is applied whenever the specific
39	* engine is reset. It's also possible that a set of engine classes share a
40	* common power domain and they are reset together. This happens on some
41	* platforms with render and compute engines. In this case (at least) one of
42	* them need to keeep the workaround programming: the approach taken in the
43	* driver is to tie those workarounds to the first compute/render engine that
44	* is registered. When executing with GuC submission, engine resets are
45	* outside of kernel driver control, hence the list of registers involved in
46	* written once, on engine initialization, and then passed to GuC, that
47	* saves/restores their values before/after the reset takes place. See
48	* ``drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c`` for reference.
49	*
50	* Workarounds for registers specific to RCS and CCS should be implemented in
51	* rcs_engine_wa_init() and ccs_engine_wa_init(), respectively; those for
52	* registers belonging to BCS, VCS or VECS should be implemented in
53	* xcs_engine_wa_init(). Workarounds for registers not belonging to a specific
54	* engine's MMIO range but that are part of of the common RCS/CCS reset domain
55	* should be implemented in general_render_compute_wa_init(). The settings
56	* about the CCS load balancing should be added in ccs_engine_wa_mode().
57	*
58	* - GT workarounds: the list of these WAs is applied whenever these registers
59	* revert to their default values: on GPU reset, suspend/resume [1]_, etc.
60	*
61	* GT workarounds should be implemented in the \*_gt_workarounds_init()
62	* variants respective to the targeted platforms.
63	*
64	* - Register whitelist: some workarounds need to be implemented in userspace,
65	* but need to touch privileged registers. The whitelist in the kernel
66	* instructs the hardware to allow the access to happen. From the kernel side,
67	* this is just a special case of a MMIO workaround (as we write the list of
68	* these to/be-whitelisted registers to some special HW registers).
69	*
70	* Register whitelisting should be done in the \*_whitelist_build() variants
71	* respective to the targeted platforms.
72	*
73	* - Workaround batchbuffers: buffers that get executed automatically by the
74	* hardware on every HW context restore. These buffers are created and
75	* programmed in the default context so the hardware always go through those
76	* programming sequences when switching contexts. The support for workaround
77	* batchbuffers is enabled these hardware mechanisms:
78	*
79	* #. INDIRECT_CTX: A batchbuffer and an offset are provided in the default
80	* context, pointing the hardware to jump to that location when that offset
81	* is reached in the context restore. Workaround batchbuffer in the driver
82	* currently uses this mechanism for all platforms.
83	*
84	* #. BB_PER_CTX_PTR: A batchbuffer is provided in the default context,
85	* pointing the hardware to a buffer to continue executing after the
86	* engine registers are restored in a context restore sequence. This is
87	* currently not used in the driver.
88	*
89	* - Other: There are WAs that, due to their nature, cannot be applied from a
90	* central place. Those are peppered around the rest of the code, as needed.
91	* Workarounds related to the display IP are the main example.
92	*
93	* .. [1] Technically, some registers are powercontext saved & restored, so they
94	* survive a suspend/resume. In practice, writing them again is not too
95	* costly and simplifies things, so it's the approach taken in the driver.
96	*/
97
98	static void wa_init_start(struct i915_wa_list *wal, struct intel_gt *gt,
99	const char *name, const char *engine_name)
100	{
101	wal->gt = gt;
102	wal->name = name;
103	wal->engine_name = engine_name;
104	}
105
106	#define WA_LIST_CHUNK (1 << 4)
107
108	static void wa_init_finish(struct i915_wa_list *wal)
109	{
110	/* Trim unused entries. */
111	if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
112	struct i915_wa *list = kmemdup(p: wal->list,
113	size: wal->count * sizeof(*list),
114	GFP_KERNEL);
115
116	if (list) {
117	kfree(objp: wal->list);
118	wal->list = list;
119	}
120	}
121
122	if (!wal->count)
123	return;
124
125	gt_dbg(wal->gt, "Initialized %u %s workarounds on %s\n",
126	wal->wa_count, wal->name, wal->engine_name);
127	}
128
129	static enum forcewake_domains
130	wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
131	{
132	enum forcewake_domains fw = 0;
133	struct i915_wa *wa;
134	unsigned int i;
135
136	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
137	fw |= intel_uncore_forcewake_for_reg(uncore,
138	reg: wa->reg,
139	FW_REG_READ |
140	FW_REG_WRITE);
141
142	return fw;
143	}
144
145	static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
146	{
147	unsigned int addr = i915_mmio_reg_offset(wa->reg);
148	struct drm_i915_private *i915 = wal->gt->i915;
149	unsigned int start = 0, end = wal->count;
150	const unsigned int grow = WA_LIST_CHUNK;
151	struct i915_wa *wa_;
152
153	GEM_BUG_ON(!is_power_of_2(grow));
154
155	if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
156	struct i915_wa *list;
157
158	list = kmalloc_array(ALIGN(wal->count + 1, grow), size: sizeof(*wa),
159	GFP_KERNEL);
160	if (!list) {
161	drm_err(&i915->drm, "No space for workaround init!\n");
162	return;
163	}
164
165	if (wal->list) {
166	memcpy(list, wal->list, sizeof(*wa) * wal->count);
167	kfree(objp: wal->list);
168	}
169
170	wal->list = list;
171	}
172
173	while (start < end) {
174	unsigned int mid = start + (end - start) / 2;
175
176	if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
177	start = mid + 1;
178	} else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
179	end = mid;
180	} else {
181	wa_ = &wal->list[mid];
182
183	if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) {
184	drm_err(&i915->drm,
185	"Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n",
186	i915_mmio_reg_offset(wa_->reg),
187	wa_->clr, wa_->set);
188
189	wa_->set &= ~wa->clr;
190	}
191
192	wal->wa_count++;
193	wa_->set |= wa->set;
194	wa_->clr |= wa->clr;
195	wa_->read |= wa->read;
196	return;
197	}
198	}
199
200	wal->wa_count++;
201	wa_ = &wal->list[wal->count++];
202	*wa_ = *wa;
203
204	while (wa_-- > wal->list) {
205	GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
206	i915_mmio_reg_offset(wa_[1].reg));
207	if (i915_mmio_reg_offset(wa_[1].reg) >
208	i915_mmio_reg_offset(wa_[0].reg))
209	break;
210
211	swap(wa_[1], wa_[0]);
212	}
213	}
214
215	static void wa_add(struct i915_wa_list *wal, i915_reg_t reg,
216	u32 clear, u32 set, u32 read_mask, bool masked_reg)
217	{
218	struct i915_wa wa = {
219	.reg = reg,
220	.clr = clear,
221	.set = set,
222	.read = read_mask,
223	.masked_reg = masked_reg,
224	};
225
226	_wa_add(wal, wa: &wa);
227	}
228
229	static void wa_mcr_add(struct i915_wa_list *wal, i915_mcr_reg_t reg,
230	u32 clear, u32 set, u32 read_mask, bool masked_reg)
231	{
232	struct i915_wa wa = {
233	.mcr_reg = reg,
234	.clr = clear,
235	.set = set,
236	.read = read_mask,
237	.masked_reg = masked_reg,
238	.is_mcr = 1,
239	};
240
241	_wa_add(wal, wa: &wa);
242	}
243
244	static void
245	wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set)
246	{
247	wa_add(wal, reg, clear, set, read_mask: clear | set, masked_reg: false);
248	}
249
250	static void
251	wa_mcr_write_clr_set(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clear, u32 set)
252	{
253	wa_mcr_add(wal, reg, clear, set, read_mask: clear | set, masked_reg: false);
254	}
255
256	static void
257	wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
258	{
259	wa_write_clr_set(wal, reg, clear: ~0, set);
260	}
261
262	static void
263	wa_mcr_write(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set)
264	{
265	wa_mcr_write_clr_set(wal, reg, clear: ~0, set);
266	}
267
268	static void
269	wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
270	{
271	wa_write_clr_set(wal, reg, clear: set, set);
272	}
273
274	static void
275	wa_mcr_write_or(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 set)
276	{
277	wa_mcr_write_clr_set(wal, reg, clear: set, set);
278	}
279
280	static void
281	wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
282	{
283	wa_write_clr_set(wal, reg, clear: clr, set: 0);
284	}
285
286	static void
287	wa_mcr_write_clr(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 clr)
288	{
289	wa_mcr_write_clr_set(wal, reg, clear: clr, set: 0);
290	}
291
292	/*
293	* WA operations on "masked register". A masked register has the upper 16 bits
294	* documented as "masked" in b-spec. Its purpose is to allow writing to just a
295	* portion of the register without a rmw: you simply write in the upper 16 bits
296	* the mask of bits you are going to modify.
297	*
298	* The wa_masked_* family of functions already does the necessary operations to
299	* calculate the mask based on the parameters passed, so user only has to
300	* provide the lower 16 bits of that register.
301	*/
302
303	static void
304	wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
305	{
306	wa_add(wal, reg, clear: 0, _MASKED_BIT_ENABLE(val), read_mask: val, masked_reg: true);
307	}
308
309	static void
310	wa_mcr_masked_en(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
311	{
312	wa_mcr_add(wal, reg, clear: 0, _MASKED_BIT_ENABLE(val), read_mask: val, masked_reg: true);
313	}
314
315	static void
316	wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
317	{
318	wa_add(wal, reg, clear: 0, _MASKED_BIT_DISABLE(val), read_mask: val, masked_reg: true);
319	}
320
321	static void
322	wa_mcr_masked_dis(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 val)
323	{
324	wa_mcr_add(wal, reg, clear: 0, _MASKED_BIT_DISABLE(val), read_mask: val, masked_reg: true);
325	}
326
327	static void
328	wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg,
329	u32 mask, u32 val)
330	{
331	wa_add(wal, reg, clear: 0, _MASKED_FIELD(mask, val), read_mask: mask, masked_reg: true);
332	}
333
334	static void
335	wa_mcr_masked_field_set(struct i915_wa_list *wal, i915_mcr_reg_t reg,
336	u32 mask, u32 val)
337	{
338	wa_mcr_add(wal, reg, clear: 0, _MASKED_FIELD(mask, val), read_mask: mask, masked_reg: true);
339	}
340
341	static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine,
342	struct i915_wa_list *wal)
343	{
344	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
345	}
346
347	static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine,
348	struct i915_wa_list *wal)
349	{
350	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
351	}
352
353	static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
354	struct i915_wa_list *wal)
355	{
356	wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING);
357
358	/* WaDisableAsyncFlipPerfMode:bdw,chv */
359	wa_masked_en(wal, RING_MI_MODE(RENDER_RING_BASE), ASYNC_FLIP_PERF_DISABLE);
360
361	/* WaDisablePartialInstShootdown:bdw,chv */
362	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
363	PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
364
365	/* Use Force Non-Coherent whenever executing a 3D context. This is a
366	* workaround for a possible hang in the unlikely event a TLB
367	* invalidation occurs during a PSD flush.
368	*/
369	/* WaForceEnableNonCoherent:bdw,chv */
370	/* WaHdcDisableFetchWhenMasked:bdw,chv */
371	wa_masked_en(wal, HDC_CHICKEN0,
372	HDC_DONOT_FETCH_MEM_WHEN_MASKED |
373	HDC_FORCE_NON_COHERENT);
374
375	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
376	* "The Hierarchical Z RAW Stall Optimization allows non-overlapping
377	* polygons in the same 8x4 pixel/sample area to be processed without
378	* stalling waiting for the earlier ones to write to Hierarchical Z
379	* buffer."
380	*
381	* This optimization is off by default for BDW and CHV; turn it on.
382	*/
383	wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
384
385	/* Wa4x4STCOptimizationDisable:bdw,chv */
386	wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
387
388	/*
389	* BSpec recommends 8x4 when MSAA is used,
390	* however in practice 16x4 seems fastest.
391	*
392	* Note that PS/WM thread counts depend on the WIZ hashing
393	* disable bit, which we don't touch here, but it's good
394	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
395	*/
396	wa_masked_field_set(wal, GEN7_GT_MODE,
397	GEN6_WIZ_HASHING_MASK,
398	GEN6_WIZ_HASHING_16x4);
399	}
400
401	static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
402	struct i915_wa_list *wal)
403	{
404	struct drm_i915_private *i915 = engine->i915;
405
406	gen8_ctx_workarounds_init(engine, wal);
407
408	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
409	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
410
411	/* WaDisableDopClockGating:bdw
412	*
413	* Also see the related UCGTCL1 write in bdw_init_clock_gating()
414	* to disable EUTC clock gating.
415	*/
416	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
417	DOP_CLOCK_GATING_DISABLE);
418
419	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN3,
420	GEN8_SAMPLER_POWER_BYPASS_DIS);
421
422	wa_masked_en(wal, HDC_CHICKEN0,
423	/* WaForceContextSaveRestoreNonCoherent:bdw */
424	HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
425	/* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
426	(IS_BROADWELL_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
427	}
428
429	static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
430	struct i915_wa_list *wal)
431	{
432	gen8_ctx_workarounds_init(engine, wal);
433
434	/* WaDisableThreadStallDopClockGating:chv */
435	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
436
437	/* Improve HiZ throughput on CHV. */
438	wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
439	}
440
441	static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
442	struct i915_wa_list *wal)
443	{
444	struct drm_i915_private *i915 = engine->i915;
445
446	if (HAS_LLC(i915)) {
447	/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
448	*
449	* Must match Display Engine. See
450	* WaCompressedResourceDisplayNewHashMode.
451	*/
452	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
453	GEN9_PBE_COMPRESSED_HASH_SELECTION);
454	wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
455	GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
456	}
457
458	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
459	/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
460	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
461	FLOW_CONTROL_ENABLE |
462	PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
463
464	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
465	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
466	wa_mcr_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7,
467	GEN9_ENABLE_YV12_BUGFIX |
468	GEN9_ENABLE_GPGPU_PREEMPTION);
469
470	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
471	/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
472	wa_masked_en(wal, CACHE_MODE_1,
473	GEN8_4x4_STC_OPTIMIZATION_DISABLE |
474	GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
475
476	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
477	wa_mcr_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5,
478	GEN9_CCS_TLB_PREFETCH_ENABLE);
479
480	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
481	wa_masked_en(wal, HDC_CHICKEN0,
482	HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
483	HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
484
485	/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
486	* both tied to WaForceContextSaveRestoreNonCoherent
487	* in some hsds for skl. We keep the tie for all gen9. The
488	* documentation is a bit hazy and so we want to get common behaviour,
489	* even though there is no clear evidence we would need both on kbl/bxt.
490	* This area has been source of system hangs so we play it safe
491	* and mimic the skl regardless of what bspec says.
492	*
493	* Use Force Non-Coherent whenever executing a 3D context. This
494	* is a workaround for a possible hang in the unlikely event
495	* a TLB invalidation occurs during a PSD flush.
496	*/
497
498	/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
499	wa_masked_en(wal, HDC_CHICKEN0,
500	HDC_FORCE_NON_COHERENT);
501
502	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
503	if (IS_SKYLAKE(i915) ||
504	IS_KABYLAKE(i915) ||
505	IS_COFFEELAKE(i915) ||
506	IS_COMETLAKE(i915))
507	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN3,
508	GEN8_SAMPLER_POWER_BYPASS_DIS);
509
510	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
511	wa_mcr_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
512
513	/*
514	* Supporting preemption with fine-granularity requires changes in the
515	* batch buffer programming. Since we can't break old userspace, we
516	* need to set our default preemption level to safe value. Userspace is
517	* still able to use more fine-grained preemption levels, since in
518	* WaEnablePreemptionGranularityControlByUMD we're whitelisting the
519	* per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
520	* not real HW workarounds, but merely a way to start using preemption
521	* while maintaining old contract with userspace.
522	*/
523
524	/* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
525	wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
526
527	/* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
528	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
529	GEN9_PREEMPT_GPGPU_LEVEL_MASK,
530	GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
531
532	/* WaClearHIZ_WM_CHICKEN3:bxt,glk */
533	if (IS_GEN9_LP(i915))
534	wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
535	}
536
537	static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
538	struct i915_wa_list *wal)
539	{
540	struct intel_gt *gt = engine->gt;
541	u8 vals[3] = { 0, 0, 0 };
542	unsigned int i;
543
544	for (i = 0; i < 3; i++) {
545	u8 ss;
546
547	/*
548	* Only consider slices where one, and only one, subslice has 7
549	* EUs
550	*/
551	if (!is_power_of_2(n: gt->info.sseu.subslice_7eu[i]))
552	continue;
553
554	/*
555	* subslice_7eu[i] != 0 (because of the check above) and
556	* ss_max == 4 (maximum number of subslices possible per slice)
557	*
558	* -> 0 <= ss <= 3;
559	*/
560	ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1;
561	vals[i] = 3 - ss;
562	}
563
564	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
565	return;
566
567	/* Tune IZ hashing. See intel_device_info_runtime_init() */
568	wa_masked_field_set(wal, GEN7_GT_MODE,
569	GEN9_IZ_HASHING_MASK(2) |
570	GEN9_IZ_HASHING_MASK(1) |
571	GEN9_IZ_HASHING_MASK(0),
572	GEN9_IZ_HASHING(2, vals[2]) |
573	GEN9_IZ_HASHING(1, vals[1]) |
574	GEN9_IZ_HASHING(0, vals[0]));
575	}
576
577	static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
578	struct i915_wa_list *wal)
579	{
580	gen9_ctx_workarounds_init(engine, wal);
581	skl_tune_iz_hashing(engine, wal);
582	}
583
584	static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
585	struct i915_wa_list *wal)
586	{
587	gen9_ctx_workarounds_init(engine, wal);
588
589	/* WaDisableThreadStallDopClockGating:bxt */
590	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN,
591	STALL_DOP_GATING_DISABLE);
592
593	/* WaToEnableHwFixForPushConstHWBug:bxt */
594	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
595	GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
596	}
597
598	static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
599	struct i915_wa_list *wal)
600	{
601	struct drm_i915_private *i915 = engine->i915;
602
603	gen9_ctx_workarounds_init(engine, wal);
604
605	/* WaToEnableHwFixForPushConstHWBug:kbl */
606	if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, STEP_C0, STEP_FOREVER))
607	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
608	GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
609
610	/* WaDisableSbeCacheDispatchPortSharing:kbl */
611	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
612	GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
613	}
614
615	static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
616	struct i915_wa_list *wal)
617	{
618	gen9_ctx_workarounds_init(engine, wal);
619
620	/* WaToEnableHwFixForPushConstHWBug:glk */
621	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
622	GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
623	}
624
625	static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
626	struct i915_wa_list *wal)
627	{
628	gen9_ctx_workarounds_init(engine, wal);
629
630	/* WaToEnableHwFixForPushConstHWBug:cfl */
631	wa_masked_en(wal, COMMON_SLICE_CHICKEN2,
632	GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
633
634	/* WaDisableSbeCacheDispatchPortSharing:cfl */
635	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
636	GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
637	}
638
639	static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
640	struct i915_wa_list *wal)
641	{
642	/* Wa_1406697149 (WaDisableBankHangMode:icl) */
643	wa_write(wal, GEN8_L3CNTLREG, GEN8_ERRDETBCTRL);
644
645	/* WaForceEnableNonCoherent:icl
646	* This is not the same workaround as in early Gen9 platforms, where
647	* lacking this could cause system hangs, but coherency performance
648	* overhead is high and only a few compute workloads really need it
649	* (the register is whitelisted in hardware now, so UMDs can opt in
650	* for coherency if they have a good reason).
651	*/
652	wa_mcr_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
653
654	/* WaEnableFloatBlendOptimization:icl */
655	wa_mcr_add(wal, GEN10_CACHE_MODE_SS, clear: 0,
656	_MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE),
657	read_mask: 0 /* write-only, so skip validation */,
658	masked_reg: true);
659
660	/* WaDisableGPGPUMidThreadPreemption:icl */
661	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
662	GEN9_PREEMPT_GPGPU_LEVEL_MASK,
663	GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
664
665	/* allow headerless messages for preemptible GPGPU context */
666	wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
667	GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
668
669	/* Wa_1604278689:icl,ehl */
670	wa_write(wal, IVB_FBC_RT_BASE, set: 0xFFFFFFFF & ~ILK_FBC_RT_VALID);
671	wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER,
672	clear: 0,
673	set: 0xFFFFFFFF);
674
675	/* Wa_1406306137:icl,ehl */
676	wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU);
677	}
678
679	/*
680	* These settings aren't actually workarounds, but general tuning settings that
681	* need to be programmed on dg2 platform.
682	*/
683	static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine,
684	struct i915_wa_list *wal)
685	{
686	wa_mcr_masked_en(wal, CHICKEN_RASTER_2, TBIMR_FAST_CLIP);
687	wa_mcr_write_clr_set(wal, XEHP_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
688	REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f));
689	wa_mcr_write_clr_set(wal, XEHP_FF_MODE2, FF_MODE2_TDS_TIMER_MASK,
690	FF_MODE2_TDS_TIMER_128);
691	}
692
693	static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine,
694	struct i915_wa_list *wal)
695	{
696	struct drm_i915_private *i915 = engine->i915;
697
698	/*
699	* Wa_1409142259:tgl,dg1,adl-p
700	* Wa_1409347922:tgl,dg1,adl-p
701	* Wa_1409252684:tgl,dg1,adl-p
702	* Wa_1409217633:tgl,dg1,adl-p
703	* Wa_1409207793:tgl,dg1,adl-p
704	* Wa_1409178076:tgl,dg1,adl-p
705	* Wa_1408979724:tgl,dg1,adl-p
706	* Wa_14010443199:tgl,rkl,dg1,adl-p
707	* Wa_14010698770:tgl,rkl,dg1,adl-s,adl-p
708	* Wa_1409342910:tgl,rkl,dg1,adl-s,adl-p
709	*/
710	wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3,
711	GEN12_DISABLE_CPS_AWARE_COLOR_PIPE);
712
713	/* WaDisableGPGPUMidThreadPreemption:gen12 */
714	wa_masked_field_set(wal, GEN8_CS_CHICKEN1,
715	GEN9_PREEMPT_GPGPU_LEVEL_MASK,
716	GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
717
718	/*
719	* Wa_16011163337 - GS_TIMER
720	*
721	* TDS_TIMER: Although some platforms refer to it as Wa_1604555607, we
722	* need to program it even on those that don't explicitly list that
723	* workaround.
724	*
725	* Note that the programming of GEN12_FF_MODE2 is further modified
726	* according to the FF_MODE2 guidance given by Wa_1608008084.
727	* Wa_1608008084 tells us the FF_MODE2 register will return the wrong
728	* value when read from the CPU.
729	*
730	* The default value for this register is zero for all fields.
731	* So instead of doing a RMW we should just write the desired values
732	* for TDS and GS timers. Note that since the readback can't be trusted,
733	* the clear mask is just set to ~0 to make sure other bits are not
734	* inadvertently set. For the same reason read verification is ignored.
735	*/
736	wa_add(wal,
737	GEN12_FF_MODE2,
738	clear: ~0,
739	FF_MODE2_TDS_TIMER_128 | FF_MODE2_GS_TIMER_224,
740	read_mask: 0, masked_reg: false);
741
742	if (!IS_DG1(i915)) {
743	/* Wa_1806527549 */
744	wa_masked_en(wal, HIZ_CHICKEN, HZ_DEPTH_TEST_LE_GE_OPT_DISABLE);
745
746	/* Wa_1606376872 */
747	wa_masked_en(wal, COMMON_SLICE_CHICKEN4, DISABLE_TDC_LOAD_BALANCING_CALC);
748	}
749	}
750
751	static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine,
752	struct i915_wa_list *wal)
753	{
754	gen12_ctx_workarounds_init(engine, wal);
755
756	/* Wa_1409044764 */
757	wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3,
758	DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN);
759
760	/* Wa_22010493298 */
761	wa_masked_en(wal, HIZ_CHICKEN,
762	DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE);
763	}
764
765	static void dg2_ctx_workarounds_init(struct intel_engine_cs *engine,
766	struct i915_wa_list *wal)
767	{
768	dg2_ctx_gt_tuning_init(engine, wal);
769
770	/* Wa_16013271637:dg2 */
771	wa_mcr_masked_en(wal, XEHP_SLICE_COMMON_ECO_CHICKEN1,
772	MSC_MSAA_REODER_BUF_BYPASS_DISABLE);
773
774	/* Wa_14014947963:dg2 */
775	wa_masked_field_set(wal, VF_PREEMPTION, PREEMPTION_VERTEX_COUNT, val: 0x4000);
776
777	/* Wa_18018764978:dg2 */
778	wa_mcr_masked_en(wal, XEHP_PSS_MODE2, SCOREBOARD_STALL_FLUSH_CONTROL);
779
780	/* Wa_18019271663:dg2 */
781	wa_masked_en(wal, CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE);
782
783	/* Wa_14019877138:dg2 */
784	wa_mcr_masked_en(wal, XEHP_PSS_CHICKEN, FD_END_COLLECT);
785	}
786
787	static void xelpg_ctx_gt_tuning_init(struct intel_engine_cs *engine,
788	struct i915_wa_list *wal)
789	{
790	struct intel_gt *gt = engine->gt;
791
792	dg2_ctx_gt_tuning_init(engine, wal);
793
794	/*
795	* Due to Wa_16014892111, the DRAW_WATERMARK tuning must be done in
796	* gen12_emit_indirect_ctx_rcs() rather than here on some early
797	* steppings.
798	*/
799	if (!(IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
800	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)))
801	wa_add(wal, DRAW_WATERMARK, VERT_WM_VAL, set: 0x3FF, read_mask: 0, masked_reg: false);
802	}
803
804	static void xelpg_ctx_workarounds_init(struct intel_engine_cs *engine,
805	struct i915_wa_list *wal)
806	{
807	struct intel_gt *gt = engine->gt;
808
809	xelpg_ctx_gt_tuning_init(engine, wal);
810
811	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
812	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)) {
813	/* Wa_14014947963 */
814	wa_masked_field_set(wal, VF_PREEMPTION,
815	PREEMPTION_VERTEX_COUNT, val: 0x4000);
816
817	/* Wa_16013271637 */
818	wa_mcr_masked_en(wal, XEHP_SLICE_COMMON_ECO_CHICKEN1,
819	MSC_MSAA_REODER_BUF_BYPASS_DISABLE);
820
821	/* Wa_18019627453 */
822	wa_mcr_masked_en(wal, VFLSKPD, VF_PREFETCH_TLB_DIS);
823
824	/* Wa_18018764978 */
825	wa_mcr_masked_en(wal, XEHP_PSS_MODE2, SCOREBOARD_STALL_FLUSH_CONTROL);
826	}
827
828	/* Wa_18019271663 */
829	wa_masked_en(wal, CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE);
830
831	/* Wa_14019877138 */
832	wa_mcr_masked_en(wal, XEHP_PSS_CHICKEN, FD_END_COLLECT);
833	}
834
835	static void fakewa_disable_nestedbb_mode(struct intel_engine_cs *engine,
836	struct i915_wa_list *wal)
837	{
838	/*
839	* This is a "fake" workaround defined by software to ensure we
840	* maintain reliable, backward-compatible behavior for userspace with
841	* regards to how nested MI_BATCH_BUFFER_START commands are handled.
842	*
843	* The per-context setting of MI_MODE[12] determines whether the bits
844	* of a nested MI_BATCH_BUFFER_START instruction should be interpreted
845	* in the traditional manner or whether they should instead use a new
846	* tgl+ meaning that breaks backward compatibility, but allows nesting
847	* into 3rd-level batchbuffers. When this new capability was first
848	* added in TGL, it remained off by default unless a context
849	* intentionally opted in to the new behavior. However Xe_HPG now
850	* flips this on by default and requires that we explicitly opt out if
851	* we don't want the new behavior.
852	*
853	* From a SW perspective, we want to maintain the backward-compatible
854	* behavior for userspace, so we'll apply a fake workaround to set it
855	* back to the legacy behavior on platforms where the hardware default
856	* is to break compatibility. At the moment there is no Linux
857	* userspace that utilizes third-level batchbuffers, so this will avoid
858	* userspace from needing to make any changes. using the legacy
859	* meaning is the correct thing to do. If/when we have userspace
860	* consumers that want to utilize third-level batch nesting, we can
861	* provide a context parameter to allow them to opt-in.
862	*/
863	wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN);
864	}
865
866	static void gen12_ctx_gt_mocs_init(struct intel_engine_cs *engine,
867	struct i915_wa_list *wal)
868	{
869	u8 mocs;
870
871	/*
872	* Some blitter commands do not have a field for MOCS, those
873	* commands will use MOCS index pointed by BLIT_CCTL.
874	* BLIT_CCTL registers are needed to be programmed to un-cached.
875	*/
876	if (engine->class == COPY_ENGINE_CLASS) {
877	mocs = engine->gt->mocs.uc_index;
878	wa_write_clr_set(wal,
879	BLIT_CCTL(engine->mmio_base),
880	BLIT_CCTL_MASK,
881	BLIT_CCTL_MOCS(mocs, mocs));
882	}
883	}
884
885	/*
886	* gen12_ctx_gt_fake_wa_init() aren't programmingan official workaround
887	* defined by the hardware team, but it programming general context registers.
888	* Adding those context register programming in context workaround
889	* allow us to use the wa framework for proper application and validation.
890	*/
891	static void
892	gen12_ctx_gt_fake_wa_init(struct intel_engine_cs *engine,
893	struct i915_wa_list *wal)
894	{
895	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55))
896	fakewa_disable_nestedbb_mode(engine, wal);
897
898	gen12_ctx_gt_mocs_init(engine, wal);
899	}
900
901	static void
902	__intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
903	struct i915_wa_list *wal,
904	const char *name)
905	{
906	struct drm_i915_private *i915 = engine->i915;
907
908	wa_init_start(wal, gt: engine->gt, name, engine_name: engine->name);
909
910	/* Applies to all engines */
911	/*
912	* Fake workarounds are not the actual workaround but
913	* programming of context registers using workaround framework.
914	*/
915	if (GRAPHICS_VER(i915) >= 12)
916	gen12_ctx_gt_fake_wa_init(engine, wal);
917
918	if (engine->class != RENDER_CLASS)
919	goto done;
920
921	if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))
922	xelpg_ctx_workarounds_init(engine, wal);
923	else if (IS_PONTEVECCHIO(i915))
924	; /* noop; none at this time */
925	else if (IS_DG2(i915))
926	dg2_ctx_workarounds_init(engine, wal);
927	else if (IS_XEHPSDV(i915))
928	; /* noop; none at this time */
929	else if (IS_DG1(i915))
930	dg1_ctx_workarounds_init(engine, wal);
931	else if (GRAPHICS_VER(i915) == 12)
932	gen12_ctx_workarounds_init(engine, wal);
933	else if (GRAPHICS_VER(i915) == 11)
934	icl_ctx_workarounds_init(engine, wal);
935	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
936	cfl_ctx_workarounds_init(engine, wal);
937	else if (IS_GEMINILAKE(i915))
938	glk_ctx_workarounds_init(engine, wal);
939	else if (IS_KABYLAKE(i915))
940	kbl_ctx_workarounds_init(engine, wal);
941	else if (IS_BROXTON(i915))
942	bxt_ctx_workarounds_init(engine, wal);
943	else if (IS_SKYLAKE(i915))
944	skl_ctx_workarounds_init(engine, wal);
945	else if (IS_CHERRYVIEW(i915))
946	chv_ctx_workarounds_init(engine, wal);
947	else if (IS_BROADWELL(i915))
948	bdw_ctx_workarounds_init(engine, wal);
949	else if (GRAPHICS_VER(i915) == 7)
950	gen7_ctx_workarounds_init(engine, wal);
951	else if (GRAPHICS_VER(i915) == 6)
952	gen6_ctx_workarounds_init(engine, wal);
953	else if (GRAPHICS_VER(i915) < 8)
954	;
955	else
956	MISSING_CASE(GRAPHICS_VER(i915));
957
958	done:
959	wa_init_finish(wal);
960	}
961
962	void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
963	{
964	__intel_engine_init_ctx_wa(engine, wal: &engine->ctx_wa_list, name: "context");
965	}
966
967	int intel_engine_emit_ctx_wa(struct i915_request *rq)
968	{
969	struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
970	struct intel_uncore *uncore = rq->engine->uncore;
971	enum forcewake_domains fw;
972	unsigned long flags;
973	struct i915_wa *wa;
974	unsigned int i;
975	u32 *cs;
976	int ret;
977
978	if (wal->count == 0)
979	return 0;
980
981	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
982	if (ret)
983	return ret;
984
985	cs = intel_ring_begin(rq, num_dwords: (wal->count * 2 + 2));
986	if (IS_ERR(ptr: cs))
987	return PTR_ERR(ptr: cs);
988
989	fw = wal_get_fw_for_rmw(uncore, wal);
990
991	intel_gt_mcr_lock(gt: wal->gt, flags: &flags);
992	spin_lock(lock: &uncore->lock);
993	intel_uncore_forcewake_get__locked(uncore, domains: fw);
994
995	*cs++ = MI_LOAD_REGISTER_IMM(wal->count);
996	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
997	u32 val;
998
999	/* Skip reading the register if it's not really needed */
1000	if (wa->masked_reg || (wa->clr | wa->set) == U32_MAX) {
1001	val = wa->set;
1002	} else {
1003	val = wa->is_mcr ?
1004	intel_gt_mcr_read_any_fw(gt: wal->gt, reg: wa->mcr_reg) :
1005	intel_uncore_read_fw(uncore, wa->reg);
1006	val &= ~wa->clr;
1007	val |= wa->set;
1008	}
1009
1010	*cs++ = i915_mmio_reg_offset(wa->reg);
1011	*cs++ = val;
1012	}
1013	*cs++ = MI_NOOP;
1014
1015	intel_uncore_forcewake_put__locked(uncore, domains: fw);
1016	spin_unlock(lock: &uncore->lock);
1017	intel_gt_mcr_unlock(gt: wal->gt, flags);
1018
1019	intel_ring_advance(rq, cs);
1020
1021	ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
1022	if (ret)
1023	return ret;
1024
1025	return 0;
1026	}
1027
1028	static void
1029	gen4_gt_workarounds_init(struct intel_gt *gt,
1030	struct i915_wa_list *wal)
1031	{
1032	/* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
1033	wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
1034	}
1035
1036	static void
1037	g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1038	{
1039	gen4_gt_workarounds_init(gt, wal);
1040
1041	/* WaDisableRenderCachePipelinedFlush:g4x,ilk */
1042	wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
1043	}
1044
1045	static void
1046	ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1047	{
1048	g4x_gt_workarounds_init(gt, wal);
1049
1050	wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
1051	}
1052
1053	static void
1054	snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1055	{
1056	}
1057
1058	static void
1059	ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1060	{
1061	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
1062	wa_masked_dis(wal,
1063	GEN7_COMMON_SLICE_CHICKEN1,
1064	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
1065
1066	/* WaApplyL3ControlAndL3ChickenMode:ivb */
1067	wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
1068	wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
1069
1070	/* WaForceL3Serialization:ivb */
1071	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
1072	}
1073
1074	static void
1075	vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1076	{
1077	/* WaForceL3Serialization:vlv */
1078	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
1079
1080	/*
1081	* WaIncreaseL3CreditsForVLVB0:vlv
1082	* This is the hardware default actually.
1083	*/
1084	wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
1085	}
1086
1087	static void
1088	hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1089	{
1090	/* L3 caching of data atomics doesn't work -- disable it. */
1091	wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
1092
1093	wa_add(wal,
1094	HSW_ROW_CHICKEN3, clear: 0,
1095	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
1096	read_mask: 0 /* XXX does this reg exist? */, masked_reg: true);
1097
1098	/* WaVSRefCountFullforceMissDisable:hsw */
1099	wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
1100	}
1101
1102	static void
1103	gen9_wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
1104	{
1105	const struct sseu_dev_info *sseu = &to_gt(i915)->info.sseu;
1106	unsigned int slice, subslice;
1107	u32 mcr, mcr_mask;
1108
1109	GEM_BUG_ON(GRAPHICS_VER(i915) != 9);
1110
1111	/*
1112	* WaProgramMgsrForCorrectSliceSpecificMmioReads:gen9,glk,kbl,cml
1113	* Before any MMIO read into slice/subslice specific registers, MCR
1114	* packet control register needs to be programmed to point to any
1115	* enabled s/ss pair. Otherwise, incorrect values will be returned.
1116	* This means each subsequent MMIO read will be forwarded to an
1117	* specific s/ss combination, but this is OK since these registers
1118	* are consistent across s/ss in almost all cases. In the rare
1119	* occasions, such as INSTDONE, where this value is dependent
1120	* on s/ss combo, the read should be done with read_subslice_reg.
1121	*/
1122	slice = ffs(sseu->slice_mask) - 1;
1123	GEM_BUG_ON(slice >= ARRAY_SIZE(sseu->subslice_mask.hsw));
1124	subslice = ffs(intel_sseu_get_hsw_subslices(sseu, slice));
1125	GEM_BUG_ON(!subslice);
1126	subslice--;
1127
1128	/*
1129	* We use GEN8_MCR..() macros to calculate the |mcr| value for
1130	* Gen9 to address WaProgramMgsrForCorrectSliceSpecificMmioReads
1131	*/
1132	mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
1133	mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK;
1134
1135	drm_dbg(&i915->drm, "MCR slice:%d/subslice:%d = %x\n", slice, subslice, mcr);
1136
1137	wa_write_clr_set(wal, GEN8_MCR_SELECTOR, clear: mcr_mask, set: mcr);
1138	}
1139
1140	static void
1141	gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1142	{
1143	struct drm_i915_private *i915 = gt->i915;
1144
1145	/* WaProgramMgsrForCorrectSliceSpecificMmioReads:glk,kbl,cml,gen9 */
1146	gen9_wa_init_mcr(i915, wal);
1147
1148	/* WaDisableKillLogic:bxt,skl,kbl */
1149	if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915))
1150	wa_write_or(wal,
1151	GAM_ECOCHK,
1152	ECOCHK_DIS_TLB);
1153
1154	if (HAS_LLC(i915)) {
1155	/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
1156	*
1157	* Must match Display Engine. See
1158	* WaCompressedResourceDisplayNewHashMode.
1159	*/
1160	wa_write_or(wal,
1161	MMCD_MISC_CTRL,
1162	MMCD_PCLA | MMCD_HOTSPOT_EN);
1163	}
1164
1165	/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1166	wa_write_or(wal,
1167	GAM_ECOCHK,
1168	BDW_DISABLE_HDC_INVALIDATION);
1169	}
1170
1171	static void
1172	skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1173	{
1174	gen9_gt_workarounds_init(gt, wal);
1175
1176	/* WaDisableGafsUnitClkGating:skl */
1177	wa_write_or(wal,
1178	GEN7_UCGCTL4,
1179	GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1180
1181	/* WaInPlaceDecompressionHang:skl */
1182	if (IS_SKYLAKE(gt->i915) && IS_GRAPHICS_STEP(gt->i915, STEP_A0, STEP_H0))
1183	wa_write_or(wal,
1184	GEN9_GAMT_ECO_REG_RW_IA,
1185	GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1186	}
1187
1188	static void
1189	kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1190	{
1191	gen9_gt_workarounds_init(gt, wal);
1192
1193	/* WaDisableDynamicCreditSharing:kbl */
1194	if (IS_KABYLAKE(gt->i915) && IS_GRAPHICS_STEP(gt->i915, 0, STEP_C0))
1195	wa_write_or(wal,
1196	GAMT_CHKN_BIT_REG,
1197	GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1198
1199	/* WaDisableGafsUnitClkGating:kbl */
1200	wa_write_or(wal,
1201	GEN7_UCGCTL4,
1202	GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1203
1204	/* WaInPlaceDecompressionHang:kbl */
1205	wa_write_or(wal,
1206	GEN9_GAMT_ECO_REG_RW_IA,
1207	GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1208	}
1209
1210	static void
1211	glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1212	{
1213	gen9_gt_workarounds_init(gt, wal);
1214	}
1215
1216	static void
1217	cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1218	{
1219	gen9_gt_workarounds_init(gt, wal);
1220
1221	/* WaDisableGafsUnitClkGating:cfl */
1222	wa_write_or(wal,
1223	GEN7_UCGCTL4,
1224	GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1225
1226	/* WaInPlaceDecompressionHang:cfl */
1227	wa_write_or(wal,
1228	GEN9_GAMT_ECO_REG_RW_IA,
1229	GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1230	}
1231
1232	static void __set_mcr_steering(struct i915_wa_list *wal,
1233	i915_reg_t steering_reg,
1234	unsigned int slice, unsigned int subslice)
1235	{
1236	u32 mcr, mcr_mask;
1237
1238	mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice);
1239	mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK;
1240
1241	wa_write_clr_set(wal, reg: steering_reg, clear: mcr_mask, set: mcr);
1242	}
1243
1244	static void debug_dump_steering(struct intel_gt *gt)
1245	{
1246	struct drm_printer p = drm_dbg_printer(drm: &gt->i915->drm, category: DRM_UT_DRIVER,
1247	prefix: "MCR Steering:");
1248
1249	if (drm_debug_enabled(DRM_UT_DRIVER))
1250	intel_gt_mcr_report_steering(p: &p, gt, dump_table: false);
1251	}
1252
1253	static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal,
1254	unsigned int slice, unsigned int subslice)
1255	{
1256	__set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice);
1257
1258	gt->default_steering.groupid = slice;
1259	gt->default_steering.instanceid = subslice;
1260
1261	debug_dump_steering(gt);
1262	}
1263
1264	static void
1265	icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1266	{
1267	const struct sseu_dev_info *sseu = &gt->info.sseu;
1268	unsigned int subslice;
1269
1270	GEM_BUG_ON(GRAPHICS_VER(gt->i915) < 11);
1271	GEM_BUG_ON(hweight8(sseu->slice_mask) > 1);
1272
1273	/*
1274	* Although a platform may have subslices, we need to always steer
1275	* reads to the lowest instance that isn't fused off. When Render
1276	* Power Gating is enabled, grabbing forcewake will only power up a
1277	* single subslice (the "minconfig") if there isn't a real workload
1278	* that needs to be run; this means that if we steer register reads to
1279	* one of the higher subslices, we run the risk of reading back 0's or
1280	* random garbage.
1281	*/
1282	subslice = __ffs(intel_sseu_get_hsw_subslices(sseu, 0));
1283
1284	/*
1285	* If the subslice we picked above also steers us to a valid L3 bank,
1286	* then we can just rely on the default steering and won't need to
1287	* worry about explicitly re-steering L3BANK reads later.
1288	*/
1289	if (gt->info.l3bank_mask & BIT(subslice))
1290	gt->steering_table[L3BANK] = NULL;
1291
1292	__add_mcr_wa(gt, wal, slice: 0, subslice);
1293	}
1294
1295	static void
1296	xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1297	{
1298	const struct sseu_dev_info *sseu = &gt->info.sseu;
1299	unsigned long slice, subslice = 0, slice_mask = 0;
1300	u32 lncf_mask = 0;
1301	int i;
1302
1303	/*
1304	* On Xe_HP the steering increases in complexity. There are now several
1305	* more units that require steering and we're not guaranteed to be able
1306	* to find a common setting for all of them. These are:
1307	* - GSLICE (fusable)
1308	* - DSS (sub-unit within gslice; fusable)
1309	* - L3 Bank (fusable)
1310	* - MSLICE (fusable)
1311	* - LNCF (sub-unit within mslice; always present if mslice is present)
1312	*
1313	* We'll do our default/implicit steering based on GSLICE (in the
1314	* sliceid field) and DSS (in the subsliceid field). If we can
1315	* find overlap between the valid MSLICE and/or LNCF values with
1316	* a suitable GSLICE, then we can just re-use the default value and
1317	* skip and explicit steering at runtime.
1318	*
1319	* We only need to look for overlap between GSLICE/MSLICE/LNCF to find
1320	* a valid sliceid value. DSS steering is the only type of steering
1321	* that utilizes the 'subsliceid' bits.
1322	*
1323	* Also note that, even though the steering domain is called "GSlice"
1324	* and it is encoded in the register using the gslice format, the spec
1325	* says that the combined (geometry | compute) fuse should be used to
1326	* select the steering.
1327	*/
1328
1329	/* Find the potential gslice candidates */
1330	slice_mask = intel_slicemask_from_xehp_dssmask(dss_mask: sseu->subslice_mask,
1331	GEN_DSS_PER_GSLICE);
1332
1333	/*
1334	* Find the potential LNCF candidates. Either LNCF within a valid
1335	* mslice is fine.
1336	*/
1337	for_each_set_bit(i, &gt->info.mslice_mask, GEN12_MAX_MSLICES)
1338	lncf_mask |= (0x3 << (i * 2));
1339
1340	/*
1341	* Are there any sliceid values that work for both GSLICE and LNCF
1342	* steering?
1343	*/
1344	if (slice_mask & lncf_mask) {
1345	slice_mask &= lncf_mask;
1346	gt->steering_table[LNCF] = NULL;
1347	}
1348
1349	/* How about sliceid values that also work for MSLICE steering? */
1350	if (slice_mask & gt->info.mslice_mask) {
1351	slice_mask &= gt->info.mslice_mask;
1352	gt->steering_table[MSLICE] = NULL;
1353	}
1354
1355	if (IS_XEHPSDV(gt->i915) && slice_mask & BIT(0))
1356	gt->steering_table[GAM] = NULL;
1357
1358	slice = __ffs(slice_mask);
1359	subslice = intel_sseu_find_first_xehp_dss(sseu, GEN_DSS_PER_GSLICE, groupnum: slice) %
1360	GEN_DSS_PER_GSLICE;
1361
1362	__add_mcr_wa(gt, wal, slice, subslice);
1363
1364	/*
1365	* SQIDI ranges are special because they use different steering
1366	* registers than everything else we work with. On XeHP SDV and
1367	* DG2-G10, any value in the steering registers will work fine since
1368	* all instances are present, but DG2-G11 only has SQIDI instances at
1369	* ID's 2 and 3, so we need to steer to one of those. For simplicity
1370	* we'll just steer to a hardcoded "2" since that value will work
1371	* everywhere.
1372	*/
1373	__set_mcr_steering(wal, MCFG_MCR_SELECTOR, slice: 0, subslice: 2);
1374	__set_mcr_steering(wal, SF_MCR_SELECTOR, slice: 0, subslice: 2);
1375
1376	/*
1377	* On DG2, GAM registers have a dedicated steering control register
1378	* and must always be programmed to a hardcoded groupid of "1."
1379	*/
1380	if (IS_DG2(gt->i915))
1381	__set_mcr_steering(wal, GAM_MCR_SELECTOR, slice: 1, subslice: 0);
1382	}
1383
1384	static void
1385	pvc_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal)
1386	{
1387	unsigned int dss;
1388
1389	/*
1390	* Setup implicit steering for COMPUTE and DSS ranges to the first
1391	* non-fused-off DSS. All other types of MCR registers will be
1392	* explicitly steered.
1393	*/
1394	dss = intel_sseu_find_first_xehp_dss(sseu: &gt->info.sseu, groupsize: 0, groupnum: 0);
1395	__add_mcr_wa(gt, wal, slice: dss / GEN_DSS_PER_CSLICE, subslice: dss % GEN_DSS_PER_CSLICE);
1396	}
1397
1398	static void
1399	icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1400	{
1401	struct drm_i915_private *i915 = gt->i915;
1402
1403	icl_wa_init_mcr(gt, wal);
1404
1405	/* WaModifyGamTlbPartitioning:icl */
1406	wa_write_clr_set(wal,
1407	GEN11_GACB_PERF_CTRL,
1408	GEN11_HASH_CTRL_MASK,
1409	GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
1410
1411	/* Wa_1405766107:icl
1412	* Formerly known as WaCL2SFHalfMaxAlloc
1413	*/
1414	wa_write_or(wal,
1415	GEN11_LSN_UNSLCVC,
1416	GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
1417	GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
1418
1419	/* Wa_220166154:icl
1420	* Formerly known as WaDisCtxReload
1421	*/
1422	wa_write_or(wal,
1423	GEN8_GAMW_ECO_DEV_RW_IA,
1424	GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
1425
1426	/* Wa_1406463099:icl
1427	* Formerly known as WaGamTlbPendError
1428	*/
1429	wa_write_or(wal,
1430	GAMT_CHKN_BIT_REG,
1431	GAMT_CHKN_DISABLE_L3_COH_PIPE);
1432
1433	/*
1434	* Wa_1408615072:icl,ehl (vsunit)
1435	* Wa_1407596294:icl,ehl (hsunit)
1436	*/
1437	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1438	VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS);
1439
1440	/* Wa_1407352427:icl,ehl */
1441	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2,
1442	PSDUNIT_CLKGATE_DIS);
1443
1444	/* Wa_1406680159:icl,ehl */
1445	wa_mcr_write_or(wal,
1446	GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE,
1447	GWUNIT_CLKGATE_DIS);
1448
1449	/* Wa_1607087056:icl,ehl,jsl */
1450	if (IS_ICELAKE(i915) ||
1451	((IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)) &&
1452	IS_GRAPHICS_STEP(i915, STEP_A0, STEP_B0)))
1453	wa_write_or(wal,
1454	GEN11_SLICE_UNIT_LEVEL_CLKGATE,
1455	L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS);
1456
1457	/*
1458	* This is not a documented workaround, but rather an optimization
1459	* to reduce sampler power.
1460	*/
1461	wa_mcr_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1462	}
1463
1464	/*
1465	* Though there are per-engine instances of these registers,
1466	* they retain their value through engine resets and should
1467	* only be provided on the GT workaround list rather than
1468	* the engine-specific workaround list.
1469	*/
1470	static void
1471	wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal)
1472	{
1473	struct intel_engine_cs *engine;
1474	int id;
1475
1476	for_each_engine(engine, gt, id) {
1477	if (engine->class != VIDEO_DECODE_CLASS ||
1478	(engine->instance % 2))
1479	continue;
1480
1481	wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base),
1482	IECPUNIT_CLKGATE_DIS);
1483	}
1484	}
1485
1486	static void
1487	gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1488	{
1489	icl_wa_init_mcr(gt, wal);
1490
1491	/* Wa_14011060649:tgl,rkl,dg1,adl-s,adl-p */
1492	wa_14011060649(gt, wal);
1493
1494	/* Wa_14011059788:tgl,rkl,adl-s,dg1,adl-p */
1495	wa_mcr_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE);
1496
1497	/*
1498	* Wa_14015795083
1499	*
1500	* Firmware on some gen12 platforms locks the MISCCPCTL register,
1501	* preventing i915 from modifying it for this workaround. Skip the
1502	* readback verification for this workaround on debug builds; if the
1503	* workaround doesn't stick due to firmware behavior, it's not an error
1504	* that we want CI to flag.
1505	*/
1506	wa_add(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE,
1507	set: 0, read_mask: 0, masked_reg: false);
1508	}
1509
1510	static void
1511	dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1512	{
1513	gen12_gt_workarounds_init(gt, wal);
1514
1515	/* Wa_1409420604:dg1 */
1516	wa_mcr_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE2,
1517	CPSSUNIT_CLKGATE_DIS);
1518
1519	/* Wa_1408615072:dg1 */
1520	/* Empirical testing shows this register is unaffected by engine reset. */
1521	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, VSUNIT_CLKGATE_DIS_TGL);
1522	}
1523
1524	static void
1525	xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1526	{
1527	struct drm_i915_private *i915 = gt->i915;
1528
1529	xehp_init_mcr(gt, wal);
1530
1531	/* Wa_1409757795:xehpsdv */
1532	wa_mcr_write_or(wal, SCCGCTL94DC, CG3DDISURB);
1533
1534	/* Wa_18011725039:xehpsdv */
1535	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_B0)) {
1536	wa_mcr_masked_dis(wal, MLTICTXCTL, TDONRENDER);
1537	wa_mcr_write_or(wal, L3SQCREG1_CCS0, FLUSHALLNONCOH);
1538	}
1539
1540	/* Wa_16011155590:xehpsdv */
1541	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
1542	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1543	TSGUNIT_CLKGATE_DIS);
1544
1545	/* Wa_14011780169:xehpsdv */
1546	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_B0, STEP_FOREVER)) {
1547	wa_write_or(wal, UNSLCGCTL9440, GAMTLBOACS_CLKGATE_DIS |
1548	GAMTLBVDBOX7_CLKGATE_DIS |
1549	GAMTLBVDBOX6_CLKGATE_DIS |
1550	GAMTLBVDBOX5_CLKGATE_DIS |
1551	GAMTLBVDBOX4_CLKGATE_DIS |
1552	GAMTLBVDBOX3_CLKGATE_DIS |
1553	GAMTLBVDBOX2_CLKGATE_DIS |
1554	GAMTLBVDBOX1_CLKGATE_DIS |
1555	GAMTLBVDBOX0_CLKGATE_DIS |
1556	GAMTLBKCR_CLKGATE_DIS |
1557	GAMTLBGUC_CLKGATE_DIS |
1558	GAMTLBBLT_CLKGATE_DIS);
1559	wa_write_or(wal, UNSLCGCTL9444, GAMTLBGFXA0_CLKGATE_DIS |
1560	GAMTLBGFXA1_CLKGATE_DIS |
1561	GAMTLBCOMPA0_CLKGATE_DIS |
1562	GAMTLBCOMPA1_CLKGATE_DIS |
1563	GAMTLBCOMPB0_CLKGATE_DIS |
1564	GAMTLBCOMPB1_CLKGATE_DIS |
1565	GAMTLBCOMPC0_CLKGATE_DIS |
1566	GAMTLBCOMPC1_CLKGATE_DIS |
1567	GAMTLBCOMPD0_CLKGATE_DIS |
1568	GAMTLBCOMPD1_CLKGATE_DIS |
1569	GAMTLBMERT_CLKGATE_DIS |
1570	GAMTLBVEBOX3_CLKGATE_DIS |
1571	GAMTLBVEBOX2_CLKGATE_DIS |
1572	GAMTLBVEBOX1_CLKGATE_DIS |
1573	GAMTLBVEBOX0_CLKGATE_DIS);
1574	}
1575
1576	/* Wa_16012725990:xehpsdv */
1577	if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A1, STEP_FOREVER))
1578	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, VFUNIT_CLKGATE_DIS);
1579
1580	/* Wa_14011060649:xehpsdv */
1581	wa_14011060649(gt, wal);
1582
1583	/* Wa_14012362059:xehpsdv */
1584	wa_mcr_write_or(wal, XEHP_MERT_MOD_CTRL, FORCE_MISS_FTLB);
1585
1586	/* Wa_14014368820:xehpsdv */
1587	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
1588	INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
1589
1590	/* Wa_14010670810:xehpsdv */
1591	wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
1592	}
1593
1594	static void
1595	dg2_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1596	{
1597	xehp_init_mcr(gt, wal);
1598
1599	/* Wa_14011060649:dg2 */
1600	wa_14011060649(gt, wal);
1601
1602	if (IS_DG2_G10(gt->i915)) {
1603	/* Wa_22010523718:dg2 */
1604	wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE,
1605	CG3DDISCFEG_CLKGATE_DIS);
1606
1607	/* Wa_14011006942:dg2 */
1608	wa_mcr_write_or(wal, GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE,
1609	DSS_ROUTER_CLKGATE_DIS);
1610	}
1611
1612	/* Wa_14014830051:dg2 */
1613	wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
1614
1615	/*
1616	* Wa_14015795083
1617	* Skip verification for possibly locked register.
1618	*/
1619	wa_add(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE,
1620	set: 0, read_mask: 0, masked_reg: false);
1621
1622	/* Wa_18018781329 */
1623	wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
1624	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
1625	wa_mcr_write_or(wal, XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
1626	wa_mcr_write_or(wal, XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
1627
1628	/* Wa_1509235366:dg2 */
1629	wa_mcr_write_or(wal, XEHP_GAMCNTRL_CTRL,
1630	INVALIDATION_BROADCAST_MODE_DIS | GLOBAL_INVALIDATION_MODE);
1631
1632	/* Wa_14010648519:dg2 */
1633	wa_mcr_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
1634	}
1635
1636	static void
1637	pvc_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1638	{
1639	pvc_init_mcr(gt, wal);
1640
1641	/* Wa_14015795083 */
1642	wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1643
1644	/* Wa_18018781329 */
1645	wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
1646	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
1647	wa_mcr_write_or(wal, XEHP_VDBX_MOD_CTRL, FORCE_MISS_FTLB);
1648	wa_mcr_write_or(wal, XEHP_VEBX_MOD_CTRL, FORCE_MISS_FTLB);
1649
1650	/* Wa_16016694945 */
1651	wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
1652	}
1653
1654	static void
1655	xelpg_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1656	{
1657	/* Wa_14018575942 / Wa_18018781329 */
1658	wa_mcr_write_or(wal, RENDER_MOD_CTRL, FORCE_MISS_FTLB);
1659	wa_mcr_write_or(wal, COMP_MOD_CTRL, FORCE_MISS_FTLB);
1660
1661	/* Wa_22016670082 */
1662	wa_write_or(wal, GEN12_SQCNT1, GEN12_STRICT_RAR_ENABLE);
1663
1664	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
1665	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)) {
1666	/* Wa_14014830051 */
1667	wa_mcr_write_clr(wal, SARB_CHICKEN1, COMP_CKN_IN);
1668
1669	/* Wa_14015795083 */
1670	wa_write_clr(wal, GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE);
1671	}
1672
1673	/*
1674	* Unlike older platforms, we no longer setup implicit steering here;
1675	* all MCR accesses are explicitly steered.
1676	*/
1677	debug_dump_steering(gt);
1678	}
1679
1680	static void
1681	wa_16021867713(struct intel_gt *gt, struct i915_wa_list *wal)
1682	{
1683	struct intel_engine_cs *engine;
1684	int id;
1685
1686	for_each_engine(engine, gt, id)
1687	if (engine->class == VIDEO_DECODE_CLASS)
1688	wa_write_or(wal, VDBOX_CGCTL3F1C(engine->mmio_base),
1689	MFXPIPE_CLKGATE_DIS);
1690	}
1691
1692	static void
1693	xelpmp_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal)
1694	{
1695	wa_16021867713(gt, wal);
1696
1697	/*
1698	* Wa_14018778641
1699	* Wa_18018781329
1700	*
1701	* Note that although these registers are MCR on the primary
1702	* GT, the media GT's versions are regular singleton registers.
1703	*/
1704	wa_write_or(wal, XELPMP_GSC_MOD_CTRL, FORCE_MISS_FTLB);
1705
1706	/* Wa_22016670082 */
1707	wa_write_or(wal, GEN12_SQCNT1, GEN12_STRICT_RAR_ENABLE);
1708
1709	debug_dump_steering(gt);
1710	}
1711
1712	/*
1713	* The bspec performance guide has recommended MMIO tuning settings. These
1714	* aren't truly "workarounds" but we want to program them through the
1715	* workaround infrastructure to make sure they're (re)applied at the proper
1716	* times.
1717	*
1718	* The programming in this function is for settings that persist through
1719	* engine resets and also are not part of any engine's register state context.
1720	* I.e., settings that only need to be re-applied in the event of a full GT
1721	* reset.
1722	*/
1723	static void gt_tuning_settings(struct intel_gt *gt, struct i915_wa_list *wal)
1724	{
1725	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74))) {
1726	wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
1727	wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
1728	}
1729
1730	if (IS_PONTEVECCHIO(gt->i915)) {
1731	wa_mcr_write(wal, XEHPC_L3SCRUB,
1732	SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
1733	wa_mcr_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_HOSTCACHEEN);
1734	}
1735
1736	if (IS_DG2(gt->i915)) {
1737	wa_mcr_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
1738	wa_mcr_write_or(wal, XEHP_SQCM, EN_32B_ACCESS);
1739	}
1740	}
1741
1742	static void
1743	gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal)
1744	{
1745	struct drm_i915_private *i915 = gt->i915;
1746
1747	gt_tuning_settings(gt, wal);
1748
1749	if (gt->type == GT_MEDIA) {
1750	if (MEDIA_VER_FULL(i915) == IP_VER(13, 0))
1751	xelpmp_gt_workarounds_init(gt, wal);
1752	else
1753	MISSING_CASE(MEDIA_VER_FULL(i915));
1754
1755	return;
1756	}
1757
1758	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74)))
1759	xelpg_gt_workarounds_init(gt, wal);
1760	else if (IS_PONTEVECCHIO(i915))
1761	pvc_gt_workarounds_init(gt, wal);
1762	else if (IS_DG2(i915))
1763	dg2_gt_workarounds_init(gt, wal);
1764	else if (IS_XEHPSDV(i915))
1765	xehpsdv_gt_workarounds_init(gt, wal);
1766	else if (IS_DG1(i915))
1767	dg1_gt_workarounds_init(gt, wal);
1768	else if (GRAPHICS_VER(i915) == 12)
1769	gen12_gt_workarounds_init(gt, wal);
1770	else if (GRAPHICS_VER(i915) == 11)
1771	icl_gt_workarounds_init(gt, wal);
1772	else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
1773	cfl_gt_workarounds_init(gt, wal);
1774	else if (IS_GEMINILAKE(i915))
1775	glk_gt_workarounds_init(gt, wal);
1776	else if (IS_KABYLAKE(i915))
1777	kbl_gt_workarounds_init(gt, wal);
1778	else if (IS_BROXTON(i915))
1779	gen9_gt_workarounds_init(gt, wal);
1780	else if (IS_SKYLAKE(i915))
1781	skl_gt_workarounds_init(gt, wal);
1782	else if (IS_HASWELL(i915))
1783	hsw_gt_workarounds_init(gt, wal);
1784	else if (IS_VALLEYVIEW(i915))
1785	vlv_gt_workarounds_init(gt, wal);
1786	else if (IS_IVYBRIDGE(i915))
1787	ivb_gt_workarounds_init(gt, wal);
1788	else if (GRAPHICS_VER(i915) == 6)
1789	snb_gt_workarounds_init(gt, wal);
1790	else if (GRAPHICS_VER(i915) == 5)
1791	ilk_gt_workarounds_init(gt, wal);
1792	else if (IS_G4X(i915))
1793	g4x_gt_workarounds_init(gt, wal);
1794	else if (GRAPHICS_VER(i915) == 4)
1795	gen4_gt_workarounds_init(gt, wal);
1796	else if (GRAPHICS_VER(i915) <= 8)
1797	;
1798	else
1799	MISSING_CASE(GRAPHICS_VER(i915));
1800	}
1801
1802	void intel_gt_init_workarounds(struct intel_gt *gt)
1803	{
1804	struct i915_wa_list *wal = &gt->wa_list;
1805
1806	wa_init_start(wal, gt, name: "GT", engine_name: "global");
1807	gt_init_workarounds(gt, wal);
1808	wa_init_finish(wal);
1809	}
1810
1811	static bool
1812	wa_verify(struct intel_gt *gt, const struct i915_wa *wa, u32 cur,
1813	const char *name, const char *from)
1814	{
1815	if ((cur ^ wa->set) & wa->read) {
1816	gt_err(gt,
1817	"%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n",
1818	name, from, i915_mmio_reg_offset(wa->reg),
1819	cur, cur & wa->read, wa->set & wa->read);
1820
1821	return false;
1822	}
1823
1824	return true;
1825	}
1826
1827	static void wa_list_apply(const struct i915_wa_list *wal)
1828	{
1829	struct intel_gt *gt = wal->gt;
1830	struct intel_uncore *uncore = gt->uncore;
1831	enum forcewake_domains fw;
1832	unsigned long flags;
1833	struct i915_wa *wa;
1834	unsigned int i;
1835
1836	if (!wal->count)
1837	return;
1838
1839	fw = wal_get_fw_for_rmw(uncore, wal);
1840
1841	intel_gt_mcr_lock(gt, flags: &flags);
1842	spin_lock(lock: &uncore->lock);
1843	intel_uncore_forcewake_get__locked(uncore, domains: fw);
1844
1845	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1846	u32 val, old = 0;
1847
1848	/* open-coded rmw due to steering */
1849	if (wa->clr)
1850	old = wa->is_mcr ?
1851	intel_gt_mcr_read_any_fw(gt, reg: wa->mcr_reg) :
1852	intel_uncore_read_fw(uncore, wa->reg);
1853	val = (old & ~wa->clr) | wa->set;
1854	if (val != old || !wa->clr) {
1855	if (wa->is_mcr)
1856	intel_gt_mcr_multicast_write_fw(gt, reg: wa->mcr_reg, value: val);
1857	else
1858	intel_uncore_write_fw(uncore, wa->reg, val);
1859	}
1860
1861	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
1862	u32 val = wa->is_mcr ?
1863	intel_gt_mcr_read_any_fw(gt, reg: wa->mcr_reg) :
1864	intel_uncore_read_fw(uncore, wa->reg);
1865
1866	wa_verify(gt, wa, cur: val, name: wal->name, from: "application");
1867	}
1868	}
1869
1870	intel_uncore_forcewake_put__locked(uncore, domains: fw);
1871	spin_unlock(lock: &uncore->lock);
1872	intel_gt_mcr_unlock(gt, flags);
1873	}
1874
1875	void intel_gt_apply_workarounds(struct intel_gt *gt)
1876	{
1877	wa_list_apply(wal: &gt->wa_list);
1878	}
1879
1880	static bool wa_list_verify(struct intel_gt *gt,
1881	const struct i915_wa_list *wal,
1882	const char *from)
1883	{
1884	struct intel_uncore *uncore = gt->uncore;
1885	struct i915_wa *wa;
1886	enum forcewake_domains fw;
1887	unsigned long flags;
1888	unsigned int i;
1889	bool ok = true;
1890
1891	fw = wal_get_fw_for_rmw(uncore, wal);
1892
1893	intel_gt_mcr_lock(gt, flags: &flags);
1894	spin_lock(lock: &uncore->lock);
1895	intel_uncore_forcewake_get__locked(uncore, domains: fw);
1896
1897	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1898	ok &= wa_verify(gt: wal->gt, wa, cur: wa->is_mcr ?
1899	intel_gt_mcr_read_any_fw(gt, reg: wa->mcr_reg) :
1900	intel_uncore_read_fw(uncore, wa->reg),
1901	name: wal->name, from);
1902
1903	intel_uncore_forcewake_put__locked(uncore, domains: fw);
1904	spin_unlock(lock: &uncore->lock);
1905	intel_gt_mcr_unlock(gt, flags);
1906
1907	return ok;
1908	}
1909
1910	bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from)
1911	{
1912	return wa_list_verify(gt, wal: &gt->wa_list, from);
1913	}
1914
1915	__maybe_unused
1916	static bool is_nonpriv_flags_valid(u32 flags)
1917	{
1918	/* Check only valid flag bits are set */
1919	if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID)
1920	return false;
1921
1922	/* NB: Only 3 out of 4 enum values are valid for access field */
1923	if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
1924	RING_FORCE_TO_NONPRIV_ACCESS_INVALID)
1925	return false;
1926
1927	return true;
1928	}
1929
1930	static void
1931	whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags)
1932	{
1933	struct i915_wa wa = {
1934	.reg = reg
1935	};
1936
1937	if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1938	return;
1939
1940	if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1941	return;
1942
1943	wa.reg.reg |= flags;
1944	_wa_add(wal, wa: &wa);
1945	}
1946
1947	static void
1948	whitelist_mcr_reg_ext(struct i915_wa_list *wal, i915_mcr_reg_t reg, u32 flags)
1949	{
1950	struct i915_wa wa = {
1951	.mcr_reg = reg,
1952	.is_mcr = 1,
1953	};
1954
1955	if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1956	return;
1957
1958	if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags)))
1959	return;
1960
1961	wa.mcr_reg.reg |= flags;
1962	_wa_add(wal, wa: &wa);
1963	}
1964
1965	static void
1966	whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1967	{
1968	whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1969	}
1970
1971	static void
1972	whitelist_mcr_reg(struct i915_wa_list *wal, i915_mcr_reg_t reg)
1973	{
1974	whitelist_mcr_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW);
1975	}
1976
1977	static void gen9_whitelist_build(struct i915_wa_list *w)
1978	{
1979	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1980	whitelist_reg(wal: w, GEN9_CTX_PREEMPT_REG);
1981
1982	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1983	whitelist_reg(wal: w, GEN8_CS_CHICKEN1);
1984
1985	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1986	whitelist_reg(wal: w, GEN8_HDC_CHICKEN1);
1987
1988	/* WaSendPushConstantsFromMMIO:skl,bxt */
1989	whitelist_reg(wal: w, COMMON_SLICE_CHICKEN2);
1990	}
1991
1992	static void skl_whitelist_build(struct intel_engine_cs *engine)
1993	{
1994	struct i915_wa_list *w = &engine->whitelist;
1995
1996	if (engine->class != RENDER_CLASS)
1997	return;
1998
1999	gen9_whitelist_build(w);
2000
2001	/* WaDisableLSQCROPERFforOCL:skl */
2002	whitelist_mcr_reg(wal: w, GEN8_L3SQCREG4);
2003	}
2004
2005	static void bxt_whitelist_build(struct intel_engine_cs *engine)
2006	{
2007	if (engine->class != RENDER_CLASS)
2008	return;
2009
2010	gen9_whitelist_build(w: &engine->whitelist);
2011	}
2012
2013	static void kbl_whitelist_build(struct intel_engine_cs *engine)
2014	{
2015	struct i915_wa_list *w = &engine->whitelist;
2016
2017	if (engine->class != RENDER_CLASS)
2018	return;
2019
2020	gen9_whitelist_build(w);
2021
2022	/* WaDisableLSQCROPERFforOCL:kbl */
2023	whitelist_mcr_reg(wal: w, GEN8_L3SQCREG4);
2024	}
2025
2026	static void glk_whitelist_build(struct intel_engine_cs *engine)
2027	{
2028	struct i915_wa_list *w = &engine->whitelist;
2029
2030	if (engine->class != RENDER_CLASS)
2031	return;
2032
2033	gen9_whitelist_build(w);
2034
2035	/* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
2036	whitelist_reg(wal: w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
2037	}
2038
2039	static void cfl_whitelist_build(struct intel_engine_cs *engine)
2040	{
2041	struct i915_wa_list *w = &engine->whitelist;
2042
2043	if (engine->class != RENDER_CLASS)
2044	return;
2045
2046	gen9_whitelist_build(w);
2047
2048	/*
2049	* WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml
2050	*
2051	* This covers 4 register which are next to one another :
2052	* - PS_INVOCATION_COUNT
2053	* - PS_INVOCATION_COUNT_UDW
2054	* - PS_DEPTH_COUNT
2055	* - PS_DEPTH_COUNT_UDW
2056	*/
2057	whitelist_reg_ext(wal: w, PS_INVOCATION_COUNT,
2058	RING_FORCE_TO_NONPRIV_ACCESS_RD |
2059	RING_FORCE_TO_NONPRIV_RANGE_4);
2060	}
2061
2062	static void allow_read_ctx_timestamp(struct intel_engine_cs *engine)
2063	{
2064	struct i915_wa_list *w = &engine->whitelist;
2065
2066	if (engine->class != RENDER_CLASS)
2067	whitelist_reg_ext(wal: w,
2068	RING_CTX_TIMESTAMP(engine->mmio_base),
2069	RING_FORCE_TO_NONPRIV_ACCESS_RD);
2070	}
2071
2072	static void cml_whitelist_build(struct intel_engine_cs *engine)
2073	{
2074	allow_read_ctx_timestamp(engine);
2075
2076	cfl_whitelist_build(engine);
2077	}
2078
2079	static void icl_whitelist_build(struct intel_engine_cs *engine)
2080	{
2081	struct i915_wa_list *w = &engine->whitelist;
2082
2083	allow_read_ctx_timestamp(engine);
2084
2085	switch (engine->class) {
2086	case RENDER_CLASS:
2087	/* WaAllowUMDToModifyHalfSliceChicken7:icl */
2088	whitelist_mcr_reg(wal: w, GEN9_HALF_SLICE_CHICKEN7);
2089
2090	/* WaAllowUMDToModifySamplerMode:icl */
2091	whitelist_mcr_reg(wal: w, GEN10_SAMPLER_MODE);
2092
2093	/* WaEnableStateCacheRedirectToCS:icl */
2094	whitelist_reg(wal: w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
2095
2096	/*
2097	* WaAllowPMDepthAndInvocationCountAccessFromUMD:icl
2098	*
2099	* This covers 4 register which are next to one another :
2100	* - PS_INVOCATION_COUNT
2101	* - PS_INVOCATION_COUNT_UDW
2102	* - PS_DEPTH_COUNT
2103	* - PS_DEPTH_COUNT_UDW
2104	*/
2105	whitelist_reg_ext(wal: w, PS_INVOCATION_COUNT,
2106	RING_FORCE_TO_NONPRIV_ACCESS_RD |
2107	RING_FORCE_TO_NONPRIV_RANGE_4);
2108	break;
2109
2110	case VIDEO_DECODE_CLASS:
2111	/* hucStatusRegOffset */
2112	whitelist_reg_ext(wal: w, _MMIO(0x2000 + engine->mmio_base),
2113	RING_FORCE_TO_NONPRIV_ACCESS_RD);
2114	/* hucUKernelHdrInfoRegOffset */
2115	whitelist_reg_ext(wal: w, _MMIO(0x2014 + engine->mmio_base),
2116	RING_FORCE_TO_NONPRIV_ACCESS_RD);
2117	/* hucStatus2RegOffset */
2118	whitelist_reg_ext(wal: w, _MMIO(0x23B0 + engine->mmio_base),
2119	RING_FORCE_TO_NONPRIV_ACCESS_RD);
2120	break;
2121
2122	default:
2123	break;
2124	}
2125	}
2126
2127	static void tgl_whitelist_build(struct intel_engine_cs *engine)
2128	{
2129	struct i915_wa_list *w = &engine->whitelist;
2130
2131	allow_read_ctx_timestamp(engine);
2132
2133	switch (engine->class) {
2134	case RENDER_CLASS:
2135	/*
2136	* WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl
2137	* Wa_1408556865:tgl
2138	*
2139	* This covers 4 registers which are next to one another :
2140	* - PS_INVOCATION_COUNT
2141	* - PS_INVOCATION_COUNT_UDW
2142	* - PS_DEPTH_COUNT
2143	* - PS_DEPTH_COUNT_UDW
2144	*/
2145	whitelist_reg_ext(wal: w, PS_INVOCATION_COUNT,
2146	RING_FORCE_TO_NONPRIV_ACCESS_RD |
2147	RING_FORCE_TO_NONPRIV_RANGE_4);
2148
2149	/*
2150	* Wa_1808121037:tgl
2151	* Wa_14012131227:dg1
2152	* Wa_1508744258:tgl,rkl,dg1,adl-s,adl-p
2153	*/
2154	whitelist_reg(wal: w, GEN7_COMMON_SLICE_CHICKEN1);
2155
2156	/* Wa_1806527549:tgl */
2157	whitelist_reg(wal: w, HIZ_CHICKEN);
2158
2159	/* Required by recommended tuning setting (not a workaround) */
2160	whitelist_reg(wal: w, GEN11_COMMON_SLICE_CHICKEN3);
2161
2162	break;
2163	default:
2164	break;
2165	}
2166	}
2167
2168	static void dg2_whitelist_build(struct intel_engine_cs *engine)
2169	{
2170	struct i915_wa_list *w = &engine->whitelist;
2171
2172	switch (engine->class) {
2173	case RENDER_CLASS:
2174	/* Required by recommended tuning setting (not a workaround) */
2175	whitelist_mcr_reg(wal: w, XEHP_COMMON_SLICE_CHICKEN3);
2176
2177	break;
2178	default:
2179	break;
2180	}
2181	}
2182
2183	static void blacklist_trtt(struct intel_engine_cs *engine)
2184	{
2185	struct i915_wa_list *w = &engine->whitelist;
2186
2187	/*
2188	* Prevent read/write access to [0x4400, 0x4600) which covers
2189	* the TRTT range across all engines. Note that normally userspace
2190	* cannot access the other engines' trtt control, but for simplicity
2191	* we cover the entire range on each engine.
2192	*/
2193	whitelist_reg_ext(wal: w, _MMIO(0x4400),
2194	RING_FORCE_TO_NONPRIV_DENY |
2195	RING_FORCE_TO_NONPRIV_RANGE_64);
2196	whitelist_reg_ext(wal: w, _MMIO(0x4500),
2197	RING_FORCE_TO_NONPRIV_DENY |
2198	RING_FORCE_TO_NONPRIV_RANGE_64);
2199	}
2200
2201	static void pvc_whitelist_build(struct intel_engine_cs *engine)
2202	{
2203	/* Wa_16014440446:pvc */
2204	blacklist_trtt(engine);
2205	}
2206
2207	static void xelpg_whitelist_build(struct intel_engine_cs *engine)
2208	{
2209	struct i915_wa_list *w = &engine->whitelist;
2210
2211	switch (engine->class) {
2212	case RENDER_CLASS:
2213	/* Required by recommended tuning setting (not a workaround) */
2214	whitelist_mcr_reg(wal: w, XEHP_COMMON_SLICE_CHICKEN3);
2215
2216	break;
2217	default:
2218	break;
2219	}
2220	}
2221
2222	void intel_engine_init_whitelist(struct intel_engine_cs *engine)
2223	{
2224	struct drm_i915_private *i915 = engine->i915;
2225	struct i915_wa_list *w = &engine->whitelist;
2226
2227	wa_init_start(wal: w, gt: engine->gt, name: "whitelist", engine_name: engine->name);
2228
2229	if (engine->gt->type == GT_MEDIA)
2230	; /* none yet */
2231	else if (IS_GFX_GT_IP_RANGE(engine->gt, IP_VER(12, 70), IP_VER(12, 74)))
2232	xelpg_whitelist_build(engine);
2233	else if (IS_PONTEVECCHIO(i915))
2234	pvc_whitelist_build(engine);
2235	else if (IS_DG2(i915))
2236	dg2_whitelist_build(engine);
2237	else if (IS_XEHPSDV(i915))
2238	; /* none needed */
2239	else if (GRAPHICS_VER(i915) == 12)
2240	tgl_whitelist_build(engine);
2241	else if (GRAPHICS_VER(i915) == 11)
2242	icl_whitelist_build(engine);
2243	else if (IS_COMETLAKE(i915))
2244	cml_whitelist_build(engine);
2245	else if (IS_COFFEELAKE(i915))
2246	cfl_whitelist_build(engine);
2247	else if (IS_GEMINILAKE(i915))
2248	glk_whitelist_build(engine);
2249	else if (IS_KABYLAKE(i915))
2250	kbl_whitelist_build(engine);
2251	else if (IS_BROXTON(i915))
2252	bxt_whitelist_build(engine);
2253	else if (IS_SKYLAKE(i915))
2254	skl_whitelist_build(engine);
2255	else if (GRAPHICS_VER(i915) <= 8)
2256	;
2257	else
2258	MISSING_CASE(GRAPHICS_VER(i915));
2259
2260	wa_init_finish(wal: w);
2261	}
2262
2263	void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
2264	{
2265	const struct i915_wa_list *wal = &engine->whitelist;
2266	struct intel_uncore *uncore = engine->uncore;
2267	const u32 base = engine->mmio_base;
2268	struct i915_wa *wa;
2269	unsigned int i;
2270
2271	if (!wal->count)
2272	return;
2273
2274	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
2275	intel_uncore_write(uncore,
2276	RING_FORCE_TO_NONPRIV(base, i),
2277	i915_mmio_reg_offset(wa->reg));
2278
2279	/* And clear the rest just in case of garbage */
2280	for (; i < RING_MAX_NONPRIV_SLOTS; i++)
2281	intel_uncore_write(uncore,
2282	RING_FORCE_TO_NONPRIV(base, i),
2283	i915_mmio_reg_offset(RING_NOPID(base)));
2284	}
2285
2286	/*
2287	* engine_fake_wa_init(), a place holder to program the registers
2288	* which are not part of an official workaround defined by the
2289	* hardware team.
2290	* Adding programming of those register inside workaround will
2291	* allow utilizing wa framework to proper application and verification.
2292	*/
2293	static void
2294	engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2295	{
2296	u8 mocs_w, mocs_r;
2297
2298	/*
2299	* RING_CMD_CCTL specifies the default MOCS entry that will be used
2300	* by the command streamer when executing commands that don't have
2301	* a way to explicitly specify a MOCS setting. The default should
2302	* usually reference whichever MOCS entry corresponds to uncached
2303	* behavior, although use of a WB cached entry is recommended by the
2304	* spec in certain circumstances on specific platforms.
2305	*/
2306	if (GRAPHICS_VER(engine->i915) >= 12) {
2307	mocs_r = engine->gt->mocs.uc_index;
2308	mocs_w = engine->gt->mocs.uc_index;
2309
2310	if (HAS_L3_CCS_READ(engine->i915) &&
2311	engine->class == COMPUTE_CLASS) {
2312	mocs_r = engine->gt->mocs.wb_index;
2313
2314	/*
2315	* Even on the few platforms where MOCS 0 is a
2316	* legitimate table entry, it's never the correct
2317	* setting to use here; we can assume the MOCS init
2318	* just forgot to initialize wb_index.
2319	*/
2320	drm_WARN_ON(&engine->i915->drm, mocs_r == 0);
2321	}
2322
2323	wa_masked_field_set(wal,
2324	RING_CMD_CCTL(engine->mmio_base),
2325	CMD_CCTL_MOCS_MASK,
2326	CMD_CCTL_MOCS_OVERRIDE(mocs_w, mocs_r));
2327	}
2328	}
2329
2330	static void
2331	rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2332	{
2333	struct drm_i915_private *i915 = engine->i915;
2334	struct intel_gt *gt = engine->gt;
2335
2336	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
2337	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)) {
2338	/* Wa_22014600077 */
2339	wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS,
2340	ENABLE_EU_COUNT_FOR_TDL_FLUSH);
2341	}
2342
2343	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
2344	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0) ||
2345	IS_DG2(i915)) {
2346	/* Wa_1509727124 */
2347	wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
2348	SC_DISABLE_POWER_OPTIMIZATION_EBB);
2349	}
2350
2351	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
2352	IS_DG2(i915)) {
2353	/* Wa_22012856258 */
2354	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
2355	GEN12_DISABLE_READ_SUPPRESSION);
2356	}
2357
2358	if (IS_DG2(i915)) {
2359	/*
2360	* Wa_22010960976:dg2
2361	* Wa_14013347512:dg2
2362	*/
2363	wa_mcr_masked_dis(wal, XEHP_HDC_CHICKEN0,
2364	LSC_L1_FLUSH_CTL_3D_DATAPORT_FLUSH_EVENTS_MASK);
2365	}
2366
2367	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 71)) ||
2368	IS_DG2(i915)) {
2369	/* Wa_14015150844 */
2370	wa_mcr_add(wal, XEHP_HDC_CHICKEN0, clear: 0,
2371	_MASKED_BIT_ENABLE(DIS_ATOMIC_CHAINING_TYPED_WRITES),
2372	read_mask: 0, masked_reg: true);
2373	}
2374
2375	if (IS_DG2(i915) || IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
2376	IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2377	/*
2378	* Wa_1606700617:tgl,dg1,adl-p
2379	* Wa_22010271021:tgl,rkl,dg1,adl-s,adl-p
2380	* Wa_14010826681:tgl,dg1,rkl,adl-p
2381	* Wa_18019627453:dg2
2382	*/
2383	wa_masked_en(wal,
2384	GEN9_CS_DEBUG_MODE1,
2385	FF_DOP_CLOCK_GATE_DISABLE);
2386	}
2387
2388	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) || IS_DG1(i915) ||
2389	IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2390	/* Wa_1606931601:tgl,rkl,dg1,adl-s,adl-p */
2391	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ);
2392
2393	/*
2394	* Wa_1407928979:tgl A*
2395	* Wa_18011464164:tgl[B0+],dg1[B0+]
2396	* Wa_22010931296:tgl[B0+],dg1[B0+]
2397	* Wa_14010919138:rkl,dg1,adl-s,adl-p
2398	*/
2399	wa_write_or(wal, GEN7_FF_THREAD_MODE,
2400	GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2401
2402	/* Wa_1406941453:tgl,rkl,dg1,adl-s,adl-p */
2403	wa_mcr_masked_en(wal,
2404	GEN10_SAMPLER_MODE,
2405	ENABLE_SMALLPL);
2406	}
2407
2408	if (IS_ALDERLAKE_P(i915) || IS_ALDERLAKE_S(i915) ||
2409	IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) {
2410	/* Wa_1409804808 */
2411	wa_mcr_masked_en(wal, GEN8_ROW_CHICKEN2,
2412	GEN12_PUSH_CONST_DEREF_HOLD_DIS);
2413
2414	/* Wa_14010229206 */
2415	wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH);
2416	}
2417
2418	if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) {
2419	/*
2420	* Wa_1607297627
2421	*
2422	* On TGL and RKL there are multiple entries for this WA in the
2423	* BSpec; some indicate this is an A0-only WA, others indicate
2424	* it applies to all steppings so we trust the "all steppings."
2425	*/
2426	wa_masked_en(wal,
2427	RING_PSMI_CTL(RENDER_RING_BASE),
2428	GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE |
2429	GEN8_RC_SEMA_IDLE_MSG_DISABLE);
2430	}
2431
2432	if (GRAPHICS_VER(i915) == 11) {
2433	/* This is not an Wa. Enable for better image quality */
2434	wa_masked_en(wal,
2435	_3D_CHICKEN3,
2436	_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
2437
2438	/*
2439	* Wa_1405543622:icl
2440	* Formerly known as WaGAPZPriorityScheme
2441	*/
2442	wa_write_or(wal,
2443	GEN8_GARBCNTL,
2444	GEN11_ARBITRATION_PRIO_ORDER_MASK);
2445
2446	/*
2447	* Wa_1604223664:icl
2448	* Formerly known as WaL3BankAddressHashing
2449	*/
2450	wa_write_clr_set(wal,
2451	GEN8_GARBCNTL,
2452	GEN11_HASH_CTRL_EXCL_MASK,
2453	GEN11_HASH_CTRL_EXCL_BIT0);
2454	wa_write_clr_set(wal,
2455	GEN11_GLBLINVL,
2456	GEN11_BANK_HASH_ADDR_EXCL_MASK,
2457	GEN11_BANK_HASH_ADDR_EXCL_BIT0);
2458
2459	/*
2460	* Wa_1405733216:icl
2461	* Formerly known as WaDisableCleanEvicts
2462	*/
2463	wa_mcr_write_or(wal,
2464	GEN8_L3SQCREG4,
2465	GEN11_LQSC_CLEAN_EVICT_DISABLE);
2466
2467	/* Wa_1606682166:icl */
2468	wa_write_or(wal,
2469	GEN7_SARCHKMD,
2470	GEN7_DISABLE_SAMPLER_PREFETCH);
2471
2472	/* Wa_1409178092:icl */
2473	wa_mcr_write_clr_set(wal,
2474	GEN11_SCRATCH2,
2475	GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE,
2476	set: 0);
2477
2478	/* WaEnable32PlaneMode:icl */
2479	wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS,
2480	GEN11_ENABLE_32_PLANE_MODE);
2481
2482	/*
2483	* Wa_1408767742:icl[a2..forever],ehl[all]
2484	* Wa_1605460711:icl[a0..c0]
2485	*/
2486	wa_write_or(wal,
2487	GEN7_FF_THREAD_MODE,
2488	GEN12_FF_TESSELATION_DOP_GATE_DISABLE);
2489
2490	/* Wa_22010271021 */
2491	wa_masked_en(wal,
2492	GEN9_CS_DEBUG_MODE1,
2493	FF_DOP_CLOCK_GATE_DISABLE);
2494	}
2495
2496	/*
2497	* Intel platforms that support fine-grained preemption (i.e., gen9 and
2498	* beyond) allow the kernel-mode driver to choose between two different
2499	* options for controlling preemption granularity and behavior.
2500	*
2501	* Option 1 (hardware default):
2502	* Preemption settings are controlled in a global manner via
2503	* kernel-only register CS_DEBUG_MODE1 (0x20EC). Any granularity
2504	* and settings chosen by the kernel-mode driver will apply to all
2505	* userspace clients.
2506	*
2507	* Option 2:
2508	* Preemption settings are controlled on a per-context basis via
2509	* register CS_CHICKEN1 (0x2580). CS_CHICKEN1 is saved/restored on
2510	* context switch and is writable by userspace (e.g., via
2511	* MI_LOAD_REGISTER_IMMEDIATE instructions placed in a batch buffer)
2512	* which allows different userspace drivers/clients to select
2513	* different settings, or to change those settings on the fly in
2514	* response to runtime needs. This option was known by name
2515	* "FtrPerCtxtPreemptionGranularityControl" at one time, although
2516	* that name is somewhat misleading as other non-granularity
2517	* preemption settings are also impacted by this decision.
2518	*
2519	* On Linux, our policy has always been to let userspace drivers
2520	* control preemption granularity/settings (Option 2). This was
2521	* originally mandatory on gen9 to prevent ABI breakage (old gen9
2522	* userspace developed before object-level preemption was enabled would
2523	* not behave well if i915 were to go with Option 1 and enable that
2524	* preemption in a global manner). On gen9 each context would have
2525	* object-level preemption disabled by default (see
2526	* WaDisable3DMidCmdPreemption in gen9_ctx_workarounds_init), but
2527	* userspace drivers could opt-in to object-level preemption as they
2528	* saw fit. For post-gen9 platforms, we continue to utilize Option 2;
2529	* even though it is no longer necessary for ABI compatibility when
2530	* enabling a new platform, it does ensure that userspace will be able
2531	* to implement any workarounds that show up requiring temporary
2532	* adjustments to preemption behavior at runtime.
2533	*
2534	* Notes/Workarounds:
2535	* - Wa_14015141709: On DG2 and early steppings of MTL,
2536	* CS_CHICKEN1[0] does not disable object-level preemption as
2537	* it is supposed to (nor does CS_DEBUG_MODE1[0] if we had been
2538	* using Option 1). Effectively this means userspace is unable
2539	* to disable object-level preemption on these platforms/steppings
2540	* despite the setting here.
2541	*
2542	* - Wa_16013994831: May require that userspace program
2543	* CS_CHICKEN1[10] when certain runtime conditions are true.
2544	* Userspace requires Option 2 to be in effect for their update of
2545	* CS_CHICKEN1[10] to be effective.
2546	*
2547	* Other workarounds may appear in the future that will also require
2548	* Option 2 behavior to allow proper userspace implementation.
2549	*/
2550	if (GRAPHICS_VER(i915) >= 9)
2551	wa_masked_en(wal,
2552	GEN7_FF_SLICE_CS_CHICKEN1,
2553	GEN9_FFSC_PERCTX_PREEMPT_CTRL);
2554
2555	if (IS_SKYLAKE(i915) ||
2556	IS_KABYLAKE(i915) ||
2557	IS_COFFEELAKE(i915) ||
2558	IS_COMETLAKE(i915)) {
2559	/* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
2560	wa_write_or(wal,
2561	GEN8_GARBCNTL,
2562	GEN9_GAPS_TSV_CREDIT_DISABLE);
2563	}
2564
2565	if (IS_BROXTON(i915)) {
2566	/* WaDisablePooledEuLoadBalancingFix:bxt */
2567	wa_masked_en(wal,
2568	FF_SLICE_CS_CHICKEN2,
2569	GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
2570	}
2571
2572	if (GRAPHICS_VER(i915) == 9) {
2573	/* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
2574	wa_masked_en(wal,
2575	GEN9_CSFE_CHICKEN1_RCS,
2576	GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
2577
2578	/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
2579	wa_mcr_write_or(wal,
2580	BDW_SCRATCH1,
2581	GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
2582
2583	/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
2584	if (IS_GEN9_LP(i915))
2585	wa_mcr_write_clr_set(wal,
2586	GEN8_L3SQCREG1,
2587	L3_PRIO_CREDITS_MASK,
2588	L3_GENERAL_PRIO_CREDITS(62) |
2589	L3_HIGH_PRIO_CREDITS(2));
2590
2591	/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
2592	wa_mcr_write_or(wal,
2593	GEN8_L3SQCREG4,
2594	GEN8_LQSC_FLUSH_COHERENT_LINES);
2595
2596	/* Disable atomics in L3 to prevent unrecoverable hangs */
2597	wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1,
2598	GEN9_LNCF_NONIA_COHERENT_ATOMICS_ENABLE, set: 0);
2599	wa_mcr_write_clr_set(wal, GEN8_L3SQCREG4,
2600	GEN8_LQSQ_NONIA_COHERENT_ATOMICS_ENABLE, set: 0);
2601	wa_mcr_write_clr_set(wal, GEN9_SCRATCH1,
2602	EVICTION_PERF_FIX_ENABLE, set: 0);
2603	}
2604
2605	if (IS_HASWELL(i915)) {
2606	/* WaSampleCChickenBitEnable:hsw */
2607	wa_masked_en(wal,
2608	HSW_HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
2609
2610	wa_masked_dis(wal,
2611	CACHE_MODE_0_GEN7,
2612	/* enable HiZ Raw Stall Optimization */
2613	HIZ_RAW_STALL_OPT_DISABLE);
2614	}
2615
2616	if (IS_VALLEYVIEW(i915)) {
2617	/* WaDisableEarlyCull:vlv */
2618	wa_masked_en(wal,
2619	_3D_CHICKEN3,
2620	_3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2621
2622	/*
2623	* WaVSThreadDispatchOverride:ivb,vlv
2624	*
2625	* This actually overrides the dispatch
2626	* mode for all thread types.
2627	*/
2628	wa_write_clr_set(wal,
2629	GEN7_FF_THREAD_MODE,
2630	GEN7_FF_SCHED_MASK,
2631	GEN7_FF_TS_SCHED_HW |
2632	GEN7_FF_VS_SCHED_HW |
2633	GEN7_FF_DS_SCHED_HW);
2634
2635	/* WaPsdDispatchEnable:vlv */
2636	/* WaDisablePSDDualDispatchEnable:vlv */
2637	wa_masked_en(wal,
2638	GEN7_HALF_SLICE_CHICKEN1,
2639	GEN7_MAX_PS_THREAD_DEP |
2640	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2641	}
2642
2643	if (IS_IVYBRIDGE(i915)) {
2644	/* WaDisableEarlyCull:ivb */
2645	wa_masked_en(wal,
2646	_3D_CHICKEN3,
2647	_3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
2648
2649	if (0) { /* causes HiZ corruption on ivb:gt1 */
2650	/* enable HiZ Raw Stall Optimization */
2651	wa_masked_dis(wal,
2652	CACHE_MODE_0_GEN7,
2653	HIZ_RAW_STALL_OPT_DISABLE);
2654	}
2655
2656	/*
2657	* WaVSThreadDispatchOverride:ivb,vlv
2658	*
2659	* This actually overrides the dispatch
2660	* mode for all thread types.
2661	*/
2662	wa_write_clr_set(wal,
2663	GEN7_FF_THREAD_MODE,
2664	GEN7_FF_SCHED_MASK,
2665	GEN7_FF_TS_SCHED_HW |
2666	GEN7_FF_VS_SCHED_HW |
2667	GEN7_FF_DS_SCHED_HW);
2668
2669	/* WaDisablePSDDualDispatchEnable:ivb */
2670	if (IS_IVB_GT1(i915))
2671	wa_masked_en(wal,
2672	GEN7_HALF_SLICE_CHICKEN1,
2673	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
2674	}
2675
2676	if (GRAPHICS_VER(i915) == 7) {
2677	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
2678	wa_masked_en(wal,
2679	RING_MODE_GEN7(RENDER_RING_BASE),
2680	GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE);
2681
2682	/* WaDisable_RenderCache_OperationalFlush:ivb,vlv,hsw */
2683	wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
2684
2685	/*
2686	* BSpec says this must be set, even though
2687	* WaDisable4x2SubspanOptimization:ivb,hsw
2688	* WaDisable4x2SubspanOptimization isn't listed for VLV.
2689	*/
2690	wa_masked_en(wal,
2691	CACHE_MODE_1,
2692	PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
2693
2694	/*
2695	* BSpec recommends 8x4 when MSAA is used,
2696	* however in practice 16x4 seems fastest.
2697	*
2698	* Note that PS/WM thread counts depend on the WIZ hashing
2699	* disable bit, which we don't touch here, but it's good
2700	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2701	*/
2702	wa_masked_field_set(wal,
2703	GEN7_GT_MODE,
2704	GEN6_WIZ_HASHING_MASK,
2705	GEN6_WIZ_HASHING_16x4);
2706	}
2707
2708	if (IS_GRAPHICS_VER(i915, 6, 7))
2709	/*
2710	* We need to disable the AsyncFlip performance optimisations in
2711	* order to use MI_WAIT_FOR_EVENT within the CS. It should
2712	* already be programmed to '1' on all products.
2713	*
2714	* WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
2715	*/
2716	wa_masked_en(wal,
2717	RING_MI_MODE(RENDER_RING_BASE),
2718	ASYNC_FLIP_PERF_DISABLE);
2719
2720	if (GRAPHICS_VER(i915) == 6) {
2721	/*
2722	* Required for the hardware to program scanline values for
2723	* waiting
2724	* WaEnableFlushTlbInvalidationMode:snb
2725	*/
2726	wa_masked_en(wal,
2727	GFX_MODE,
2728	GFX_TLB_INVALIDATE_EXPLICIT);
2729
2730	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
2731	wa_masked_en(wal,
2732	_3D_CHICKEN,
2733	_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
2734
2735	wa_masked_en(wal,
2736	_3D_CHICKEN3,
2737	/* WaStripsFansDisableFastClipPerformanceFix:snb */
2738	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
2739	/*
2740	* Bspec says:
2741	* "This bit must be set if 3DSTATE_CLIP clip mode is set
2742	* to normal and 3DSTATE_SF number of SF output attributes
2743	* is more than 16."
2744	*/
2745	_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
2746
2747	/*
2748	* BSpec recommends 8x4 when MSAA is used,
2749	* however in practice 16x4 seems fastest.
2750	*
2751	* Note that PS/WM thread counts depend on the WIZ hashing
2752	* disable bit, which we don't touch here, but it's good
2753	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
2754	*/
2755	wa_masked_field_set(wal,
2756	GEN6_GT_MODE,
2757	GEN6_WIZ_HASHING_MASK,
2758	GEN6_WIZ_HASHING_16x4);
2759
2760	/* WaDisable_RenderCache_OperationalFlush:snb */
2761	wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
2762
2763	/*
2764	* From the Sandybridge PRM, volume 1 part 3, page 24:
2765	* "If this bit is set, STCunit will have LRA as replacement
2766	* policy. [...] This bit must be reset. LRA replacement
2767	* policy is not supported."
2768	*/
2769	wa_masked_dis(wal,
2770	CACHE_MODE_0,
2771	CM0_STC_EVICT_DISABLE_LRA_SNB);
2772	}
2773
2774	if (IS_GRAPHICS_VER(i915, 4, 6))
2775	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
2776	wa_add(wal, RING_MI_MODE(RENDER_RING_BASE),
2777	clear: 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH),
2778	/* XXX bit doesn't stick on Broadwater */
2779	IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH, masked_reg: true);
2780
2781	if (GRAPHICS_VER(i915) == 4)
2782	/*
2783	* Disable CONSTANT_BUFFER before it is loaded from the context
2784	* image. For as it is loaded, it is executed and the stored
2785	* address may no longer be valid, leading to a GPU hang.
2786	*
2787	* This imposes the requirement that userspace reload their
2788	* CONSTANT_BUFFER on every batch, fortunately a requirement
2789	* they are already accustomed to from before contexts were
2790	* enabled.
2791	*/
2792	wa_add(wal, ECOSKPD(RENDER_RING_BASE),
2793	clear: 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE),
2794	read_mask: 0 /* XXX bit doesn't stick on Broadwater */,
2795	masked_reg: true);
2796	}
2797
2798	static void
2799	xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2800	{
2801	struct drm_i915_private *i915 = engine->i915;
2802
2803	/* WaKBLVECSSemaphoreWaitPoll:kbl */
2804	if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, STEP_A0, STEP_F0)) {
2805	wa_write(wal,
2806	RING_SEMA_WAIT_POLL(engine->mmio_base),
2807	set: 1);
2808	}
2809	/* Wa_16018031267, Wa_16018063123 */
2810	if (NEEDS_FASTCOLOR_BLT_WABB(engine))
2811	wa_masked_field_set(wal, ECOSKPD(engine->mmio_base),
2812	XEHP_BLITTER_SCHEDULING_MODE_MASK,
2813	XEHP_BLITTER_ROUND_ROBIN_MODE);
2814	}
2815
2816	static void
2817	ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2818	{
2819	if (IS_PVC_CT_STEP(engine->i915, STEP_A0, STEP_C0)) {
2820	/* Wa_14014999345:pvc */
2821	wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS, DISABLE_ECC);
2822	}
2823	}
2824
2825	/*
2826	* The bspec performance guide has recommended MMIO tuning settings. These
2827	* aren't truly "workarounds" but we want to program them with the same
2828	* workaround infrastructure to ensure that they're automatically added to
2829	* the GuC save/restore lists, re-applied at the right times, and checked for
2830	* any conflicting programming requested by real workarounds.
2831	*
2832	* Programming settings should be added here only if their registers are not
2833	* part of an engine's register state context. If a register is part of a
2834	* context, then any tuning settings should be programmed in an appropriate
2835	* function invoked by __intel_engine_init_ctx_wa().
2836	*/
2837	static void
2838	add_render_compute_tuning_settings(struct intel_gt *gt,
2839	struct i915_wa_list *wal)
2840	{
2841	struct drm_i915_private *i915 = gt->i915;
2842
2843	if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74)) || IS_DG2(i915))
2844	wa_mcr_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512);
2845
2846	/*
2847	* This tuning setting proves beneficial only on ATS-M designs; the
2848	* default "age based" setting is optimal on regular DG2 and other
2849	* platforms.
2850	*/
2851	if (INTEL_INFO(i915)->tuning_thread_rr_after_dep)
2852	wa_mcr_masked_field_set(wal, GEN9_ROW_CHICKEN4, THREAD_EX_ARB_MODE,
2853	THREAD_EX_ARB_MODE_RR_AFTER_DEP);
2854
2855	if (GRAPHICS_VER(i915) == 12 && GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
2856	wa_write_clr(wal, GEN8_GARBCNTL, GEN12_BUS_HASH_CTL_BIT_EXC);
2857	}
2858
2859	static void ccs_engine_wa_mode(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2860	{
2861	struct intel_gt *gt = engine->gt;
2862
2863	if (!IS_DG2(gt->i915))
2864	return;
2865
2866	/*
2867	* Wa_14019159160: This workaround, along with others, leads to
2868	* significant challenges in utilizing load balancing among the
2869	* CCS slices. Consequently, an architectural decision has been
2870	* made to completely disable automatic CCS load balancing.
2871	*/
2872	wa_masked_en(wal, GEN12_RCU_MODE, XEHP_RCU_MODE_FIXED_SLICE_CCS_MODE);
2873
2874	/*
2875	* After having disabled automatic load balancing we need to
2876	* assign all slices to a single CCS. We will call it CCS mode 1
2877	*/
2878	intel_gt_apply_ccs_mode(gt);
2879	}
2880
2881	/*
2882	* The workarounds in this function apply to shared registers in
2883	* the general render reset domain that aren't tied to a
2884	* specific engine. Since all render+compute engines get reset
2885	* together, and the contents of these registers are lost during
2886	* the shared render domain reset, we'll define such workarounds
2887	* here and then add them to just a single RCS or CCS engine's
2888	* workaround list (whichever engine has the XXXX flag).
2889	*/
2890	static void
2891	general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
2892	{
2893	struct drm_i915_private *i915 = engine->i915;
2894	struct intel_gt *gt = engine->gt;
2895
2896	add_render_compute_tuning_settings(gt, wal);
2897
2898	if (GRAPHICS_VER(i915) >= 11) {
2899	/* This is not a Wa (although referred to as
2900	* WaSetInidrectStateOverride in places), this allows
2901	* applications that reference sampler states through
2902	* the BindlessSamplerStateBaseAddress to have their
2903	* border color relative to DynamicStateBaseAddress
2904	* rather than BindlessSamplerStateBaseAddress.
2905	*
2906	* Otherwise SAMPLER_STATE border colors have to be
2907	* copied in multiple heaps (DynamicStateBaseAddress &
2908	* BindlessSamplerStateBaseAddress)
2909	*
2910	* BSpec: 46052
2911	*/
2912	wa_mcr_masked_en(wal,
2913	GEN10_SAMPLER_MODE,
2914	GEN11_INDIRECT_STATE_BASE_ADDR_OVERRIDE);
2915	}
2916
2917	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_B0, STEP_FOREVER) ||
2918	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_B0, STEP_FOREVER) ||
2919	IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 74), IP_VER(12, 74)))
2920	/* Wa_14017856879 */
2921	wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN3, MTL_DISABLE_FIX_FOR_EOT_FLUSH);
2922
2923	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
2924	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))
2925	/*
2926	* Wa_14017066071
2927	* Wa_14017654203
2928	*/
2929	wa_mcr_masked_en(wal, GEN10_SAMPLER_MODE,
2930	MTL_DISABLE_SAMPLER_SC_OOO);
2931
2932	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0))
2933	/* Wa_22015279794 */
2934	wa_mcr_masked_en(wal, GEN10_CACHE_MODE_SS,
2935	DISABLE_PREFETCH_INTO_IC);
2936
2937	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
2938	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0) ||
2939	IS_DG2(i915)) {
2940	/* Wa_22013037850 */
2941	wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW,
2942	DISABLE_128B_EVICTION_COMMAND_UDW);
2943
2944	/* Wa_18017747507 */
2945	wa_masked_en(wal, VFG_PREEMPTION_CHICKEN, POLYGON_TRIFAN_LINELOOP_DISABLE);
2946	}
2947
2948	if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
2949	IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0) ||
2950	IS_PONTEVECCHIO(i915) ||
2951	IS_DG2(i915)) {
2952	/* Wa_22014226127 */
2953	wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0, DISABLE_D8_D16_COASLESCE);
2954	}
2955
2956	if (IS_PONTEVECCHIO(i915) || IS_DG2(i915)) {
2957	/* Wa_14015227452:dg2,pvc */
2958	wa_mcr_masked_en(wal, GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE);
2959
2960	/* Wa_16015675438:dg2,pvc */
2961	wa_masked_en(wal, FF_SLICE_CS_CHICKEN2, GEN12_PERF_FIX_BALANCING_CFE_DISABLE);
2962	}
2963
2964	if (IS_DG2(i915)) {
2965	/*
2966	* Wa_16011620976:dg2_g11
2967	* Wa_22015475538:dg2
2968	*/
2969	wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8);
2970
2971	/* Wa_18028616096 */
2972	wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0_UDW, UGM_FRAGMENT_THRESHOLD_TO_3);
2973	}
2974
2975	if (IS_DG2_G11(i915)) {
2976	/*
2977	* Wa_22012826095:dg2
2978	* Wa_22013059131:dg2
2979	*/
2980	wa_mcr_write_clr_set(wal, LSC_CHICKEN_BIT_0_UDW,
2981	MAXREQS_PER_BANK,
2982	REG_FIELD_PREP(MAXREQS_PER_BANK, 2));
2983
2984	/* Wa_22013059131:dg2 */
2985	wa_mcr_write_or(wal, LSC_CHICKEN_BIT_0,
2986	FORCE_1_SUB_MESSAGE_PER_FRAGMENT);
2987
2988	/*
2989	* Wa_22012654132
2990	*
2991	* Note that register 0xE420 is write-only and cannot be read
2992	* back for verification on DG2 (due to Wa_14012342262), so
2993	* we need to explicitly skip the readback.
2994	*/
2995	wa_mcr_add(wal, GEN10_CACHE_MODE_SS, clear: 0,
2996	_MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
2997	read_mask: 0 /* write-only, so skip validation */,
2998	masked_reg: true);
2999	}
3000
3001	if (IS_XEHPSDV(i915)) {
3002	/* Wa_1409954639 */
3003	wa_mcr_masked_en(wal,
3004	GEN8_ROW_CHICKEN,
3005	SYSTOLIC_DOP_CLOCK_GATING_DIS);
3006
3007	/* Wa_1607196519 */
3008	wa_mcr_masked_en(wal,
3009	GEN9_ROW_CHICKEN4,
3010	GEN12_DISABLE_GRF_CLEAR);
3011
3012	/* Wa_14010449647:xehpsdv */
3013	wa_mcr_masked_en(wal, GEN8_HALF_SLICE_CHICKEN1,
3014	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
3015	}
3016	}
3017
3018	static void
3019	engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
3020	{
3021	if (GRAPHICS_VER(engine->i915) < 4)
3022	return;
3023
3024	engine_fake_wa_init(engine, wal);
3025
3026	/*
3027	* These are common workarounds that just need to applied
3028	* to a single RCS/CCS engine's workaround list since
3029	* they're reset as part of the general render domain reset.
3030	*/
3031	if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) {
3032	general_render_compute_wa_init(engine, wal);
3033	ccs_engine_wa_mode(engine, wal);
3034	}
3035
3036	if (engine->class == COMPUTE_CLASS)
3037	ccs_engine_wa_init(engine, wal);
3038	else if (engine->class == RENDER_CLASS)
3039	rcs_engine_wa_init(engine, wal);
3040	else
3041	xcs_engine_wa_init(engine, wal);
3042	}
3043
3044	void intel_engine_init_workarounds(struct intel_engine_cs *engine)
3045	{
3046	struct i915_wa_list *wal = &engine->wa_list;
3047
3048	wa_init_start(wal, gt: engine->gt, name: "engine", engine_name: engine->name);
3049	engine_init_workarounds(engine, wal);
3050	wa_init_finish(wal);
3051	}
3052
3053	void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
3054	{
3055	wa_list_apply(wal: &engine->wa_list);
3056	}
3057
3058	static const struct i915_range mcr_ranges_gen8[] = {
3059	{ .start = 0x5500, .end = 0x55ff },
3060	{ .start = 0x7000, .end = 0x7fff },
3061	{ .start = 0x9400, .end = 0x97ff },
3062	{ .start = 0xb000, .end = 0xb3ff },
3063	{ .start = 0xe000, .end = 0xe7ff },
3064	{},
3065	};
3066
3067	static const struct i915_range mcr_ranges_gen12[] = {
3068	{ .start = 0x8150, .end = 0x815f },
3069	{ .start = 0x9520, .end = 0x955f },
3070	{ .start = 0xb100, .end = 0xb3ff },
3071	{ .start = 0xde80, .end = 0xe8ff },
3072	{ .start = 0x24a00, .end = 0x24a7f },
3073	{},
3074	};
3075
3076	static const struct i915_range mcr_ranges_xehp[] = {
3077	{ .start = 0x4000, .end = 0x4aff },
3078	{ .start = 0x5200, .end = 0x52ff },
3079	{ .start = 0x5400, .end = 0x7fff },
3080	{ .start = 0x8140, .end = 0x815f },
3081	{ .start = 0x8c80, .end = 0x8dff },
3082	{ .start = 0x94d0, .end = 0x955f },
3083	{ .start = 0x9680, .end = 0x96ff },
3084	{ .start = 0xb000, .end = 0xb3ff },
3085	{ .start = 0xc800, .end = 0xcfff },
3086	{ .start = 0xd800, .end = 0xd8ff },
3087	{ .start = 0xdc00, .end = 0xffff },
3088	{ .start = 0x17000, .end = 0x17fff },
3089	{ .start = 0x24a00, .end = 0x24a7f },
3090	{},
3091	};
3092
3093	static bool mcr_range(struct drm_i915_private *i915, u32 offset)
3094	{
3095	const struct i915_range *mcr_ranges;
3096	int i;
3097
3098	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50))
3099	mcr_ranges = mcr_ranges_xehp;
3100	else if (GRAPHICS_VER(i915) >= 12)
3101	mcr_ranges = mcr_ranges_gen12;
3102	else if (GRAPHICS_VER(i915) >= 8)
3103	mcr_ranges = mcr_ranges_gen8;
3104	else
3105	return false;
3106
3107	/*
3108	* Registers in these ranges are affected by the MCR selector
3109	* which only controls CPU initiated MMIO. Routing does not
3110	* work for CS access so we cannot verify them on this path.
3111	*/
3112	for (i = 0; mcr_ranges[i].start; i++)
3113	if (offset >= mcr_ranges[i].start &&
3114	offset <= mcr_ranges[i].end)
3115	return true;
3116
3117	return false;
3118	}
3119
3120	static int
3121	wa_list_srm(struct i915_request *rq,
3122	const struct i915_wa_list *wal,
3123	struct i915_vma *vma)
3124	{
3125	struct drm_i915_private *i915 = rq->i915;
3126	unsigned int i, count = 0;
3127	const struct i915_wa *wa;
3128	u32 srm, *cs;
3129
3130	srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
3131	if (GRAPHICS_VER(i915) >= 8)
3132	srm++;
3133
3134	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3135	if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg)))
3136	count++;
3137	}
3138
3139	cs = intel_ring_begin(rq, num_dwords: 4 * count);
3140	if (IS_ERR(ptr: cs))
3141	return PTR_ERR(ptr: cs);
3142
3143	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3144	u32 offset = i915_mmio_reg_offset(wa->reg);
3145
3146	if (mcr_range(i915, offset))
3147	continue;
3148
3149	*cs++ = srm;
3150	*cs++ = offset;
3151	*cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
3152	*cs++ = 0;
3153	}
3154	intel_ring_advance(rq, cs);
3155
3156	return 0;
3157	}
3158
3159	static int engine_wa_list_verify(struct intel_context *ce,
3160	const struct i915_wa_list * const wal,
3161	const char *from)
3162	{
3163	const struct i915_wa *wa;
3164	struct i915_request *rq;
3165	struct i915_vma *vma;
3166	struct i915_gem_ww_ctx ww;
3167	unsigned int i;
3168	u32 *results;
3169	int err;
3170
3171	if (!wal->count)
3172	return 0;
3173
3174	vma = __vm_create_scratch_for_read(vm: &ce->engine->gt->ggtt->vm,
3175	size: wal->count * sizeof(u32));
3176	if (IS_ERR(ptr: vma))
3177	return PTR_ERR(ptr: vma);
3178
3179	intel_engine_pm_get(engine: ce->engine);
3180	i915_gem_ww_ctx_init(ctx: &ww, intr: false);
3181	retry:
3182	err = i915_gem_object_lock(obj: vma->obj, ww: &ww);
3183	if (err == 0)
3184	err = intel_context_pin_ww(ce, ww: &ww);
3185	if (err)
3186	goto err_pm;
3187
3188	err = i915_vma_pin_ww(vma, ww: &ww, size: 0, alignment: 0,
3189	flags: i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
3190	if (err)
3191	goto err_unpin;
3192
3193	rq = i915_request_create(ce);
3194	if (IS_ERR(ptr: rq)) {
3195	err = PTR_ERR(ptr: rq);
3196	goto err_vma;
3197	}
3198
3199	err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
3200	if (err == 0)
3201	err = wa_list_srm(rq, wal, vma);
3202
3203	i915_request_get(rq);
3204	if (err)
3205	i915_request_set_error_once(rq, error: err);
3206	i915_request_add(rq);
3207
3208	if (err)
3209	goto err_rq;
3210
3211	if (i915_request_wait(rq, flags: 0, HZ / 5) < 0) {
3212	err = -ETIME;
3213	goto err_rq;
3214	}
3215
3216	results = i915_gem_object_pin_map(obj: vma->obj, type: I915_MAP_WB);
3217	if (IS_ERR(ptr: results)) {
3218	err = PTR_ERR(ptr: results);
3219	goto err_rq;
3220	}
3221
3222	err = 0;
3223	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
3224	if (mcr_range(i915: rq->i915, i915_mmio_reg_offset(wa->reg)))
3225	continue;
3226
3227	if (!wa_verify(gt: wal->gt, wa, cur: results[i], name: wal->name, from))
3228	err = -ENXIO;
3229	}
3230
3231	i915_gem_object_unpin_map(obj: vma->obj);
3232
3233	err_rq:
3234	i915_request_put(rq);
3235	err_vma:
3236	i915_vma_unpin(vma);
3237	err_unpin:
3238	intel_context_unpin(ce);
3239	err_pm:
3240	if (err == -EDEADLK) {
3241	err = i915_gem_ww_ctx_backoff(ctx: &ww);
3242	if (!err)
3243	goto retry;
3244	}
3245	i915_gem_ww_ctx_fini(ctx: &ww);
3246	intel_engine_pm_put(engine: ce->engine);
3247	i915_vma_put(vma);
3248	return err;
3249	}
3250
3251	int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
3252	const char *from)
3253	{
3254	return engine_wa_list_verify(ce: engine->kernel_context,
3255	wal: &engine->wa_list,
3256	from);
3257	}
3258
3259	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3260	#include "selftest_workarounds.c"
3261	#endif
3262