1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2022 Intel Corporation |
4 | */ |
5 | |
6 | #include "i915_drv.h" |
7 | #include "intel_gt.h" |
8 | #include "intel_gt_mcr.h" |
9 | #include "intel_gt_print.h" |
10 | #include "intel_gt_regs.h" |
11 | |
12 | /** |
13 | * DOC: GT Multicast/Replicated (MCR) Register Support |
14 | * |
15 | * Some GT registers are designed as "multicast" or "replicated" registers: |
16 | * multiple instances of the same register share a single MMIO offset. MCR |
17 | * registers are generally used when the hardware needs to potentially track |
18 | * independent values of a register per hardware unit (e.g., per-subslice, |
19 | * per-L3bank, etc.). The specific types of replication that exist vary |
20 | * per-platform. |
21 | * |
22 | * MMIO accesses to MCR registers are controlled according to the settings |
23 | * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR |
24 | * registers can be done in either a (i.e., a single write updates all |
25 | * instances of the register to the same value) or unicast (a write updates only |
26 | * one specific instance). Reads of MCR registers always operate in a unicast |
27 | * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR. |
28 | * Selection of a specific MCR instance for unicast operations is referred to |
29 | * as "steering." |
30 | * |
31 | * If MCR register operations are steered toward a hardware unit that is |
32 | * fused off or currently powered down due to power gating, the MMIO operation |
33 | * is "terminated" by the hardware. Terminated read operations will return a |
34 | * value of zero and terminated unicast write operations will be silently |
35 | * ignored. |
36 | */ |
37 | |
38 | #define HAS_MSLICE_STEERING(i915) (INTEL_INFO(i915)->has_mslice_steering) |
39 | |
40 | static const char * const intel_steering_types[] = { |
41 | "L3BANK" , |
42 | "MSLICE" , |
43 | "LNCF" , |
44 | "GAM" , |
45 | "DSS" , |
46 | "OADDRM" , |
47 | "INSTANCE 0" , |
48 | }; |
49 | |
50 | static const struct intel_mmio_range icl_l3bank_steering_table[] = { |
51 | { 0x00B100, 0x00B3FF }, |
52 | {}, |
53 | }; |
54 | |
55 | /* |
56 | * Although the bspec lists more "MSLICE" ranges than shown here, some of those |
57 | * are of a "GAM" subclass that has special rules. Thus we use a separate |
58 | * GAM table farther down for those. |
59 | */ |
60 | static const struct intel_mmio_range xehpsdv_mslice_steering_table[] = { |
61 | { 0x00DD00, 0x00DDFF }, |
62 | { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ |
63 | {}, |
64 | }; |
65 | |
66 | static const struct intel_mmio_range xehpsdv_gam_steering_table[] = { |
67 | { 0x004000, 0x004AFF }, |
68 | { 0x00C800, 0x00CFFF }, |
69 | {}, |
70 | }; |
71 | |
72 | static const struct intel_mmio_range xehpsdv_lncf_steering_table[] = { |
73 | { 0x00B000, 0x00B0FF }, |
74 | { 0x00D800, 0x00D8FF }, |
75 | {}, |
76 | }; |
77 | |
78 | static const struct intel_mmio_range dg2_lncf_steering_table[] = { |
79 | { 0x00B000, 0x00B0FF }, |
80 | { 0x00D880, 0x00D8FF }, |
81 | {}, |
82 | }; |
83 | |
84 | /* |
85 | * We have several types of MCR registers on PVC where steering to (0,0) |
86 | * will always provide us with a non-terminated value. We'll stick them |
87 | * all in the same table for simplicity. |
88 | */ |
89 | static const struct intel_mmio_range pvc_instance0_steering_table[] = { |
90 | { 0x004000, 0x004AFF }, /* HALF-BSLICE */ |
91 | { 0x008800, 0x00887F }, /* CC */ |
92 | { 0x008A80, 0x008AFF }, /* TILEPSMI */ |
93 | { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ |
94 | { 0x00B100, 0x00B3FF }, /* L3BANK */ |
95 | { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ |
96 | { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ |
97 | { 0x00DD00, 0x00DDFF }, /* BSLICE */ |
98 | { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ |
99 | { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ |
100 | { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ |
101 | { 0x024180, 0x0241FF }, /* HALF-BSLICE */ |
102 | {}, |
103 | }; |
104 | |
105 | static const struct intel_mmio_range xelpg_instance0_steering_table[] = { |
106 | { 0x000B00, 0x000BFF }, /* SQIDI */ |
107 | { 0x001000, 0x001FFF }, /* SQIDI */ |
108 | { 0x004000, 0x0048FF }, /* GAM */ |
109 | { 0x008700, 0x0087FF }, /* SQIDI */ |
110 | { 0x00B000, 0x00B0FF }, /* NODE */ |
111 | { 0x00C800, 0x00CFFF }, /* GAM */ |
112 | { 0x00D880, 0x00D8FF }, /* NODE */ |
113 | { 0x00DD00, 0x00DDFF }, /* OAAL2 */ |
114 | {}, |
115 | }; |
116 | |
117 | static const struct intel_mmio_range xelpg_l3bank_steering_table[] = { |
118 | { 0x00B100, 0x00B3FF }, |
119 | {}, |
120 | }; |
121 | |
122 | /* DSS steering is used for SLICE ranges as well */ |
123 | static const struct intel_mmio_range xelpg_dss_steering_table[] = { |
124 | { 0x005200, 0x0052FF }, /* SLICE */ |
125 | { 0x005500, 0x007FFF }, /* SLICE */ |
126 | { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ |
127 | { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ |
128 | { 0x009680, 0x0096FF }, /* DSS */ |
129 | { 0x00D800, 0x00D87F }, /* SLICE */ |
130 | { 0x00DC00, 0x00DCFF }, /* SLICE */ |
131 | { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ |
132 | {}, |
133 | }; |
134 | |
135 | static const struct intel_mmio_range xelpmp_oaddrm_steering_table[] = { |
136 | { 0x393200, 0x39323F }, |
137 | { 0x393400, 0x3934FF }, |
138 | {}, |
139 | }; |
140 | |
141 | void intel_gt_mcr_init(struct intel_gt *gt) |
142 | { |
143 | struct drm_i915_private *i915 = gt->i915; |
144 | unsigned long fuse; |
145 | int i; |
146 | |
147 | spin_lock_init(>->mcr_lock); |
148 | |
149 | /* |
150 | * An mslice is unavailable only if both the meml3 for the slice is |
151 | * disabled *and* all of the DSS in the slice (quadrant) are disabled. |
152 | */ |
153 | if (HAS_MSLICE_STEERING(i915)) { |
154 | gt->info.mslice_mask = |
155 | intel_slicemask_from_xehp_dssmask(dss_mask: gt->info.sseu.subslice_mask, |
156 | GEN_DSS_PER_MSLICE); |
157 | gt->info.mslice_mask |= |
158 | (intel_uncore_read(uncore: gt->uncore, GEN10_MIRROR_FUSE3) & |
159 | GEN12_MEML3_EN_MASK); |
160 | |
161 | if (!gt->info.mslice_mask) /* should be impossible! */ |
162 | gt_warn(gt, "mslice mask all zero!\n" ); |
163 | } |
164 | |
165 | if (MEDIA_VER(i915) >= 13 && gt->type == GT_MEDIA) { |
166 | gt->steering_table[OADDRM] = xelpmp_oaddrm_steering_table; |
167 | } else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) { |
168 | /* Wa_14016747170 */ |
169 | if (IS_GFX_GT_IP_STEP(gt, IP_VER(12, 70), STEP_A0, STEP_B0) || |
170 | IS_GFX_GT_IP_STEP(gt, IP_VER(12, 71), STEP_A0, STEP_B0)) |
171 | fuse = REG_FIELD_GET(MTL_GT_L3_EXC_MASK, |
172 | intel_uncore_read(gt->uncore, |
173 | MTL_GT_ACTIVITY_FACTOR)); |
174 | else |
175 | fuse = REG_FIELD_GET(GT_L3_EXC_MASK, |
176 | intel_uncore_read(gt->uncore, XEHP_FUSE4)); |
177 | |
178 | /* |
179 | * Despite the register field being named "exclude mask" the |
180 | * bits actually represent enabled banks (two banks per bit). |
181 | */ |
182 | for_each_set_bit(i, &fuse, 3) |
183 | gt->info.l3bank_mask |= 0x3 << 2 * i; |
184 | |
185 | gt->steering_table[INSTANCE0] = xelpg_instance0_steering_table; |
186 | gt->steering_table[L3BANK] = xelpg_l3bank_steering_table; |
187 | gt->steering_table[DSS] = xelpg_dss_steering_table; |
188 | } else if (IS_PONTEVECCHIO(i915)) { |
189 | gt->steering_table[INSTANCE0] = pvc_instance0_steering_table; |
190 | } else if (IS_DG2(i915)) { |
191 | gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; |
192 | gt->steering_table[LNCF] = dg2_lncf_steering_table; |
193 | /* |
194 | * No need to hook up the GAM table since it has a dedicated |
195 | * steering control register on DG2 and can use implicit |
196 | * steering. |
197 | */ |
198 | } else if (IS_XEHPSDV(i915)) { |
199 | gt->steering_table[MSLICE] = xehpsdv_mslice_steering_table; |
200 | gt->steering_table[LNCF] = xehpsdv_lncf_steering_table; |
201 | gt->steering_table[GAM] = xehpsdv_gam_steering_table; |
202 | } else if (GRAPHICS_VER(i915) >= 11 && |
203 | GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) { |
204 | gt->steering_table[L3BANK] = icl_l3bank_steering_table; |
205 | gt->info.l3bank_mask = |
206 | ~intel_uncore_read(uncore: gt->uncore, GEN10_MIRROR_FUSE3) & |
207 | GEN10_L3BANK_MASK; |
208 | if (!gt->info.l3bank_mask) /* should be impossible! */ |
209 | gt_warn(gt, "L3 bank mask is all zero!\n" ); |
210 | } else if (GRAPHICS_VER(i915) >= 11) { |
211 | /* |
212 | * We expect all modern platforms to have at least some |
213 | * type of steering that needs to be initialized. |
214 | */ |
215 | MISSING_CASE(INTEL_INFO(i915)->platform); |
216 | } |
217 | } |
218 | |
219 | /* |
220 | * Although the rest of the driver should use MCR-specific functions to |
221 | * read/write MCR registers, we still use the regular intel_uncore_* functions |
222 | * internally to implement those, so we need a way for the functions in this |
223 | * file to "cast" an i915_mcr_reg_t into an i915_reg_t. |
224 | */ |
225 | static i915_reg_t mcr_reg_cast(const i915_mcr_reg_t mcr) |
226 | { |
227 | i915_reg_t r = { .reg = mcr.reg }; |
228 | |
229 | return r; |
230 | } |
231 | |
232 | /* |
233 | * rw_with_mcr_steering_fw - Access a register with specific MCR steering |
234 | * @gt: GT to read register from |
235 | * @reg: register being accessed |
236 | * @rw_flag: FW_REG_READ for read access or FW_REG_WRITE for write access |
237 | * @group: group number (documented as "sliceid" on older platforms) |
238 | * @instance: instance number (documented as "subsliceid" on older platforms) |
239 | * @value: register value to be written (ignored for read) |
240 | * |
241 | * Context: The caller must hold the MCR lock |
242 | * Return: 0 for write access. register value for read access. |
243 | * |
244 | * Caller needs to make sure the relevant forcewake wells are up. |
245 | */ |
246 | static u32 rw_with_mcr_steering_fw(struct intel_gt *gt, |
247 | i915_mcr_reg_t reg, u8 rw_flag, |
248 | int group, int instance, u32 value) |
249 | { |
250 | struct intel_uncore *uncore = gt->uncore; |
251 | u32 mcr_mask, mcr_ss, mcr, old_mcr, val = 0; |
252 | |
253 | lockdep_assert_held(>->mcr_lock); |
254 | |
255 | if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70)) { |
256 | /* |
257 | * Always leave the hardware in multicast mode when doing reads |
258 | * (see comment about Wa_22013088509 below) and only change it |
259 | * to unicast mode when doing writes of a specific instance. |
260 | * |
261 | * No need to save old steering reg value. |
262 | */ |
263 | intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, |
264 | REG_FIELD_PREP(MTL_MCR_GROUPID, group) | |
265 | REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance) | |
266 | (rw_flag == FW_REG_READ ? GEN11_MCR_MULTICAST : 0)); |
267 | } else if (GRAPHICS_VER(uncore->i915) >= 11) { |
268 | mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; |
269 | mcr_ss = GEN11_MCR_SLICE(group) | GEN11_MCR_SUBSLICE(instance); |
270 | |
271 | /* |
272 | * Wa_22013088509 |
273 | * |
274 | * The setting of the multicast/unicast bit usually wouldn't |
275 | * matter for read operations (which always return the value |
276 | * from a single register instance regardless of how that bit |
277 | * is set), but some platforms have a workaround requiring us |
278 | * to remain in multicast mode for reads. There's no real |
279 | * downside to this, so we'll just go ahead and do so on all |
280 | * platforms; we'll only clear the multicast bit from the mask |
281 | * when exlicitly doing a write operation. |
282 | */ |
283 | if (rw_flag == FW_REG_WRITE) |
284 | mcr_mask |= GEN11_MCR_MULTICAST; |
285 | |
286 | mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); |
287 | old_mcr = mcr; |
288 | |
289 | mcr &= ~mcr_mask; |
290 | mcr |= mcr_ss; |
291 | intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); |
292 | } else { |
293 | mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; |
294 | mcr_ss = GEN8_MCR_SLICE(group) | GEN8_MCR_SUBSLICE(instance); |
295 | |
296 | mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR); |
297 | old_mcr = mcr; |
298 | |
299 | mcr &= ~mcr_mask; |
300 | mcr |= mcr_ss; |
301 | intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr); |
302 | } |
303 | |
304 | if (rw_flag == FW_REG_READ) |
305 | val = intel_uncore_read_fw(uncore, mcr_reg_cast(reg)); |
306 | else |
307 | intel_uncore_write_fw(uncore, mcr_reg_cast(reg), value); |
308 | |
309 | /* |
310 | * For pre-MTL platforms, we need to restore the old value of the |
311 | * steering control register to ensure that implicit steering continues |
312 | * to behave as expected. For MTL and beyond, we need only reinstate |
313 | * the 'multicast' bit (and only if we did a write that cleared it). |
314 | */ |
315 | if (GRAPHICS_VER_FULL(uncore->i915) >= IP_VER(12, 70) && rw_flag == FW_REG_WRITE) |
316 | intel_uncore_write_fw(uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); |
317 | else if (GRAPHICS_VER_FULL(uncore->i915) < IP_VER(12, 70)) |
318 | intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, old_mcr); |
319 | |
320 | return val; |
321 | } |
322 | |
323 | static u32 rw_with_mcr_steering(struct intel_gt *gt, |
324 | i915_mcr_reg_t reg, u8 rw_flag, |
325 | int group, int instance, |
326 | u32 value) |
327 | { |
328 | struct intel_uncore *uncore = gt->uncore; |
329 | enum forcewake_domains fw_domains; |
330 | unsigned long flags; |
331 | u32 val; |
332 | |
333 | fw_domains = intel_uncore_forcewake_for_reg(uncore, reg: mcr_reg_cast(mcr: reg), |
334 | op: rw_flag); |
335 | fw_domains |= intel_uncore_forcewake_for_reg(uncore, |
336 | GEN8_MCR_SELECTOR, |
337 | FW_REG_READ | FW_REG_WRITE); |
338 | |
339 | intel_gt_mcr_lock(gt, flags: &flags); |
340 | spin_lock(lock: &uncore->lock); |
341 | intel_uncore_forcewake_get__locked(uncore, domains: fw_domains); |
342 | |
343 | val = rw_with_mcr_steering_fw(gt, reg, rw_flag, group, instance, value); |
344 | |
345 | intel_uncore_forcewake_put__locked(uncore, domains: fw_domains); |
346 | spin_unlock(lock: &uncore->lock); |
347 | intel_gt_mcr_unlock(gt, flags); |
348 | |
349 | return val; |
350 | } |
351 | |
352 | /** |
353 | * intel_gt_mcr_lock - Acquire MCR steering lock |
354 | * @gt: GT structure |
355 | * @flags: storage to save IRQ flags to |
356 | * |
357 | * Performs locking to protect the steering for the duration of an MCR |
358 | * operation. On MTL and beyond, a hardware lock will also be taken to |
359 | * serialize access not only for the driver, but also for external hardware and |
360 | * firmware agents. |
361 | * |
362 | * Context: Takes gt->mcr_lock. uncore->lock should *not* be held when this |
363 | * function is called, although it may be acquired after this |
364 | * function call. |
365 | */ |
366 | void intel_gt_mcr_lock(struct intel_gt *gt, unsigned long *flags) |
367 | __acquires(>->mcr_lock) |
368 | { |
369 | unsigned long __flags; |
370 | int err = 0; |
371 | |
372 | lockdep_assert_not_held(>->uncore->lock); |
373 | |
374 | /* |
375 | * Starting with MTL, we need to coordinate not only with other |
376 | * driver threads, but also with hardware/firmware agents. A dedicated |
377 | * locking register is used. |
378 | */ |
379 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { |
380 | /* |
381 | * The steering control and semaphore registers are inside an |
382 | * "always on" power domain with respect to RC6. However there |
383 | * are some issues if higher-level platform sleep states are |
384 | * entering/exiting at the same time these registers are |
385 | * accessed. Grabbing GT forcewake and holding it over the |
386 | * entire lock/steer/unlock cycle ensures that those sleep |
387 | * states have been fully exited before we access these |
388 | * registers. This wakeref will be released in the unlock |
389 | * routine. |
390 | * |
391 | * Wa_22018931422 |
392 | */ |
393 | intel_uncore_forcewake_get(uncore: gt->uncore, domains: FORCEWAKE_GT); |
394 | |
395 | err = wait_for(intel_uncore_read_fw(gt->uncore, |
396 | MTL_STEER_SEMAPHORE) == 0x1, 100); |
397 | } |
398 | |
399 | /* |
400 | * Even on platforms with a hardware lock, we'll continue to grab |
401 | * a software spinlock too for lockdep purposes. If the hardware lock |
402 | * was already acquired, there should never be contention on the |
403 | * software lock. |
404 | */ |
405 | spin_lock_irqsave(>->mcr_lock, __flags); |
406 | |
407 | *flags = __flags; |
408 | |
409 | /* |
410 | * In theory we should never fail to acquire the HW semaphore; this |
411 | * would indicate some hardware/firmware is misbehaving and not |
412 | * releasing it properly. |
413 | */ |
414 | if (err == -ETIMEDOUT) { |
415 | gt_err_ratelimited(gt, "hardware MCR steering semaphore timed out" ); |
416 | add_taint_for_CI(i915: gt->i915, TAINT_WARN); /* CI is now unreliable */ |
417 | } |
418 | } |
419 | |
420 | /** |
421 | * intel_gt_mcr_unlock - Release MCR steering lock |
422 | * @gt: GT structure |
423 | * @flags: IRQ flags to restore |
424 | * |
425 | * Releases the lock acquired by intel_gt_mcr_lock(). |
426 | * |
427 | * Context: Releases gt->mcr_lock |
428 | */ |
429 | void intel_gt_mcr_unlock(struct intel_gt *gt, unsigned long flags) |
430 | __releases(>->mcr_lock) |
431 | { |
432 | spin_unlock_irqrestore(lock: >->mcr_lock, flags); |
433 | |
434 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { |
435 | intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1); |
436 | |
437 | intel_uncore_forcewake_put(uncore: gt->uncore, domains: FORCEWAKE_GT); |
438 | } |
439 | } |
440 | |
441 | /** |
442 | * intel_gt_mcr_lock_sanitize - Sanitize MCR steering lock |
443 | * @gt: GT structure |
444 | * |
445 | * This will be used to sanitize the initial status of the hardware lock |
446 | * during driver load and resume since there won't be any concurrent access |
447 | * from other agents at those times, but it's possible that boot firmware |
448 | * may have left the lock in a bad state. |
449 | * |
450 | */ |
451 | void intel_gt_mcr_lock_sanitize(struct intel_gt *gt) |
452 | { |
453 | /* |
454 | * This gets called at load/resume time, so we shouldn't be |
455 | * racing with other driver threads grabbing the mcr lock. |
456 | */ |
457 | lockdep_assert_not_held(>->mcr_lock); |
458 | |
459 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) |
460 | intel_uncore_write_fw(gt->uncore, MTL_STEER_SEMAPHORE, 0x1); |
461 | } |
462 | |
463 | /** |
464 | * intel_gt_mcr_read - read a specific instance of an MCR register |
465 | * @gt: GT structure |
466 | * @reg: the MCR register to read |
467 | * @group: the MCR group |
468 | * @instance: the MCR instance |
469 | * |
470 | * Context: Takes and releases gt->mcr_lock |
471 | * |
472 | * Returns the value read from an MCR register after steering toward a specific |
473 | * group/instance. |
474 | */ |
475 | u32 intel_gt_mcr_read(struct intel_gt *gt, |
476 | i915_mcr_reg_t reg, |
477 | int group, int instance) |
478 | { |
479 | return rw_with_mcr_steering(gt, reg, FW_REG_READ, group, instance, value: 0); |
480 | } |
481 | |
482 | /** |
483 | * intel_gt_mcr_unicast_write - write a specific instance of an MCR register |
484 | * @gt: GT structure |
485 | * @reg: the MCR register to write |
486 | * @value: value to write |
487 | * @group: the MCR group |
488 | * @instance: the MCR instance |
489 | * |
490 | * Write an MCR register in unicast mode after steering toward a specific |
491 | * group/instance. |
492 | * |
493 | * Context: Calls a function that takes and releases gt->mcr_lock |
494 | */ |
495 | void intel_gt_mcr_unicast_write(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value, |
496 | int group, int instance) |
497 | { |
498 | rw_with_mcr_steering(gt, reg, FW_REG_WRITE, group, instance, value); |
499 | } |
500 | |
501 | /** |
502 | * intel_gt_mcr_multicast_write - write a value to all instances of an MCR register |
503 | * @gt: GT structure |
504 | * @reg: the MCR register to write |
505 | * @value: value to write |
506 | * |
507 | * Write an MCR register in multicast mode to update all instances. |
508 | * |
509 | * Context: Takes and releases gt->mcr_lock |
510 | */ |
511 | void intel_gt_mcr_multicast_write(struct intel_gt *gt, |
512 | i915_mcr_reg_t reg, u32 value) |
513 | { |
514 | unsigned long flags; |
515 | |
516 | intel_gt_mcr_lock(gt, flags: &flags); |
517 | |
518 | /* |
519 | * Ensure we have multicast behavior, just in case some non-i915 agent |
520 | * left the hardware in unicast mode. |
521 | */ |
522 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) |
523 | intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); |
524 | |
525 | intel_uncore_write(uncore: gt->uncore, reg: mcr_reg_cast(mcr: reg), val: value); |
526 | |
527 | intel_gt_mcr_unlock(gt, flags); |
528 | } |
529 | |
530 | /** |
531 | * intel_gt_mcr_multicast_write_fw - write a value to all instances of an MCR register |
532 | * @gt: GT structure |
533 | * @reg: the MCR register to write |
534 | * @value: value to write |
535 | * |
536 | * Write an MCR register in multicast mode to update all instances. This |
537 | * function assumes the caller is already holding any necessary forcewake |
538 | * domains; use intel_gt_mcr_multicast_write() in cases where forcewake should |
539 | * be obtained automatically. |
540 | * |
541 | * Context: The caller must hold gt->mcr_lock. |
542 | */ |
543 | void intel_gt_mcr_multicast_write_fw(struct intel_gt *gt, i915_mcr_reg_t reg, u32 value) |
544 | { |
545 | lockdep_assert_held(>->mcr_lock); |
546 | |
547 | /* |
548 | * Ensure we have multicast behavior, just in case some non-i915 agent |
549 | * left the hardware in unicast mode. |
550 | */ |
551 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) |
552 | intel_uncore_write_fw(gt->uncore, MTL_MCR_SELECTOR, GEN11_MCR_MULTICAST); |
553 | |
554 | intel_uncore_write_fw(gt->uncore, mcr_reg_cast(reg), value); |
555 | } |
556 | |
557 | /** |
558 | * intel_gt_mcr_multicast_rmw - Performs a multicast RMW operations |
559 | * @gt: GT structure |
560 | * @reg: the MCR register to read and write |
561 | * @clear: bits to clear during RMW |
562 | * @set: bits to set during RMW |
563 | * |
564 | * Performs a read-modify-write on an MCR register in a multicast manner. |
565 | * This operation only makes sense on MCR registers where all instances are |
566 | * expected to have the same value. The read will target any non-terminated |
567 | * instance and the write will be applied to all instances. |
568 | * |
569 | * This function assumes the caller is already holding any necessary forcewake |
570 | * domains; use intel_gt_mcr_multicast_rmw() in cases where forcewake should |
571 | * be obtained automatically. |
572 | * |
573 | * Context: Calls functions that take and release gt->mcr_lock |
574 | * |
575 | * Returns the old (unmodified) value read. |
576 | */ |
577 | u32 intel_gt_mcr_multicast_rmw(struct intel_gt *gt, i915_mcr_reg_t reg, |
578 | u32 clear, u32 set) |
579 | { |
580 | u32 val = intel_gt_mcr_read_any(gt, reg); |
581 | |
582 | intel_gt_mcr_multicast_write(gt, reg, value: (val & ~clear) | set); |
583 | |
584 | return val; |
585 | } |
586 | |
587 | /* |
588 | * reg_needs_read_steering - determine whether a register read requires |
589 | * explicit steering |
590 | * @gt: GT structure |
591 | * @reg: the register to check steering requirements for |
592 | * @type: type of multicast steering to check |
593 | * |
594 | * Determines whether @reg needs explicit steering of a specific type for |
595 | * reads. |
596 | * |
597 | * Returns false if @reg does not belong to a register range of the given |
598 | * steering type, or if the default (subslice-based) steering IDs are suitable |
599 | * for @type steering too. |
600 | */ |
601 | static bool reg_needs_read_steering(struct intel_gt *gt, |
602 | i915_mcr_reg_t reg, |
603 | enum intel_steering_type type) |
604 | { |
605 | u32 offset = i915_mmio_reg_offset(reg); |
606 | const struct intel_mmio_range *entry; |
607 | |
608 | if (likely(!gt->steering_table[type])) |
609 | return false; |
610 | |
611 | if (IS_GSI_REG(offset)) |
612 | offset += gt->uncore->gsi_offset; |
613 | |
614 | for (entry = gt->steering_table[type]; entry->end; entry++) { |
615 | if (offset >= entry->start && offset <= entry->end) |
616 | return true; |
617 | } |
618 | |
619 | return false; |
620 | } |
621 | |
622 | /* |
623 | * get_nonterminated_steering - determines valid IDs for a class of MCR steering |
624 | * @gt: GT structure |
625 | * @type: multicast register type |
626 | * @group: Group ID returned |
627 | * @instance: Instance ID returned |
628 | * |
629 | * Determines group and instance values that will steer reads of the specified |
630 | * MCR class to a non-terminated instance. |
631 | */ |
632 | static void get_nonterminated_steering(struct intel_gt *gt, |
633 | enum intel_steering_type type, |
634 | u8 *group, u8 *instance) |
635 | { |
636 | u32 dss; |
637 | |
638 | switch (type) { |
639 | case L3BANK: |
640 | *group = 0; /* unused */ |
641 | *instance = __ffs(gt->info.l3bank_mask); |
642 | break; |
643 | case MSLICE: |
644 | GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); |
645 | *group = __ffs(gt->info.mslice_mask); |
646 | *instance = 0; /* unused */ |
647 | break; |
648 | case LNCF: |
649 | /* |
650 | * An LNCF is always present if its mslice is present, so we |
651 | * can safely just steer to LNCF 0 in all cases. |
652 | */ |
653 | GEM_WARN_ON(!HAS_MSLICE_STEERING(gt->i915)); |
654 | *group = __ffs(gt->info.mslice_mask) << 1; |
655 | *instance = 0; /* unused */ |
656 | break; |
657 | case GAM: |
658 | *group = IS_DG2(gt->i915) ? 1 : 0; |
659 | *instance = 0; |
660 | break; |
661 | case DSS: |
662 | dss = intel_sseu_find_first_xehp_dss(sseu: >->info.sseu, groupsize: 0, groupnum: 0); |
663 | *group = dss / GEN_DSS_PER_GSLICE; |
664 | *instance = dss % GEN_DSS_PER_GSLICE; |
665 | break; |
666 | case INSTANCE0: |
667 | /* |
668 | * There are a lot of MCR types for which instance (0, 0) |
669 | * will always provide a non-terminated value. |
670 | */ |
671 | *group = 0; |
672 | *instance = 0; |
673 | break; |
674 | case OADDRM: |
675 | if ((VDBOX_MASK(gt) | VEBOX_MASK(gt) | gt->info.sfc_mask) & BIT(0)) |
676 | *group = 0; |
677 | else |
678 | *group = 1; |
679 | *instance = 0; |
680 | break; |
681 | default: |
682 | MISSING_CASE(type); |
683 | *group = 0; |
684 | *instance = 0; |
685 | } |
686 | } |
687 | |
688 | /** |
689 | * intel_gt_mcr_get_nonterminated_steering - find group/instance values that |
690 | * will steer a register to a non-terminated instance |
691 | * @gt: GT structure |
692 | * @reg: register for which the steering is required |
693 | * @group: return variable for group steering |
694 | * @instance: return variable for instance steering |
695 | * |
696 | * This function returns a group/instance pair that is guaranteed to work for |
697 | * read steering of the given register. Note that a value will be returned even |
698 | * if the register is not replicated and therefore does not actually require |
699 | * steering. |
700 | */ |
701 | void intel_gt_mcr_get_nonterminated_steering(struct intel_gt *gt, |
702 | i915_mcr_reg_t reg, |
703 | u8 *group, u8 *instance) |
704 | { |
705 | int type; |
706 | |
707 | for (type = 0; type < NUM_STEERING_TYPES; type++) { |
708 | if (reg_needs_read_steering(gt, reg, type)) { |
709 | get_nonterminated_steering(gt, type, group, instance); |
710 | return; |
711 | } |
712 | } |
713 | |
714 | *group = gt->default_steering.groupid; |
715 | *instance = gt->default_steering.instanceid; |
716 | } |
717 | |
718 | /** |
719 | * intel_gt_mcr_read_any_fw - reads one instance of an MCR register |
720 | * @gt: GT structure |
721 | * @reg: register to read |
722 | * |
723 | * Reads a GT MCR register. The read will be steered to a non-terminated |
724 | * instance (i.e., one that isn't fused off or powered down by power gating). |
725 | * This function assumes the caller is already holding any necessary forcewake |
726 | * domains; use intel_gt_mcr_read_any() in cases where forcewake should be |
727 | * obtained automatically. |
728 | * |
729 | * Context: The caller must hold gt->mcr_lock. |
730 | * |
731 | * Returns the value from a non-terminated instance of @reg. |
732 | */ |
733 | u32 intel_gt_mcr_read_any_fw(struct intel_gt *gt, i915_mcr_reg_t reg) |
734 | { |
735 | int type; |
736 | u8 group, instance; |
737 | |
738 | lockdep_assert_held(>->mcr_lock); |
739 | |
740 | for (type = 0; type < NUM_STEERING_TYPES; type++) { |
741 | if (reg_needs_read_steering(gt, reg, type)) { |
742 | get_nonterminated_steering(gt, type, group: &group, instance: &instance); |
743 | return rw_with_mcr_steering_fw(gt, reg, |
744 | FW_REG_READ, |
745 | group, instance, value: 0); |
746 | } |
747 | } |
748 | |
749 | return intel_uncore_read_fw(gt->uncore, mcr_reg_cast(reg)); |
750 | } |
751 | |
752 | /** |
753 | * intel_gt_mcr_read_any - reads one instance of an MCR register |
754 | * @gt: GT structure |
755 | * @reg: register to read |
756 | * |
757 | * Reads a GT MCR register. The read will be steered to a non-terminated |
758 | * instance (i.e., one that isn't fused off or powered down by power gating). |
759 | * |
760 | * Context: Calls a function that takes and releases gt->mcr_lock. |
761 | * |
762 | * Returns the value from a non-terminated instance of @reg. |
763 | */ |
764 | u32 intel_gt_mcr_read_any(struct intel_gt *gt, i915_mcr_reg_t reg) |
765 | { |
766 | int type; |
767 | u8 group, instance; |
768 | |
769 | for (type = 0; type < NUM_STEERING_TYPES; type++) { |
770 | if (reg_needs_read_steering(gt, reg, type)) { |
771 | get_nonterminated_steering(gt, type, group: &group, instance: &instance); |
772 | return rw_with_mcr_steering(gt, reg, |
773 | FW_REG_READ, |
774 | group, instance, value: 0); |
775 | } |
776 | } |
777 | |
778 | return intel_uncore_read(uncore: gt->uncore, reg: mcr_reg_cast(mcr: reg)); |
779 | } |
780 | |
781 | static void report_steering_type(struct drm_printer *p, |
782 | struct intel_gt *gt, |
783 | enum intel_steering_type type, |
784 | bool dump_table) |
785 | { |
786 | const struct intel_mmio_range *entry; |
787 | u8 group, instance; |
788 | |
789 | BUILD_BUG_ON(ARRAY_SIZE(intel_steering_types) != NUM_STEERING_TYPES); |
790 | |
791 | if (!gt->steering_table[type]) { |
792 | drm_printf(p, f: "%s steering: uses default steering\n" , |
793 | intel_steering_types[type]); |
794 | return; |
795 | } |
796 | |
797 | get_nonterminated_steering(gt, type, group: &group, instance: &instance); |
798 | drm_printf(p, f: "%s steering: group=0x%x, instance=0x%x\n" , |
799 | intel_steering_types[type], group, instance); |
800 | |
801 | if (!dump_table) |
802 | return; |
803 | |
804 | for (entry = gt->steering_table[type]; entry->end; entry++) |
805 | drm_printf(p, f: "\t0x%06x - 0x%06x\n" , entry->start, entry->end); |
806 | } |
807 | |
808 | void intel_gt_mcr_report_steering(struct drm_printer *p, struct intel_gt *gt, |
809 | bool dump_table) |
810 | { |
811 | /* |
812 | * Starting with MTL we no longer have default steering; |
813 | * all ranges are explicitly steered. |
814 | */ |
815 | if (GRAPHICS_VER_FULL(gt->i915) < IP_VER(12, 70)) |
816 | drm_printf(p, f: "Default steering: group=0x%x, instance=0x%x\n" , |
817 | gt->default_steering.groupid, |
818 | gt->default_steering.instanceid); |
819 | |
820 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 70)) { |
821 | for (int i = 0; i < NUM_STEERING_TYPES; i++) |
822 | if (gt->steering_table[i]) |
823 | report_steering_type(p, gt, type: i, dump_table); |
824 | } else if (IS_PONTEVECCHIO(gt->i915)) { |
825 | report_steering_type(p, gt, type: INSTANCE0, dump_table); |
826 | } else if (HAS_MSLICE_STEERING(gt->i915)) { |
827 | report_steering_type(p, gt, type: MSLICE, dump_table); |
828 | report_steering_type(p, gt, type: LNCF, dump_table); |
829 | } |
830 | } |
831 | |
832 | /** |
833 | * intel_gt_mcr_get_ss_steering - returns the group/instance steering for a SS |
834 | * @gt: GT structure |
835 | * @dss: DSS ID to obtain steering for |
836 | * @group: pointer to storage for steering group ID |
837 | * @instance: pointer to storage for steering instance ID |
838 | * |
839 | * Returns the steering IDs (via the @group and @instance parameters) that |
840 | * correspond to a specific subslice/DSS ID. |
841 | */ |
842 | void intel_gt_mcr_get_ss_steering(struct intel_gt *gt, unsigned int dss, |
843 | unsigned int *group, unsigned int *instance) |
844 | { |
845 | if (IS_PONTEVECCHIO(gt->i915)) { |
846 | *group = dss / GEN_DSS_PER_CSLICE; |
847 | *instance = dss % GEN_DSS_PER_CSLICE; |
848 | } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) { |
849 | *group = dss / GEN_DSS_PER_GSLICE; |
850 | *instance = dss % GEN_DSS_PER_GSLICE; |
851 | } else { |
852 | *group = dss / GEN_MAX_SS_PER_HSW_SLICE; |
853 | *instance = dss % GEN_MAX_SS_PER_HSW_SLICE; |
854 | return; |
855 | } |
856 | } |
857 | |
858 | /** |
859 | * intel_gt_mcr_wait_for_reg - wait until MCR register matches expected state |
860 | * @gt: GT structure |
861 | * @reg: the register to read |
862 | * @mask: mask to apply to register value |
863 | * @value: value to wait for |
864 | * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait |
865 | * @slow_timeout_ms: slow timeout in millisecond |
866 | * |
867 | * This routine waits until the target register @reg contains the expected |
868 | * @value after applying the @mask, i.e. it waits until :: |
869 | * |
870 | * (intel_gt_mcr_read_any_fw(gt, reg) & mask) == value |
871 | * |
872 | * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds. |
873 | * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us |
874 | * must be not larger than 20,0000 microseconds. |
875 | * |
876 | * This function is basically an MCR-friendly version of |
877 | * __intel_wait_for_register_fw(). Generally this function will only be used |
878 | * on GAM registers which are a bit special --- although they're MCR registers, |
879 | * reads (e.g., waiting for status updates) are always directed to the primary |
880 | * instance. |
881 | * |
882 | * Note that this routine assumes the caller holds forcewake asserted, it is |
883 | * not suitable for very long waits. |
884 | * |
885 | * Context: Calls a function that takes and releases gt->mcr_lock |
886 | * Return: 0 if the register matches the desired condition, or -ETIMEDOUT. |
887 | */ |
888 | int intel_gt_mcr_wait_for_reg(struct intel_gt *gt, |
889 | i915_mcr_reg_t reg, |
890 | u32 mask, |
891 | u32 value, |
892 | unsigned int fast_timeout_us, |
893 | unsigned int slow_timeout_ms) |
894 | { |
895 | int ret; |
896 | |
897 | lockdep_assert_not_held(>->mcr_lock); |
898 | |
899 | #define done ((intel_gt_mcr_read_any(gt, reg) & mask) == value) |
900 | |
901 | /* Catch any overuse of this function */ |
902 | might_sleep_if(slow_timeout_ms); |
903 | GEM_BUG_ON(fast_timeout_us > 20000); |
904 | GEM_BUG_ON(!fast_timeout_us && !slow_timeout_ms); |
905 | |
906 | ret = -ETIMEDOUT; |
907 | if (fast_timeout_us && fast_timeout_us <= 20000) |
908 | ret = _wait_for_atomic(done, fast_timeout_us, 0); |
909 | if (ret && slow_timeout_ms) |
910 | ret = wait_for(done, slow_timeout_ms); |
911 | |
912 | return ret; |
913 | #undef done |
914 | } |
915 | |