1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2015 Intel Corporation |
4 | */ |
5 | |
6 | #include "i915_drv.h" |
7 | |
8 | #include "intel_engine.h" |
9 | #include "intel_gt.h" |
10 | #include "intel_gt_mcr.h" |
11 | #include "intel_gt_regs.h" |
12 | #include "intel_mocs.h" |
13 | #include "intel_ring.h" |
14 | |
15 | /* structures required */ |
16 | struct drm_i915_mocs_entry { |
17 | u32 control_value; |
18 | u16 l3cc_value; |
19 | u16 used; |
20 | }; |
21 | |
22 | struct drm_i915_mocs_table { |
23 | unsigned int size; |
24 | unsigned int n_entries; |
25 | const struct drm_i915_mocs_entry *table; |
26 | u8 uc_index; |
27 | u8 wb_index; /* Only used on HAS_L3_CCS_READ() platforms */ |
28 | u8 unused_entries_index; |
29 | }; |
30 | |
31 | /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */ |
32 | #define _LE_CACHEABILITY(value) ((value) << 0) |
33 | #define _LE_TGT_CACHE(value) ((value) << 2) |
34 | #define LE_LRUM(value) ((value) << 4) |
35 | #define LE_AOM(value) ((value) << 6) |
36 | #define LE_RSC(value) ((value) << 7) |
37 | #define LE_SCC(value) ((value) << 8) |
38 | #define LE_PFM(value) ((value) << 11) |
39 | #define LE_SCF(value) ((value) << 14) |
40 | #define LE_COS(value) ((value) << 15) |
41 | #define LE_SSE(value) ((value) << 17) |
42 | |
43 | /* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */ |
44 | #define _L4_CACHEABILITY(value) ((value) << 2) |
45 | #define IG_PAT(value) ((value) << 8) |
46 | |
47 | /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */ |
48 | #define L3_ESC(value) ((value) << 0) |
49 | #define L3_SCC(value) ((value) << 1) |
50 | #define _L3_CACHEABILITY(value) ((value) << 4) |
51 | #define L3_GLBGO(value) ((value) << 6) |
52 | #define L3_LKUP(value) ((value) << 7) |
53 | |
54 | /* Helper defines */ |
55 | #define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ |
56 | #define PVC_NUM_MOCS_ENTRIES 3 |
57 | #define MTL_NUM_MOCS_ENTRIES 16 |
58 | |
59 | /* (e)LLC caching options */ |
60 | /* |
61 | * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means |
62 | * the same as LE_UC |
63 | */ |
64 | #define LE_0_PAGETABLE _LE_CACHEABILITY(0) |
65 | #define LE_1_UC _LE_CACHEABILITY(1) |
66 | #define LE_2_WT _LE_CACHEABILITY(2) |
67 | #define LE_3_WB _LE_CACHEABILITY(3) |
68 | |
69 | /* Target cache */ |
70 | #define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0) |
71 | #define LE_TC_1_LLC _LE_TGT_CACHE(1) |
72 | #define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2) |
73 | #define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3) |
74 | |
75 | /* L3 caching options */ |
76 | #define L3_0_DIRECT _L3_CACHEABILITY(0) |
77 | #define L3_1_UC _L3_CACHEABILITY(1) |
78 | #define L3_2_RESERVED _L3_CACHEABILITY(2) |
79 | #define L3_3_WB _L3_CACHEABILITY(3) |
80 | |
81 | /* L4 caching options */ |
82 | #define L4_0_WB _L4_CACHEABILITY(0) |
83 | #define L4_1_WT _L4_CACHEABILITY(1) |
84 | #define L4_2_RESERVED _L4_CACHEABILITY(2) |
85 | #define L4_3_UC _L4_CACHEABILITY(3) |
86 | |
87 | #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ |
88 | [__idx] = { \ |
89 | .control_value = __control_value, \ |
90 | .l3cc_value = __l3cc_value, \ |
91 | .used = 1, \ |
92 | } |
93 | |
94 | /* |
95 | * MOCS tables |
96 | * |
97 | * These are the MOCS tables that are programmed across all the rings. |
98 | * The control value is programmed to all the rings that support the |
99 | * MOCS registers. While the l3cc_values are only programmed to the |
100 | * LNCFCMOCS0 - LNCFCMOCS32 registers. |
101 | * |
102 | * These tables are intended to be kept reasonably consistent across |
103 | * HW platforms, and for ICL+, be identical across OSes. To achieve |
104 | * that, for Icelake and above, list of entries is published as part |
105 | * of bspec. |
106 | * |
107 | * Entries not part of the following tables are undefined as far as |
108 | * userspace is concerned and shouldn't be relied upon. For Gen < 12 |
109 | * they will be initialized to PTE. Gen >= 12 don't have a setting for |
110 | * PTE and those platforms except TGL/RKL will be initialized L3 WB to |
111 | * catch accidental use of reserved and unused mocs indexes. |
112 | * |
113 | * The last few entries are reserved by the hardware. For ICL+ they |
114 | * should be initialized according to bspec and never used, for older |
115 | * platforms they should never be written to. |
116 | * |
117 | * NOTE1: These tables are part of bspec and defined as part of hardware |
118 | * interface for ICL+. For older platforms, they are part of kernel |
119 | * ABI. It is expected that, for specific hardware platform, existing |
120 | * entries will remain constant and the table will only be updated by |
121 | * adding new entries, filling unused positions. |
122 | * |
123 | * NOTE2: For GEN >= 12 except TGL and RKL, reserved and unspecified MOCS |
124 | * indices have been set to L3 WB. These reserved entries should never |
125 | * be used, they may be changed to low performant variants with better |
126 | * coherency in the future if more entries are needed. |
127 | * For TGL/RKL, all the unspecified MOCS indexes are mapped to L3 UC. |
128 | */ |
129 | #define GEN9_MOCS_ENTRIES \ |
130 | MOCS_ENTRY(I915_MOCS_UNCACHED, \ |
131 | LE_1_UC | LE_TC_2_LLC_ELLC, \ |
132 | L3_1_UC), \ |
133 | MOCS_ENTRY(I915_MOCS_PTE, \ |
134 | LE_0_PAGETABLE | LE_TC_0_PAGETABLE | LE_LRUM(3), \ |
135 | L3_3_WB) |
136 | |
137 | static const struct drm_i915_mocs_entry skl_mocs_table[] = { |
138 | GEN9_MOCS_ENTRIES, |
139 | MOCS_ENTRY(I915_MOCS_CACHED, |
140 | LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3), |
141 | L3_3_WB), |
142 | |
143 | /* |
144 | * mocs:63 |
145 | * - used by the L3 for all of its evictions. |
146 | * Thus it is expected to allow LLC cacheability to enable coherent |
147 | * flows to be maintained. |
148 | * - used to force L3 uncachable cycles. |
149 | * Thus it is expected to make the surface L3 uncacheable. |
150 | */ |
151 | MOCS_ENTRY(63, |
152 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
153 | L3_1_UC) |
154 | }; |
155 | |
156 | /* NOTE: the LE_TGT_CACHE is not used on Broxton */ |
157 | static const struct drm_i915_mocs_entry broxton_mocs_table[] = { |
158 | GEN9_MOCS_ENTRIES, |
159 | MOCS_ENTRY(I915_MOCS_CACHED, |
160 | LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3), |
161 | L3_3_WB) |
162 | }; |
163 | |
164 | #define GEN11_MOCS_ENTRIES \ |
165 | /* Entries 0 and 1 are defined per-platform */ \ |
166 | /* Base - L3 + LLC */ \ |
167 | MOCS_ENTRY(2, \ |
168 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ |
169 | L3_3_WB), \ |
170 | /* Base - Uncached */ \ |
171 | MOCS_ENTRY(3, \ |
172 | LE_1_UC | LE_TC_1_LLC, \ |
173 | L3_1_UC), \ |
174 | /* Base - L3 */ \ |
175 | MOCS_ENTRY(4, \ |
176 | LE_1_UC | LE_TC_1_LLC, \ |
177 | L3_3_WB), \ |
178 | /* Base - LLC */ \ |
179 | MOCS_ENTRY(5, \ |
180 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ |
181 | L3_1_UC), \ |
182 | /* Age 0 - LLC */ \ |
183 | MOCS_ENTRY(6, \ |
184 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ |
185 | L3_1_UC), \ |
186 | /* Age 0 - L3 + LLC */ \ |
187 | MOCS_ENTRY(7, \ |
188 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ |
189 | L3_3_WB), \ |
190 | /* Age: Don't Chg. - LLC */ \ |
191 | MOCS_ENTRY(8, \ |
192 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ |
193 | L3_1_UC), \ |
194 | /* Age: Don't Chg. - L3 + LLC */ \ |
195 | MOCS_ENTRY(9, \ |
196 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ |
197 | L3_3_WB), \ |
198 | /* No AOM - LLC */ \ |
199 | MOCS_ENTRY(10, \ |
200 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ |
201 | L3_1_UC), \ |
202 | /* No AOM - L3 + LLC */ \ |
203 | MOCS_ENTRY(11, \ |
204 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ |
205 | L3_3_WB), \ |
206 | /* No AOM; Age 0 - LLC */ \ |
207 | MOCS_ENTRY(12, \ |
208 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ |
209 | L3_1_UC), \ |
210 | /* No AOM; Age 0 - L3 + LLC */ \ |
211 | MOCS_ENTRY(13, \ |
212 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ |
213 | L3_3_WB), \ |
214 | /* No AOM; Age:DC - LLC */ \ |
215 | MOCS_ENTRY(14, \ |
216 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ |
217 | L3_1_UC), \ |
218 | /* No AOM; Age:DC - L3 + LLC */ \ |
219 | MOCS_ENTRY(15, \ |
220 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ |
221 | L3_3_WB), \ |
222 | /* Bypass LLC - Uncached (EHL+) */ \ |
223 | MOCS_ENTRY(16, \ |
224 | LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \ |
225 | L3_1_UC), \ |
226 | /* Bypass LLC - L3 (Read-Only) (EHL+) */ \ |
227 | MOCS_ENTRY(17, \ |
228 | LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \ |
229 | L3_3_WB), \ |
230 | /* Self-Snoop - L3 + LLC */ \ |
231 | MOCS_ENTRY(18, \ |
232 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \ |
233 | L3_3_WB), \ |
234 | /* Skip Caching - L3 + LLC(12.5%) */ \ |
235 | MOCS_ENTRY(19, \ |
236 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \ |
237 | L3_3_WB), \ |
238 | /* Skip Caching - L3 + LLC(25%) */ \ |
239 | MOCS_ENTRY(20, \ |
240 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \ |
241 | L3_3_WB), \ |
242 | /* Skip Caching - L3 + LLC(50%) */ \ |
243 | MOCS_ENTRY(21, \ |
244 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \ |
245 | L3_3_WB), \ |
246 | /* Skip Caching - L3 + LLC(75%) */ \ |
247 | MOCS_ENTRY(22, \ |
248 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \ |
249 | L3_3_WB), \ |
250 | /* Skip Caching - L3 + LLC(87.5%) */ \ |
251 | MOCS_ENTRY(23, \ |
252 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \ |
253 | L3_3_WB), \ |
254 | /* HW Reserved - SW program but never use */ \ |
255 | MOCS_ENTRY(62, \ |
256 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ |
257 | L3_1_UC), \ |
258 | /* HW Reserved - SW program but never use */ \ |
259 | MOCS_ENTRY(63, \ |
260 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ |
261 | L3_1_UC) |
262 | |
263 | static const struct drm_i915_mocs_entry tgl_mocs_table[] = { |
264 | /* |
265 | * NOTE: |
266 | * Reserved and unspecified MOCS indices have been set to (L3 + LCC). |
267 | * These reserved entries should never be used, they may be changed |
268 | * to low performant variants with better coherency in the future if |
269 | * more entries are needed. We are programming index I915_MOCS_PTE(1) |
270 | * only, __init_mocs_table() take care to program unused index with |
271 | * this entry. |
272 | */ |
273 | MOCS_ENTRY(I915_MOCS_PTE, |
274 | LE_0_PAGETABLE | LE_TC_0_PAGETABLE, |
275 | L3_1_UC), |
276 | GEN11_MOCS_ENTRIES, |
277 | |
278 | /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ |
279 | MOCS_ENTRY(48, |
280 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
281 | L3_3_WB), |
282 | /* Implicitly enable L1 - HDC:L1 + L3 */ |
283 | MOCS_ENTRY(49, |
284 | LE_1_UC | LE_TC_1_LLC, |
285 | L3_3_WB), |
286 | /* Implicitly enable L1 - HDC:L1 + LLC */ |
287 | MOCS_ENTRY(50, |
288 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
289 | L3_1_UC), |
290 | /* Implicitly enable L1 - HDC:L1 */ |
291 | MOCS_ENTRY(51, |
292 | LE_1_UC | LE_TC_1_LLC, |
293 | L3_1_UC), |
294 | /* HW Special Case (CCS) */ |
295 | MOCS_ENTRY(60, |
296 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
297 | L3_1_UC), |
298 | /* HW Special Case (Displayable) */ |
299 | MOCS_ENTRY(61, |
300 | LE_1_UC | LE_TC_1_LLC, |
301 | L3_3_WB), |
302 | }; |
303 | |
304 | static const struct drm_i915_mocs_entry icl_mocs_table[] = { |
305 | /* Base - Uncached (Deprecated) */ |
306 | MOCS_ENTRY(I915_MOCS_UNCACHED, |
307 | LE_1_UC | LE_TC_1_LLC, |
308 | L3_1_UC), |
309 | /* Base - L3 + LeCC:PAT (Deprecated) */ |
310 | MOCS_ENTRY(I915_MOCS_PTE, |
311 | LE_0_PAGETABLE | LE_TC_0_PAGETABLE, |
312 | L3_3_WB), |
313 | |
314 | GEN11_MOCS_ENTRIES |
315 | }; |
316 | |
317 | static const struct drm_i915_mocs_entry dg1_mocs_table[] = { |
318 | |
319 | /* UC */ |
320 | MOCS_ENTRY(1, 0, L3_1_UC), |
321 | /* WB - L3 */ |
322 | MOCS_ENTRY(5, 0, L3_3_WB), |
323 | /* WB - L3 50% */ |
324 | MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB), |
325 | /* WB - L3 25% */ |
326 | MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB), |
327 | /* WB - L3 12.5% */ |
328 | MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB), |
329 | |
330 | /* HDC:L1 + L3 */ |
331 | MOCS_ENTRY(48, 0, L3_3_WB), |
332 | /* HDC:L1 */ |
333 | MOCS_ENTRY(49, 0, L3_1_UC), |
334 | |
335 | /* HW Reserved */ |
336 | MOCS_ENTRY(60, 0, L3_1_UC), |
337 | MOCS_ENTRY(61, 0, L3_1_UC), |
338 | MOCS_ENTRY(62, 0, L3_1_UC), |
339 | MOCS_ENTRY(63, 0, L3_1_UC), |
340 | }; |
341 | |
342 | static const struct drm_i915_mocs_entry gen12_mocs_table[] = { |
343 | GEN11_MOCS_ENTRIES, |
344 | /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ |
345 | MOCS_ENTRY(48, |
346 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
347 | L3_3_WB), |
348 | /* Implicitly enable L1 - HDC:L1 + L3 */ |
349 | MOCS_ENTRY(49, |
350 | LE_1_UC | LE_TC_1_LLC, |
351 | L3_3_WB), |
352 | /* Implicitly enable L1 - HDC:L1 + LLC */ |
353 | MOCS_ENTRY(50, |
354 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
355 | L3_1_UC), |
356 | /* Implicitly enable L1 - HDC:L1 */ |
357 | MOCS_ENTRY(51, |
358 | LE_1_UC | LE_TC_1_LLC, |
359 | L3_1_UC), |
360 | /* HW Special Case (CCS) */ |
361 | MOCS_ENTRY(60, |
362 | LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), |
363 | L3_1_UC), |
364 | /* HW Special Case (Displayable) */ |
365 | MOCS_ENTRY(61, |
366 | LE_1_UC | LE_TC_1_LLC, |
367 | L3_3_WB), |
368 | }; |
369 | |
370 | static const struct drm_i915_mocs_entry xehpsdv_mocs_table[] = { |
371 | /* wa_1608975824 */ |
372 | MOCS_ENTRY(0, 0, L3_3_WB | L3_LKUP(1)), |
373 | |
374 | /* UC - Coherent; GO:L3 */ |
375 | MOCS_ENTRY(1, 0, L3_1_UC | L3_LKUP(1)), |
376 | /* UC - Coherent; GO:Memory */ |
377 | MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), |
378 | /* UC - Non-Coherent; GO:Memory */ |
379 | MOCS_ENTRY(3, 0, L3_1_UC | L3_GLBGO(1)), |
380 | /* UC - Non-Coherent; GO:L3 */ |
381 | MOCS_ENTRY(4, 0, L3_1_UC), |
382 | |
383 | /* WB */ |
384 | MOCS_ENTRY(5, 0, L3_3_WB | L3_LKUP(1)), |
385 | |
386 | /* HW Reserved - SW program but never use. */ |
387 | MOCS_ENTRY(48, 0, L3_3_WB | L3_LKUP(1)), |
388 | MOCS_ENTRY(49, 0, L3_1_UC | L3_LKUP(1)), |
389 | MOCS_ENTRY(60, 0, L3_1_UC), |
390 | MOCS_ENTRY(61, 0, L3_1_UC), |
391 | MOCS_ENTRY(62, 0, L3_1_UC), |
392 | MOCS_ENTRY(63, 0, L3_1_UC), |
393 | }; |
394 | |
395 | static const struct drm_i915_mocs_entry dg2_mocs_table[] = { |
396 | /* UC - Coherent; GO:L3 */ |
397 | MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), |
398 | /* UC - Coherent; GO:Memory */ |
399 | MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), |
400 | /* UC - Non-Coherent; GO:Memory */ |
401 | MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), |
402 | |
403 | /* WB - LC */ |
404 | MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), |
405 | }; |
406 | |
407 | static const struct drm_i915_mocs_entry pvc_mocs_table[] = { |
408 | /* Error */ |
409 | MOCS_ENTRY(0, 0, L3_3_WB), |
410 | |
411 | /* UC */ |
412 | MOCS_ENTRY(1, 0, L3_1_UC), |
413 | |
414 | /* WB */ |
415 | MOCS_ENTRY(2, 0, L3_3_WB), |
416 | }; |
417 | |
418 | static const struct drm_i915_mocs_entry mtl_mocs_table[] = { |
419 | /* Error - Reserved for Non-Use */ |
420 | MOCS_ENTRY(0, |
421 | IG_PAT(0), |
422 | L3_LKUP(1) | L3_3_WB), |
423 | /* Cached - L3 + L4 */ |
424 | MOCS_ENTRY(1, |
425 | IG_PAT(1), |
426 | L3_LKUP(1) | L3_3_WB), |
427 | /* L4 - GO:L3 */ |
428 | MOCS_ENTRY(2, |
429 | IG_PAT(1), |
430 | L3_LKUP(1) | L3_1_UC), |
431 | /* Uncached - GO:L3 */ |
432 | MOCS_ENTRY(3, |
433 | IG_PAT(1) | L4_3_UC, |
434 | L3_LKUP(1) | L3_1_UC), |
435 | /* L4 - GO:Mem */ |
436 | MOCS_ENTRY(4, |
437 | IG_PAT(1), |
438 | L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), |
439 | /* Uncached - GO:Mem */ |
440 | MOCS_ENTRY(5, |
441 | IG_PAT(1) | L4_3_UC, |
442 | L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), |
443 | /* L4 - L3:NoLKUP; GO:L3 */ |
444 | MOCS_ENTRY(6, |
445 | IG_PAT(1), |
446 | L3_1_UC), |
447 | /* Uncached - L3:NoLKUP; GO:L3 */ |
448 | MOCS_ENTRY(7, |
449 | IG_PAT(1) | L4_3_UC, |
450 | L3_1_UC), |
451 | /* L4 - L3:NoLKUP; GO:Mem */ |
452 | MOCS_ENTRY(8, |
453 | IG_PAT(1), |
454 | L3_GLBGO(1) | L3_1_UC), |
455 | /* Uncached - L3:NoLKUP; GO:Mem */ |
456 | MOCS_ENTRY(9, |
457 | IG_PAT(1) | L4_3_UC, |
458 | L3_GLBGO(1) | L3_1_UC), |
459 | /* Display - L3; L4:WT */ |
460 | MOCS_ENTRY(14, |
461 | IG_PAT(1) | L4_1_WT, |
462 | L3_LKUP(1) | L3_3_WB), |
463 | /* CCS - Non-Displayable */ |
464 | MOCS_ENTRY(15, |
465 | IG_PAT(1), |
466 | L3_GLBGO(1) | L3_1_UC), |
467 | }; |
468 | |
469 | enum { |
470 | HAS_GLOBAL_MOCS = BIT(0), |
471 | HAS_ENGINE_MOCS = BIT(1), |
472 | HAS_RENDER_L3CC = BIT(2), |
473 | }; |
474 | |
475 | static bool has_l3cc(const struct drm_i915_private *i915) |
476 | { |
477 | return true; |
478 | } |
479 | |
480 | static bool has_global_mocs(const struct drm_i915_private *i915) |
481 | { |
482 | return HAS_GLOBAL_MOCS_REGISTERS(i915); |
483 | } |
484 | |
485 | static bool has_mocs(const struct drm_i915_private *i915) |
486 | { |
487 | return !IS_DGFX(i915); |
488 | } |
489 | |
490 | static unsigned int get_mocs_settings(struct drm_i915_private *i915, |
491 | struct drm_i915_mocs_table *table) |
492 | { |
493 | unsigned int flags; |
494 | |
495 | memset(table, 0, sizeof(struct drm_i915_mocs_table)); |
496 | |
497 | table->unused_entries_index = I915_MOCS_PTE; |
498 | if (IS_GFX_GT_IP_RANGE(to_gt(i915), IP_VER(12, 70), IP_VER(12, 74))) { |
499 | table->size = ARRAY_SIZE(mtl_mocs_table); |
500 | table->table = mtl_mocs_table; |
501 | table->n_entries = MTL_NUM_MOCS_ENTRIES; |
502 | table->uc_index = 9; |
503 | table->unused_entries_index = 1; |
504 | } else if (IS_PONTEVECCHIO(i915)) { |
505 | table->size = ARRAY_SIZE(pvc_mocs_table); |
506 | table->table = pvc_mocs_table; |
507 | table->n_entries = PVC_NUM_MOCS_ENTRIES; |
508 | table->uc_index = 1; |
509 | table->wb_index = 2; |
510 | table->unused_entries_index = 2; |
511 | } else if (IS_DG2(i915)) { |
512 | table->size = ARRAY_SIZE(dg2_mocs_table); |
513 | table->table = dg2_mocs_table; |
514 | table->uc_index = 1; |
515 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
516 | table->unused_entries_index = 3; |
517 | } else if (IS_XEHPSDV(i915)) { |
518 | table->size = ARRAY_SIZE(xehpsdv_mocs_table); |
519 | table->table = xehpsdv_mocs_table; |
520 | table->uc_index = 2; |
521 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
522 | table->unused_entries_index = 5; |
523 | } else if (IS_DG1(i915)) { |
524 | table->size = ARRAY_SIZE(dg1_mocs_table); |
525 | table->table = dg1_mocs_table; |
526 | table->uc_index = 1; |
527 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
528 | table->uc_index = 1; |
529 | table->unused_entries_index = 5; |
530 | } else if (IS_TIGERLAKE(i915) || IS_ROCKETLAKE(i915)) { |
531 | /* For TGL/RKL, Can't be changed now for ABI reasons */ |
532 | table->size = ARRAY_SIZE(tgl_mocs_table); |
533 | table->table = tgl_mocs_table; |
534 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
535 | table->uc_index = 3; |
536 | } else if (GRAPHICS_VER(i915) >= 12) { |
537 | table->size = ARRAY_SIZE(gen12_mocs_table); |
538 | table->table = gen12_mocs_table; |
539 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
540 | table->uc_index = 3; |
541 | table->unused_entries_index = 2; |
542 | } else if (GRAPHICS_VER(i915) == 11) { |
543 | table->size = ARRAY_SIZE(icl_mocs_table); |
544 | table->table = icl_mocs_table; |
545 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
546 | } else if (IS_GEN9_BC(i915)) { |
547 | table->size = ARRAY_SIZE(skl_mocs_table); |
548 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
549 | table->table = skl_mocs_table; |
550 | } else if (IS_GEN9_LP(i915)) { |
551 | table->size = ARRAY_SIZE(broxton_mocs_table); |
552 | table->n_entries = GEN9_NUM_MOCS_ENTRIES; |
553 | table->table = broxton_mocs_table; |
554 | } else { |
555 | drm_WARN_ONCE(&i915->drm, GRAPHICS_VER(i915) >= 9, |
556 | "Platform that should have a MOCS table does not.\n" ); |
557 | return 0; |
558 | } |
559 | |
560 | if (GEM_DEBUG_WARN_ON(table->size > table->n_entries)) |
561 | return 0; |
562 | |
563 | /* WaDisableSkipCaching:skl,bxt,kbl,glk */ |
564 | if (GRAPHICS_VER(i915) == 9) { |
565 | int i; |
566 | |
567 | for (i = 0; i < table->size; i++) |
568 | if (GEM_DEBUG_WARN_ON(table->table[i].l3cc_value & |
569 | (L3_ESC(1) | L3_SCC(0x7)))) |
570 | return 0; |
571 | } |
572 | |
573 | flags = 0; |
574 | if (has_mocs(i915)) { |
575 | if (has_global_mocs(i915)) |
576 | flags |= HAS_GLOBAL_MOCS; |
577 | else |
578 | flags |= HAS_ENGINE_MOCS; |
579 | } |
580 | if (has_l3cc(i915)) |
581 | flags |= HAS_RENDER_L3CC; |
582 | |
583 | return flags; |
584 | } |
585 | |
586 | /* |
587 | * Get control_value from MOCS entry taking into account when it's not used |
588 | * then if unused_entries_index is non-zero then its value will be returned |
589 | * otherwise I915_MOCS_PTE's value is returned in this case. |
590 | */ |
591 | static u32 get_entry_control(const struct drm_i915_mocs_table *table, |
592 | unsigned int index) |
593 | { |
594 | if (index < table->size && table->table[index].used) |
595 | return table->table[index].control_value; |
596 | return table->table[table->unused_entries_index].control_value; |
597 | } |
598 | |
599 | #define for_each_mocs(mocs, t, i) \ |
600 | for (i = 0; \ |
601 | i < (t)->n_entries ? (mocs = get_entry_control((t), i)), 1 : 0;\ |
602 | i++) |
603 | |
604 | static void __init_mocs_table(struct intel_uncore *uncore, |
605 | const struct drm_i915_mocs_table *table, |
606 | u32 addr) |
607 | { |
608 | unsigned int i; |
609 | u32 mocs; |
610 | |
611 | drm_WARN_ONCE(&uncore->i915->drm, !table->unused_entries_index, |
612 | "Unused entries index should have been defined\n" ); |
613 | for_each_mocs(mocs, table, i) |
614 | intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs); |
615 | } |
616 | |
617 | static u32 mocs_offset(const struct intel_engine_cs *engine) |
618 | { |
619 | static const u32 offset[] = { |
620 | [RCS0] = __GEN9_RCS0_MOCS0, |
621 | [VCS0] = __GEN9_VCS0_MOCS0, |
622 | [VCS1] = __GEN9_VCS1_MOCS0, |
623 | [VECS0] = __GEN9_VECS0_MOCS0, |
624 | [BCS0] = __GEN9_BCS0_MOCS0, |
625 | [VCS2] = __GEN11_VCS2_MOCS0, |
626 | }; |
627 | |
628 | GEM_BUG_ON(engine->id >= ARRAY_SIZE(offset)); |
629 | return offset[engine->id]; |
630 | } |
631 | |
632 | static void init_mocs_table(struct intel_engine_cs *engine, |
633 | const struct drm_i915_mocs_table *table) |
634 | { |
635 | __init_mocs_table(uncore: engine->uncore, table, addr: mocs_offset(engine)); |
636 | } |
637 | |
638 | /* |
639 | * Get l3cc_value from MOCS entry taking into account when it's not used |
640 | * then if unused_entries_index is not zero then its value will be returned |
641 | * otherwise I915_MOCS_PTE's value is returned in this case. |
642 | */ |
643 | static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table, |
644 | unsigned int index) |
645 | { |
646 | if (index < table->size && table->table[index].used) |
647 | return table->table[index].l3cc_value; |
648 | return table->table[table->unused_entries_index].l3cc_value; |
649 | } |
650 | |
651 | static u32 l3cc_combine(u16 low, u16 high) |
652 | { |
653 | return low | (u32)high << 16; |
654 | } |
655 | |
656 | #define for_each_l3cc(l3cc, t, i) \ |
657 | for (i = 0; \ |
658 | i < ((t)->n_entries + 1) / 2 ? \ |
659 | (l3cc = l3cc_combine(get_entry_l3cc((t), 2 * i), \ |
660 | get_entry_l3cc((t), 2 * i + 1))), 1 : \ |
661 | 0; \ |
662 | i++) |
663 | |
664 | static void init_l3cc_table(struct intel_gt *gt, |
665 | const struct drm_i915_mocs_table *table) |
666 | { |
667 | unsigned long flags; |
668 | unsigned int i; |
669 | u32 l3cc; |
670 | |
671 | intel_gt_mcr_lock(gt, flags: &flags); |
672 | for_each_l3cc(l3cc, table, i) |
673 | if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 50)) |
674 | intel_gt_mcr_multicast_write_fw(gt, XEHP_LNCFCMOCS(i), value: l3cc); |
675 | else |
676 | intel_uncore_write_fw(gt->uncore, GEN9_LNCFCMOCS(i), l3cc); |
677 | intel_gt_mcr_unlock(gt, flags); |
678 | } |
679 | |
680 | void intel_mocs_init_engine(struct intel_engine_cs *engine) |
681 | { |
682 | struct drm_i915_mocs_table table; |
683 | unsigned int flags; |
684 | |
685 | /* Called under a blanket forcewake */ |
686 | assert_forcewakes_active(uncore: engine->uncore, fw_domains: FORCEWAKE_ALL); |
687 | |
688 | flags = get_mocs_settings(i915: engine->i915, table: &table); |
689 | if (!flags) |
690 | return; |
691 | |
692 | /* Platforms with global MOCS do not need per-engine initialization. */ |
693 | if (flags & HAS_ENGINE_MOCS) |
694 | init_mocs_table(engine, table: &table); |
695 | |
696 | if (flags & HAS_RENDER_L3CC && engine->class == RENDER_CLASS) |
697 | init_l3cc_table(gt: engine->gt, table: &table); |
698 | } |
699 | |
700 | static u32 global_mocs_offset(void) |
701 | { |
702 | return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0)); |
703 | } |
704 | |
705 | void intel_set_mocs_index(struct intel_gt *gt) |
706 | { |
707 | struct drm_i915_mocs_table table; |
708 | |
709 | get_mocs_settings(i915: gt->i915, table: &table); |
710 | gt->mocs.uc_index = table.uc_index; |
711 | if (HAS_L3_CCS_READ(gt->i915)) |
712 | gt->mocs.wb_index = table.wb_index; |
713 | } |
714 | |
715 | void intel_mocs_init(struct intel_gt *gt) |
716 | { |
717 | struct drm_i915_mocs_table table; |
718 | unsigned int flags; |
719 | |
720 | /* |
721 | * LLC and eDRAM control values are not applicable to dgfx |
722 | */ |
723 | flags = get_mocs_settings(i915: gt->i915, table: &table); |
724 | if (flags & HAS_GLOBAL_MOCS) |
725 | __init_mocs_table(uncore: gt->uncore, table: &table, addr: global_mocs_offset()); |
726 | |
727 | /* |
728 | * Initialize the L3CC table as part of mocs initalization to make |
729 | * sure the LNCFCMOCSx registers are programmed for the subsequent |
730 | * memory transactions including guc transactions |
731 | */ |
732 | if (flags & HAS_RENDER_L3CC) |
733 | init_l3cc_table(gt, table: &table); |
734 | } |
735 | |
736 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
737 | #include "selftest_mocs.c" |
738 | #endif |
739 | |