1 | /* |
2 | * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
21 | * SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Kevin Tian <kevin.tian@intel.com> |
25 | * Eddie Dong <eddie.dong@intel.com> |
26 | * Zhiyuan Lv <zhiyuan.lv@intel.com> |
27 | * |
28 | * Contributors: |
29 | * Min He <min.he@intel.com> |
30 | * Tina Zhang <tina.zhang@intel.com> |
31 | * Pei Zhang <pei.zhang@intel.com> |
32 | * Niu Bing <bing.niu@intel.com> |
33 | * Ping Gao <ping.a.gao@intel.com> |
34 | * Zhi Wang <zhi.a.wang@intel.com> |
35 | * |
36 | |
37 | */ |
38 | |
39 | #include "i915_drv.h" |
40 | #include "i915_reg.h" |
41 | #include "gvt.h" |
42 | #include "i915_pvinfo.h" |
43 | #include "intel_mchbar_regs.h" |
44 | #include "display/intel_display_types.h" |
45 | #include "display/intel_dmc_regs.h" |
46 | #include "display/intel_dp_aux_regs.h" |
47 | #include "display/intel_dpio_phy.h" |
48 | #include "display/intel_fbc.h" |
49 | #include "display/intel_fdi_regs.h" |
50 | #include "display/intel_pps_regs.h" |
51 | #include "display/intel_psr_regs.h" |
52 | #include "display/skl_watermark_regs.h" |
53 | #include "display/vlv_dsi_pll_regs.h" |
54 | #include "gt/intel_gt_regs.h" |
55 | |
56 | /* XXX FIXME i915 has changed PP_XXX definition */ |
57 | #define PCH_PP_STATUS _MMIO(0xc7200) |
58 | #define PCH_PP_CONTROL _MMIO(0xc7204) |
59 | #define PCH_PP_ON_DELAYS _MMIO(0xc7208) |
60 | #define PCH_PP_OFF_DELAYS _MMIO(0xc720c) |
61 | #define PCH_PP_DIVISOR _MMIO(0xc7210) |
62 | |
63 | unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt) |
64 | { |
65 | struct drm_i915_private *i915 = gvt->gt->i915; |
66 | |
67 | if (IS_BROADWELL(i915)) |
68 | return D_BDW; |
69 | else if (IS_SKYLAKE(i915)) |
70 | return D_SKL; |
71 | else if (IS_KABYLAKE(i915)) |
72 | return D_KBL; |
73 | else if (IS_BROXTON(i915)) |
74 | return D_BXT; |
75 | else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) |
76 | return D_CFL; |
77 | |
78 | return 0; |
79 | } |
80 | |
81 | static bool intel_gvt_match_device(struct intel_gvt *gvt, |
82 | unsigned long device) |
83 | { |
84 | return intel_gvt_get_device_type(gvt) & device; |
85 | } |
86 | |
87 | static void read_vreg(struct intel_vgpu *vgpu, unsigned int offset, |
88 | void *p_data, unsigned int bytes) |
89 | { |
90 | memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes); |
91 | } |
92 | |
93 | static void write_vreg(struct intel_vgpu *vgpu, unsigned int offset, |
94 | void *p_data, unsigned int bytes) |
95 | { |
96 | memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes); |
97 | } |
98 | |
99 | struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt, |
100 | unsigned int offset) |
101 | { |
102 | struct intel_gvt_mmio_info *e; |
103 | |
104 | hash_for_each_possible(gvt->mmio.mmio_info_table, e, node, offset) { |
105 | if (e->offset == offset) |
106 | return e; |
107 | } |
108 | return NULL; |
109 | } |
110 | |
111 | static int setup_mmio_info(struct intel_gvt *gvt, u32 offset, u32 size, |
112 | u16 flags, u32 addr_mask, u32 ro_mask, u32 device, |
113 | gvt_mmio_func read, gvt_mmio_func write) |
114 | { |
115 | struct intel_gvt_mmio_info *p; |
116 | u32 start, end, i; |
117 | |
118 | if (!intel_gvt_match_device(gvt, device)) |
119 | return 0; |
120 | |
121 | if (WARN_ON(!IS_ALIGNED(offset, 4))) |
122 | return -EINVAL; |
123 | |
124 | start = offset; |
125 | end = offset + size; |
126 | |
127 | for (i = start; i < end; i += 4) { |
128 | p = intel_gvt_find_mmio_info(gvt, offset: i); |
129 | if (!p) { |
130 | WARN(1, "assign a handler to a non-tracked mmio %x\n" , |
131 | i); |
132 | return -ENODEV; |
133 | } |
134 | p->ro_mask = ro_mask; |
135 | gvt->mmio.mmio_attribute[i / 4] = flags; |
136 | if (read) |
137 | p->read = read; |
138 | if (write) |
139 | p->write = write; |
140 | } |
141 | return 0; |
142 | } |
143 | |
144 | /** |
145 | * intel_gvt_render_mmio_to_engine - convert a mmio offset into the engine |
146 | * @gvt: a GVT device |
147 | * @offset: register offset |
148 | * |
149 | * Returns: |
150 | * The engine containing the offset within its mmio page. |
151 | */ |
152 | const struct intel_engine_cs * |
153 | intel_gvt_render_mmio_to_engine(struct intel_gvt *gvt, unsigned int offset) |
154 | { |
155 | struct intel_engine_cs *engine; |
156 | enum intel_engine_id id; |
157 | |
158 | offset &= ~GENMASK(11, 0); |
159 | for_each_engine(engine, gvt->gt, id) |
160 | if (engine->mmio_base == offset) |
161 | return engine; |
162 | |
163 | return NULL; |
164 | } |
165 | |
166 | #define offset_to_fence_num(offset) \ |
167 | ((offset - i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) >> 3) |
168 | |
169 | #define fence_num_to_offset(num) \ |
170 | (num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) |
171 | |
172 | |
173 | void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason) |
174 | { |
175 | switch (reason) { |
176 | case GVT_FAILSAFE_UNSUPPORTED_GUEST: |
177 | pr_err("Detected your guest driver doesn't support GVT-g.\n" ); |
178 | break; |
179 | case GVT_FAILSAFE_INSUFFICIENT_RESOURCE: |
180 | pr_err("Graphics resource is not enough for the guest\n" ); |
181 | break; |
182 | case GVT_FAILSAFE_GUEST_ERR: |
183 | pr_err("GVT Internal error for the guest\n" ); |
184 | break; |
185 | default: |
186 | break; |
187 | } |
188 | pr_err("Now vgpu %d will enter failsafe mode.\n" , vgpu->id); |
189 | vgpu->failsafe = true; |
190 | } |
191 | |
192 | static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu, |
193 | unsigned int fence_num, void *p_data, unsigned int bytes) |
194 | { |
195 | unsigned int max_fence = vgpu_fence_sz(vgpu); |
196 | |
197 | if (fence_num >= max_fence) { |
198 | gvt_vgpu_err("access oob fence reg %d/%d\n" , |
199 | fence_num, max_fence); |
200 | |
201 | /* When guest access oob fence regs without access |
202 | * pv_info first, we treat guest not supporting GVT, |
203 | * and we will let vgpu enter failsafe mode. |
204 | */ |
205 | if (!vgpu->pv_notified) |
206 | enter_failsafe_mode(vgpu, |
207 | reason: GVT_FAILSAFE_UNSUPPORTED_GUEST); |
208 | |
209 | memset(p_data, 0, bytes); |
210 | return -EINVAL; |
211 | } |
212 | return 0; |
213 | } |
214 | |
215 | static int gamw_echo_dev_rw_ia_write(struct intel_vgpu *vgpu, |
216 | unsigned int offset, void *p_data, unsigned int bytes) |
217 | { |
218 | u32 ips = (*(u32 *)p_data) & GAMW_ECO_ENABLE_64K_IPS_FIELD; |
219 | |
220 | if (GRAPHICS_VER(vgpu->gvt->gt->i915) <= 10) { |
221 | if (ips == GAMW_ECO_ENABLE_64K_IPS_FIELD) |
222 | gvt_dbg_core("vgpu%d: ips enabled\n" , vgpu->id); |
223 | else if (!ips) |
224 | gvt_dbg_core("vgpu%d: ips disabled\n" , vgpu->id); |
225 | else { |
226 | /* All engines must be enabled together for vGPU, |
227 | * since we don't know which engine the ppgtt will |
228 | * bind to when shadowing. |
229 | */ |
230 | gvt_vgpu_err("Unsupported IPS setting %x, cannot enable 64K gtt.\n" , |
231 | ips); |
232 | return -EINVAL; |
233 | } |
234 | } |
235 | |
236 | write_vreg(vgpu, offset, p_data, bytes); |
237 | return 0; |
238 | } |
239 | |
240 | static int fence_mmio_read(struct intel_vgpu *vgpu, unsigned int off, |
241 | void *p_data, unsigned int bytes) |
242 | { |
243 | int ret; |
244 | |
245 | ret = sanitize_fence_mmio_access(vgpu, offset_to_fence_num(off), |
246 | p_data, bytes); |
247 | if (ret) |
248 | return ret; |
249 | read_vreg(vgpu, offset: off, p_data, bytes); |
250 | return 0; |
251 | } |
252 | |
253 | static int fence_mmio_write(struct intel_vgpu *vgpu, unsigned int off, |
254 | void *p_data, unsigned int bytes) |
255 | { |
256 | struct intel_gvt *gvt = vgpu->gvt; |
257 | unsigned int fence_num = offset_to_fence_num(off); |
258 | int ret; |
259 | |
260 | ret = sanitize_fence_mmio_access(vgpu, fence_num, p_data, bytes); |
261 | if (ret) |
262 | return ret; |
263 | write_vreg(vgpu, offset: off, p_data, bytes); |
264 | |
265 | mmio_hw_access_pre(gt: gvt->gt); |
266 | intel_vgpu_write_fence(vgpu, fence: fence_num, |
267 | vgpu_vreg64(vgpu, fence_num_to_offset(fence_num))); |
268 | mmio_hw_access_post(gt: gvt->gt); |
269 | return 0; |
270 | } |
271 | |
272 | #define CALC_MODE_MASK_REG(old, new) \ |
273 | (((new) & GENMASK(31, 16)) \ |
274 | | ((((old) & GENMASK(15, 0)) & ~((new) >> 16)) \ |
275 | | ((new) & ((new) >> 16)))) |
276 | |
277 | static int mul_force_wake_write(struct intel_vgpu *vgpu, |
278 | unsigned int offset, void *p_data, unsigned int bytes) |
279 | { |
280 | u32 old, new; |
281 | u32 ack_reg_offset; |
282 | |
283 | old = vgpu_vreg(vgpu, offset); |
284 | new = CALC_MODE_MASK_REG(old, *(u32 *)p_data); |
285 | |
286 | if (GRAPHICS_VER(vgpu->gvt->gt->i915) >= 9) { |
287 | switch (offset) { |
288 | case FORCEWAKE_RENDER_GEN9_REG: |
289 | ack_reg_offset = FORCEWAKE_ACK_RENDER_GEN9_REG; |
290 | break; |
291 | case FORCEWAKE_GT_GEN9_REG: |
292 | ack_reg_offset = FORCEWAKE_ACK_GT_GEN9_REG; |
293 | break; |
294 | case FORCEWAKE_MEDIA_GEN9_REG: |
295 | ack_reg_offset = FORCEWAKE_ACK_MEDIA_GEN9_REG; |
296 | break; |
297 | default: |
298 | /*should not hit here*/ |
299 | gvt_vgpu_err("invalid forcewake offset 0x%x\n" , offset); |
300 | return -EINVAL; |
301 | } |
302 | } else { |
303 | ack_reg_offset = FORCEWAKE_ACK_HSW_REG; |
304 | } |
305 | |
306 | vgpu_vreg(vgpu, offset) = new; |
307 | vgpu_vreg(vgpu, ack_reg_offset) = (new & GENMASK(15, 0)); |
308 | return 0; |
309 | } |
310 | |
311 | static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
312 | void *p_data, unsigned int bytes) |
313 | { |
314 | intel_engine_mask_t engine_mask = 0; |
315 | u32 data; |
316 | |
317 | write_vreg(vgpu, offset, p_data, bytes); |
318 | data = vgpu_vreg(vgpu, offset); |
319 | |
320 | if (data & GEN6_GRDOM_FULL) { |
321 | gvt_dbg_mmio("vgpu%d: request full GPU reset\n" , vgpu->id); |
322 | engine_mask = ALL_ENGINES; |
323 | } else { |
324 | if (data & GEN6_GRDOM_RENDER) { |
325 | gvt_dbg_mmio("vgpu%d: request RCS reset\n" , vgpu->id); |
326 | engine_mask |= BIT(RCS0); |
327 | } |
328 | if (data & GEN6_GRDOM_MEDIA) { |
329 | gvt_dbg_mmio("vgpu%d: request VCS reset\n" , vgpu->id); |
330 | engine_mask |= BIT(VCS0); |
331 | } |
332 | if (data & GEN6_GRDOM_BLT) { |
333 | gvt_dbg_mmio("vgpu%d: request BCS Reset\n" , vgpu->id); |
334 | engine_mask |= BIT(BCS0); |
335 | } |
336 | if (data & GEN6_GRDOM_VECS) { |
337 | gvt_dbg_mmio("vgpu%d: request VECS Reset\n" , vgpu->id); |
338 | engine_mask |= BIT(VECS0); |
339 | } |
340 | if (data & GEN8_GRDOM_MEDIA2) { |
341 | gvt_dbg_mmio("vgpu%d: request VCS2 Reset\n" , vgpu->id); |
342 | engine_mask |= BIT(VCS1); |
343 | } |
344 | if (data & GEN9_GRDOM_GUC) { |
345 | gvt_dbg_mmio("vgpu%d: request GUC Reset\n" , vgpu->id); |
346 | vgpu_vreg_t(vgpu, GUC_STATUS) |= GS_MIA_IN_RESET; |
347 | } |
348 | engine_mask &= vgpu->gvt->gt->info.engine_mask; |
349 | } |
350 | |
351 | /* vgpu_lock already hold by emulate mmio r/w */ |
352 | intel_gvt_reset_vgpu_locked(vgpu, dmlr: false, engine_mask); |
353 | |
354 | /* sw will wait for the device to ack the reset request */ |
355 | vgpu_vreg(vgpu, offset) = 0; |
356 | |
357 | return 0; |
358 | } |
359 | |
360 | static int gmbus_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, |
361 | void *p_data, unsigned int bytes) |
362 | { |
363 | return intel_gvt_i2c_handle_gmbus_read(vgpu, offset, p_data, bytes); |
364 | } |
365 | |
366 | static int gmbus_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
367 | void *p_data, unsigned int bytes) |
368 | { |
369 | return intel_gvt_i2c_handle_gmbus_write(vgpu, offset, p_data, bytes); |
370 | } |
371 | |
372 | static int pch_pp_control_mmio_write(struct intel_vgpu *vgpu, |
373 | unsigned int offset, void *p_data, unsigned int bytes) |
374 | { |
375 | write_vreg(vgpu, offset, p_data, bytes); |
376 | |
377 | if (vgpu_vreg(vgpu, offset) & PANEL_POWER_ON) { |
378 | vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_ON; |
379 | vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE; |
380 | vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN; |
381 | vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE; |
382 | |
383 | } else |
384 | vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= |
385 | ~(PP_ON | PP_SEQUENCE_POWER_DOWN |
386 | | PP_CYCLE_DELAY_ACTIVE); |
387 | return 0; |
388 | } |
389 | |
390 | static int transconf_mmio_write(struct intel_vgpu *vgpu, |
391 | unsigned int offset, void *p_data, unsigned int bytes) |
392 | { |
393 | write_vreg(vgpu, offset, p_data, bytes); |
394 | |
395 | if (vgpu_vreg(vgpu, offset) & TRANS_ENABLE) |
396 | vgpu_vreg(vgpu, offset) |= TRANS_STATE_ENABLE; |
397 | else |
398 | vgpu_vreg(vgpu, offset) &= ~TRANS_STATE_ENABLE; |
399 | return 0; |
400 | } |
401 | |
402 | static int lcpll_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
403 | void *p_data, unsigned int bytes) |
404 | { |
405 | write_vreg(vgpu, offset, p_data, bytes); |
406 | |
407 | if (vgpu_vreg(vgpu, offset) & LCPLL_PLL_DISABLE) |
408 | vgpu_vreg(vgpu, offset) &= ~LCPLL_PLL_LOCK; |
409 | else |
410 | vgpu_vreg(vgpu, offset) |= LCPLL_PLL_LOCK; |
411 | |
412 | if (vgpu_vreg(vgpu, offset) & LCPLL_CD_SOURCE_FCLK) |
413 | vgpu_vreg(vgpu, offset) |= LCPLL_CD_SOURCE_FCLK_DONE; |
414 | else |
415 | vgpu_vreg(vgpu, offset) &= ~LCPLL_CD_SOURCE_FCLK_DONE; |
416 | |
417 | return 0; |
418 | } |
419 | |
420 | static int dpy_reg_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, |
421 | void *p_data, unsigned int bytes) |
422 | { |
423 | switch (offset) { |
424 | case 0xe651c: |
425 | case 0xe661c: |
426 | case 0xe671c: |
427 | case 0xe681c: |
428 | vgpu_vreg(vgpu, offset) = 1 << 17; |
429 | break; |
430 | case 0xe6c04: |
431 | vgpu_vreg(vgpu, offset) = 0x3; |
432 | break; |
433 | case 0xe6e1c: |
434 | vgpu_vreg(vgpu, offset) = 0x2f << 16; |
435 | break; |
436 | default: |
437 | return -EINVAL; |
438 | } |
439 | |
440 | read_vreg(vgpu, offset, p_data, bytes); |
441 | return 0; |
442 | } |
443 | |
444 | /* |
445 | * Only PIPE_A is enabled in current vGPU display and PIPE_A is tied to |
446 | * TRANSCODER_A in HW. DDI/PORT could be PORT_x depends on |
447 | * setup_virtual_dp_monitor(). |
448 | * emulate_monitor_status_change() set up PLL for PORT_x as the initial enabled |
449 | * DPLL. Later guest driver may setup a different DPLLx when setting mode. |
450 | * So the correct sequence to find DP stream clock is: |
451 | * Check TRANS_DDI_FUNC_CTL on TRANSCODER_A to get PORT_x. |
452 | * Check correct PLLx for PORT_x to get PLL frequency and DP bitrate. |
453 | * Then Refresh rate then can be calculated based on follow equations: |
454 | * Pixel clock = h_total * v_total * refresh_rate |
455 | * stream clock = Pixel clock |
456 | * ls_clk = DP bitrate |
457 | * Link M/N = strm_clk / ls_clk |
458 | */ |
459 | |
460 | static u32 bdw_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port) |
461 | { |
462 | u32 dp_br = 0; |
463 | u32 ddi_pll_sel = vgpu_vreg_t(vgpu, PORT_CLK_SEL(port)); |
464 | |
465 | switch (ddi_pll_sel) { |
466 | case PORT_CLK_SEL_LCPLL_2700: |
467 | dp_br = 270000 * 2; |
468 | break; |
469 | case PORT_CLK_SEL_LCPLL_1350: |
470 | dp_br = 135000 * 2; |
471 | break; |
472 | case PORT_CLK_SEL_LCPLL_810: |
473 | dp_br = 81000 * 2; |
474 | break; |
475 | case PORT_CLK_SEL_SPLL: |
476 | { |
477 | switch (vgpu_vreg_t(vgpu, SPLL_CTL) & SPLL_FREQ_MASK) { |
478 | case SPLL_FREQ_810MHz: |
479 | dp_br = 81000 * 2; |
480 | break; |
481 | case SPLL_FREQ_1350MHz: |
482 | dp_br = 135000 * 2; |
483 | break; |
484 | case SPLL_FREQ_2700MHz: |
485 | dp_br = 270000 * 2; |
486 | break; |
487 | default: |
488 | gvt_dbg_dpy("vgpu-%d PORT_%c can't get freq from SPLL 0x%08x\n" , |
489 | vgpu->id, port_name(port), vgpu_vreg_t(vgpu, SPLL_CTL)); |
490 | break; |
491 | } |
492 | break; |
493 | } |
494 | case PORT_CLK_SEL_WRPLL1: |
495 | case PORT_CLK_SEL_WRPLL2: |
496 | { |
497 | u32 wrpll_ctl; |
498 | int refclk, n, p, r; |
499 | |
500 | if (ddi_pll_sel == PORT_CLK_SEL_WRPLL1) |
501 | wrpll_ctl = vgpu_vreg_t(vgpu, WRPLL_CTL(DPLL_ID_WRPLL1)); |
502 | else |
503 | wrpll_ctl = vgpu_vreg_t(vgpu, WRPLL_CTL(DPLL_ID_WRPLL2)); |
504 | |
505 | switch (wrpll_ctl & WRPLL_REF_MASK) { |
506 | case WRPLL_REF_PCH_SSC: |
507 | refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.ssc; |
508 | break; |
509 | case WRPLL_REF_LCPLL: |
510 | refclk = 2700000; |
511 | break; |
512 | default: |
513 | gvt_dbg_dpy("vgpu-%d PORT_%c WRPLL can't get refclk 0x%08x\n" , |
514 | vgpu->id, port_name(port), wrpll_ctl); |
515 | goto out; |
516 | } |
517 | |
518 | r = wrpll_ctl & WRPLL_DIVIDER_REF_MASK; |
519 | p = (wrpll_ctl & WRPLL_DIVIDER_POST_MASK) >> WRPLL_DIVIDER_POST_SHIFT; |
520 | n = (wrpll_ctl & WRPLL_DIVIDER_FB_MASK) >> WRPLL_DIVIDER_FB_SHIFT; |
521 | |
522 | dp_br = (refclk * n / 10) / (p * r) * 2; |
523 | break; |
524 | } |
525 | default: |
526 | gvt_dbg_dpy("vgpu-%d PORT_%c has invalid clock select 0x%08x\n" , |
527 | vgpu->id, port_name(port), vgpu_vreg_t(vgpu, PORT_CLK_SEL(port))); |
528 | break; |
529 | } |
530 | |
531 | out: |
532 | return dp_br; |
533 | } |
534 | |
535 | static u32 bxt_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port) |
536 | { |
537 | u32 dp_br = 0; |
538 | int refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.nssc; |
539 | enum dpio_phy phy = DPIO_PHY0; |
540 | enum dpio_channel ch = DPIO_CH0; |
541 | struct dpll clock = {}; |
542 | u32 temp; |
543 | |
544 | /* Port to PHY mapping is fixed, see bxt_ddi_phy_info{} */ |
545 | switch (port) { |
546 | case PORT_A: |
547 | phy = DPIO_PHY1; |
548 | ch = DPIO_CH0; |
549 | break; |
550 | case PORT_B: |
551 | phy = DPIO_PHY0; |
552 | ch = DPIO_CH0; |
553 | break; |
554 | case PORT_C: |
555 | phy = DPIO_PHY0; |
556 | ch = DPIO_CH1; |
557 | break; |
558 | default: |
559 | gvt_dbg_dpy("vgpu-%d no PHY for PORT_%c\n" , vgpu->id, port_name(port)); |
560 | goto out; |
561 | } |
562 | |
563 | temp = vgpu_vreg_t(vgpu, BXT_PORT_PLL_ENABLE(port)); |
564 | if (!(temp & PORT_PLL_ENABLE) || !(temp & PORT_PLL_LOCK)) { |
565 | gvt_dbg_dpy("vgpu-%d PORT_%c PLL_ENABLE 0x%08x isn't enabled or locked\n" , |
566 | vgpu->id, port_name(port), temp); |
567 | goto out; |
568 | } |
569 | |
570 | clock.m1 = 2; |
571 | clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, |
572 | vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 0))) << 22; |
573 | if (vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 3)) & PORT_PLL_M2_FRAC_ENABLE) |
574 | clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, |
575 | vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 2))); |
576 | clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, |
577 | vgpu_vreg_t(vgpu, BXT_PORT_PLL(phy, ch, 1))); |
578 | clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, |
579 | vgpu_vreg_t(vgpu, BXT_PORT_PLL_EBB_0(phy, ch))); |
580 | clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, |
581 | vgpu_vreg_t(vgpu, BXT_PORT_PLL_EBB_0(phy, ch))); |
582 | clock.m = clock.m1 * clock.m2; |
583 | clock.p = clock.p1 * clock.p2 * 5; |
584 | |
585 | if (clock.n == 0 || clock.p == 0) { |
586 | gvt_dbg_dpy("vgpu-%d PORT_%c PLL has invalid divider\n" , vgpu->id, port_name(port)); |
587 | goto out; |
588 | } |
589 | |
590 | clock.vco = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(refclk, clock.m), clock.n << 22); |
591 | clock.dot = DIV_ROUND_CLOSEST(clock.vco, clock.p); |
592 | |
593 | dp_br = clock.dot; |
594 | |
595 | out: |
596 | return dp_br; |
597 | } |
598 | |
599 | static u32 skl_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port) |
600 | { |
601 | u32 dp_br = 0; |
602 | enum intel_dpll_id dpll_id = DPLL_ID_SKL_DPLL0; |
603 | |
604 | /* Find the enabled DPLL for the DDI/PORT */ |
605 | if (!(vgpu_vreg_t(vgpu, DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_OFF(port)) && |
606 | (vgpu_vreg_t(vgpu, DPLL_CTRL2) & DPLL_CTRL2_DDI_SEL_OVERRIDE(port))) { |
607 | dpll_id += (vgpu_vreg_t(vgpu, DPLL_CTRL2) & |
608 | DPLL_CTRL2_DDI_CLK_SEL_MASK(port)) >> |
609 | DPLL_CTRL2_DDI_CLK_SEL_SHIFT(port); |
610 | } else { |
611 | gvt_dbg_dpy("vgpu-%d DPLL for PORT_%c isn't turned on\n" , |
612 | vgpu->id, port_name(port)); |
613 | return dp_br; |
614 | } |
615 | |
616 | /* Find PLL output frequency from correct DPLL, and get bir rate */ |
617 | switch ((vgpu_vreg_t(vgpu, DPLL_CTRL1) & |
618 | DPLL_CTRL1_LINK_RATE_MASK(dpll_id)) >> |
619 | DPLL_CTRL1_LINK_RATE_SHIFT(dpll_id)) { |
620 | case DPLL_CTRL1_LINK_RATE_810: |
621 | dp_br = 81000 * 2; |
622 | break; |
623 | case DPLL_CTRL1_LINK_RATE_1080: |
624 | dp_br = 108000 * 2; |
625 | break; |
626 | case DPLL_CTRL1_LINK_RATE_1350: |
627 | dp_br = 135000 * 2; |
628 | break; |
629 | case DPLL_CTRL1_LINK_RATE_1620: |
630 | dp_br = 162000 * 2; |
631 | break; |
632 | case DPLL_CTRL1_LINK_RATE_2160: |
633 | dp_br = 216000 * 2; |
634 | break; |
635 | case DPLL_CTRL1_LINK_RATE_2700: |
636 | dp_br = 270000 * 2; |
637 | break; |
638 | default: |
639 | dp_br = 0; |
640 | gvt_dbg_dpy("vgpu-%d PORT_%c fail to get DPLL-%d freq\n" , |
641 | vgpu->id, port_name(port), dpll_id); |
642 | } |
643 | |
644 | return dp_br; |
645 | } |
646 | |
647 | static void vgpu_update_refresh_rate(struct intel_vgpu *vgpu) |
648 | { |
649 | struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; |
650 | enum port port; |
651 | u32 dp_br, link_m, link_n, htotal, vtotal; |
652 | |
653 | /* Find DDI/PORT assigned to TRANSCODER_A, expect B or D */ |
654 | port = (vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) & |
655 | TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT; |
656 | if (port != PORT_B && port != PORT_D) { |
657 | gvt_dbg_dpy("vgpu-%d unsupported PORT_%c\n" , vgpu->id, port_name(port)); |
658 | return; |
659 | } |
660 | |
661 | /* Calculate DP bitrate from PLL */ |
662 | if (IS_BROADWELL(dev_priv)) |
663 | dp_br = bdw_vgpu_get_dp_bitrate(vgpu, port); |
664 | else if (IS_BROXTON(dev_priv)) |
665 | dp_br = bxt_vgpu_get_dp_bitrate(vgpu, port); |
666 | else |
667 | dp_br = skl_vgpu_get_dp_bitrate(vgpu, port); |
668 | |
669 | /* Get DP link symbol clock M/N */ |
670 | link_m = vgpu_vreg_t(vgpu, PIPE_LINK_M1(TRANSCODER_A)); |
671 | link_n = vgpu_vreg_t(vgpu, PIPE_LINK_N1(TRANSCODER_A)); |
672 | |
673 | /* Get H/V total from transcoder timing */ |
674 | htotal = (vgpu_vreg_t(vgpu, TRANS_HTOTAL(TRANSCODER_A)) >> TRANS_HTOTAL_SHIFT); |
675 | vtotal = (vgpu_vreg_t(vgpu, TRANS_VTOTAL(TRANSCODER_A)) >> TRANS_VTOTAL_SHIFT); |
676 | |
677 | if (dp_br && link_n && htotal && vtotal) { |
678 | u64 pixel_clk = 0; |
679 | u32 new_rate = 0; |
680 | u32 *old_rate = &(intel_vgpu_port(vgpu, vgpu->display.port_num)->vrefresh_k); |
681 | |
682 | /* Calcuate pixel clock by (ls_clk * M / N) */ |
683 | pixel_clk = div_u64(dividend: mul_u32_u32(a: link_m, b: dp_br), divisor: link_n); |
684 | pixel_clk *= MSEC_PER_SEC; |
685 | |
686 | /* Calcuate refresh rate by (pixel_clk / (h_total * v_total)) */ |
687 | new_rate = DIV64_U64_ROUND_CLOSEST(mul_u64_u32_shr(pixel_clk, MSEC_PER_SEC, 0), mul_u32_u32(htotal + 1, vtotal + 1)); |
688 | |
689 | if (*old_rate != new_rate) |
690 | *old_rate = new_rate; |
691 | |
692 | gvt_dbg_dpy("vgpu-%d PIPE_%c refresh rate updated to %d\n" , |
693 | vgpu->id, pipe_name(PIPE_A), new_rate); |
694 | } |
695 | } |
696 | |
697 | static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
698 | void *p_data, unsigned int bytes) |
699 | { |
700 | u32 data; |
701 | |
702 | write_vreg(vgpu, offset, p_data, bytes); |
703 | data = vgpu_vreg(vgpu, offset); |
704 | |
705 | if (data & TRANSCONF_ENABLE) { |
706 | vgpu_vreg(vgpu, offset) |= TRANSCONF_STATE_ENABLE; |
707 | vgpu_update_refresh_rate(vgpu); |
708 | vgpu_update_vblank_emulation(vgpu, turnon: true); |
709 | } else { |
710 | vgpu_vreg(vgpu, offset) &= ~TRANSCONF_STATE_ENABLE; |
711 | vgpu_update_vblank_emulation(vgpu, turnon: false); |
712 | } |
713 | return 0; |
714 | } |
715 | |
716 | /* sorted in ascending order */ |
717 | static i915_reg_t force_nonpriv_white_list[] = { |
718 | _MMIO(0xd80), |
719 | GEN9_CS_DEBUG_MODE1, //_MMIO(0x20ec) |
720 | GEN9_CTX_PREEMPT_REG,//_MMIO(0x2248) |
721 | CL_PRIMITIVES_COUNT, //_MMIO(0x2340) |
722 | PS_INVOCATION_COUNT, //_MMIO(0x2348) |
723 | PS_DEPTH_COUNT, //_MMIO(0x2350) |
724 | GEN8_CS_CHICKEN1,//_MMIO(0x2580) |
725 | _MMIO(0x2690), |
726 | _MMIO(0x2694), |
727 | _MMIO(0x2698), |
728 | _MMIO(0x2754), |
729 | _MMIO(0x28a0), |
730 | _MMIO(0x4de0), |
731 | _MMIO(0x4de4), |
732 | _MMIO(0x4dfc), |
733 | GEN7_COMMON_SLICE_CHICKEN1,//_MMIO(0x7010) |
734 | _MMIO(0x7014), |
735 | HDC_CHICKEN0,//_MMIO(0x7300) |
736 | GEN8_HDC_CHICKEN1,//_MMIO(0x7304) |
737 | _MMIO(0x7700), |
738 | _MMIO(0x7704), |
739 | _MMIO(0x7708), |
740 | _MMIO(0x770c), |
741 | _MMIO(0x83a8), |
742 | _MMIO(0xb110), |
743 | _MMIO(0xb118), |
744 | _MMIO(0xe100), |
745 | _MMIO(0xe18c), |
746 | _MMIO(0xe48c), |
747 | _MMIO(0xe5f4), |
748 | _MMIO(0x64844), |
749 | }; |
750 | |
751 | /* a simple bsearch */ |
752 | static inline bool in_whitelist(u32 reg) |
753 | { |
754 | int left = 0, right = ARRAY_SIZE(force_nonpriv_white_list); |
755 | i915_reg_t *array = force_nonpriv_white_list; |
756 | |
757 | while (left < right) { |
758 | int mid = (left + right)/2; |
759 | |
760 | if (reg > array[mid].reg) |
761 | left = mid + 1; |
762 | else if (reg < array[mid].reg) |
763 | right = mid; |
764 | else |
765 | return true; |
766 | } |
767 | return false; |
768 | } |
769 | |
770 | static int force_nonpriv_write(struct intel_vgpu *vgpu, |
771 | unsigned int offset, void *p_data, unsigned int bytes) |
772 | { |
773 | u32 reg_nonpriv = (*(u32 *)p_data) & REG_GENMASK(25, 2); |
774 | const struct intel_engine_cs *engine = |
775 | intel_gvt_render_mmio_to_engine(gvt: vgpu->gvt, offset); |
776 | |
777 | if (bytes != 4 || !IS_ALIGNED(offset, bytes) || !engine) { |
778 | gvt_err("vgpu(%d) Invalid FORCE_NONPRIV offset %x(%dB)\n" , |
779 | vgpu->id, offset, bytes); |
780 | return -EINVAL; |
781 | } |
782 | |
783 | if (!in_whitelist(reg: reg_nonpriv) && |
784 | reg_nonpriv != i915_mmio_reg_offset(RING_NOPID(engine->mmio_base))) { |
785 | gvt_err("vgpu(%d) Invalid FORCE_NONPRIV write %x at offset %x\n" , |
786 | vgpu->id, reg_nonpriv, offset); |
787 | } else |
788 | intel_vgpu_default_mmio_write(vgpu, offset, p_data, bytes); |
789 | |
790 | return 0; |
791 | } |
792 | |
793 | static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
794 | void *p_data, unsigned int bytes) |
795 | { |
796 | write_vreg(vgpu, offset, p_data, bytes); |
797 | |
798 | if (vgpu_vreg(vgpu, offset) & DDI_BUF_CTL_ENABLE) { |
799 | vgpu_vreg(vgpu, offset) &= ~DDI_BUF_IS_IDLE; |
800 | } else { |
801 | vgpu_vreg(vgpu, offset) |= DDI_BUF_IS_IDLE; |
802 | if (offset == i915_mmio_reg_offset(DDI_BUF_CTL(PORT_E))) |
803 | vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) |
804 | &= ~DP_TP_STATUS_AUTOTRAIN_DONE; |
805 | } |
806 | return 0; |
807 | } |
808 | |
809 | static int fdi_rx_iir_mmio_write(struct intel_vgpu *vgpu, |
810 | unsigned int offset, void *p_data, unsigned int bytes) |
811 | { |
812 | vgpu_vreg(vgpu, offset) &= ~*(u32 *)p_data; |
813 | return 0; |
814 | } |
815 | |
816 | #define FDI_LINK_TRAIN_PATTERN1 0 |
817 | #define FDI_LINK_TRAIN_PATTERN2 1 |
818 | |
819 | static int fdi_auto_training_started(struct intel_vgpu *vgpu) |
820 | { |
821 | u32 ddi_buf_ctl = vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_E)); |
822 | u32 rx_ctl = vgpu_vreg(vgpu, _FDI_RXA_CTL); |
823 | u32 tx_ctl = vgpu_vreg_t(vgpu, DP_TP_CTL(PORT_E)); |
824 | |
825 | if ((ddi_buf_ctl & DDI_BUF_CTL_ENABLE) && |
826 | (rx_ctl & FDI_RX_ENABLE) && |
827 | (rx_ctl & FDI_AUTO_TRAINING) && |
828 | (tx_ctl & DP_TP_CTL_ENABLE) && |
829 | (tx_ctl & DP_TP_CTL_FDI_AUTOTRAIN)) |
830 | return 1; |
831 | else |
832 | return 0; |
833 | } |
834 | |
835 | static int check_fdi_rx_train_status(struct intel_vgpu *vgpu, |
836 | enum pipe pipe, unsigned int train_pattern) |
837 | { |
838 | i915_reg_t fdi_rx_imr, fdi_tx_ctl, fdi_rx_ctl; |
839 | unsigned int fdi_rx_check_bits, fdi_tx_check_bits; |
840 | unsigned int fdi_rx_train_bits, fdi_tx_train_bits; |
841 | unsigned int fdi_iir_check_bits; |
842 | |
843 | fdi_rx_imr = FDI_RX_IMR(pipe); |
844 | fdi_tx_ctl = FDI_TX_CTL(pipe); |
845 | fdi_rx_ctl = FDI_RX_CTL(pipe); |
846 | |
847 | if (train_pattern == FDI_LINK_TRAIN_PATTERN1) { |
848 | fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_1_CPT; |
849 | fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_1; |
850 | fdi_iir_check_bits = FDI_RX_BIT_LOCK; |
851 | } else if (train_pattern == FDI_LINK_TRAIN_PATTERN2) { |
852 | fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_2_CPT; |
853 | fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_2; |
854 | fdi_iir_check_bits = FDI_RX_SYMBOL_LOCK; |
855 | } else { |
856 | gvt_vgpu_err("Invalid train pattern %d\n" , train_pattern); |
857 | return -EINVAL; |
858 | } |
859 | |
860 | fdi_rx_check_bits = FDI_RX_ENABLE | fdi_rx_train_bits; |
861 | fdi_tx_check_bits = FDI_TX_ENABLE | fdi_tx_train_bits; |
862 | |
863 | /* If imr bit has been masked */ |
864 | if (vgpu_vreg_t(vgpu, fdi_rx_imr) & fdi_iir_check_bits) |
865 | return 0; |
866 | |
867 | if (((vgpu_vreg_t(vgpu, fdi_tx_ctl) & fdi_tx_check_bits) |
868 | == fdi_tx_check_bits) |
869 | && ((vgpu_vreg_t(vgpu, fdi_rx_ctl) & fdi_rx_check_bits) |
870 | == fdi_rx_check_bits)) |
871 | return 1; |
872 | else |
873 | return 0; |
874 | } |
875 | |
876 | #define INVALID_INDEX (~0U) |
877 | |
878 | static unsigned int calc_index(unsigned int offset, unsigned int start, |
879 | unsigned int next, unsigned int end, i915_reg_t i915_end) |
880 | { |
881 | unsigned int range = next - start; |
882 | |
883 | if (!end) |
884 | end = i915_mmio_reg_offset(i915_end); |
885 | if (offset < start || offset > end) |
886 | return INVALID_INDEX; |
887 | offset -= start; |
888 | return offset / range; |
889 | } |
890 | |
891 | #define FDI_RX_CTL_TO_PIPE(offset) \ |
892 | calc_index(offset, _FDI_RXA_CTL, _FDI_RXB_CTL, 0, FDI_RX_CTL(PIPE_C)) |
893 | |
894 | #define FDI_TX_CTL_TO_PIPE(offset) \ |
895 | calc_index(offset, _FDI_TXA_CTL, _FDI_TXB_CTL, 0, FDI_TX_CTL(PIPE_C)) |
896 | |
897 | #define FDI_RX_IMR_TO_PIPE(offset) \ |
898 | calc_index(offset, _FDI_RXA_IMR, _FDI_RXB_IMR, 0, FDI_RX_IMR(PIPE_C)) |
899 | |
900 | static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu, |
901 | unsigned int offset, void *p_data, unsigned int bytes) |
902 | { |
903 | i915_reg_t fdi_rx_iir; |
904 | unsigned int index; |
905 | int ret; |
906 | |
907 | if (FDI_RX_CTL_TO_PIPE(offset) != INVALID_INDEX) |
908 | index = FDI_RX_CTL_TO_PIPE(offset); |
909 | else if (FDI_TX_CTL_TO_PIPE(offset) != INVALID_INDEX) |
910 | index = FDI_TX_CTL_TO_PIPE(offset); |
911 | else if (FDI_RX_IMR_TO_PIPE(offset) != INVALID_INDEX) |
912 | index = FDI_RX_IMR_TO_PIPE(offset); |
913 | else { |
914 | gvt_vgpu_err("Unsupported registers %x\n" , offset); |
915 | return -EINVAL; |
916 | } |
917 | |
918 | write_vreg(vgpu, offset, p_data, bytes); |
919 | |
920 | fdi_rx_iir = FDI_RX_IIR(index); |
921 | |
922 | ret = check_fdi_rx_train_status(vgpu, pipe: index, FDI_LINK_TRAIN_PATTERN1); |
923 | if (ret < 0) |
924 | return ret; |
925 | if (ret) |
926 | vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK; |
927 | |
928 | ret = check_fdi_rx_train_status(vgpu, pipe: index, FDI_LINK_TRAIN_PATTERN2); |
929 | if (ret < 0) |
930 | return ret; |
931 | if (ret) |
932 | vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK; |
933 | |
934 | if (offset == _FDI_RXA_CTL) |
935 | if (fdi_auto_training_started(vgpu)) |
936 | vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) |= |
937 | DP_TP_STATUS_AUTOTRAIN_DONE; |
938 | return 0; |
939 | } |
940 | |
941 | #define DP_TP_CTL_TO_PORT(offset) \ |
942 | calc_index(offset, _DP_TP_CTL_A, _DP_TP_CTL_B, 0, DP_TP_CTL(PORT_E)) |
943 | |
944 | static int dp_tp_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
945 | void *p_data, unsigned int bytes) |
946 | { |
947 | i915_reg_t status_reg; |
948 | unsigned int index; |
949 | u32 data; |
950 | |
951 | write_vreg(vgpu, offset, p_data, bytes); |
952 | |
953 | index = DP_TP_CTL_TO_PORT(offset); |
954 | data = (vgpu_vreg(vgpu, offset) & GENMASK(10, 8)) >> 8; |
955 | if (data == 0x2) { |
956 | status_reg = DP_TP_STATUS(index); |
957 | vgpu_vreg_t(vgpu, status_reg) |= (1 << 25); |
958 | } |
959 | return 0; |
960 | } |
961 | |
962 | static int dp_tp_status_mmio_write(struct intel_vgpu *vgpu, |
963 | unsigned int offset, void *p_data, unsigned int bytes) |
964 | { |
965 | u32 reg_val; |
966 | u32 sticky_mask; |
967 | |
968 | reg_val = *((u32 *)p_data); |
969 | sticky_mask = GENMASK(27, 26) | (1 << 24); |
970 | |
971 | vgpu_vreg(vgpu, offset) = (reg_val & ~sticky_mask) | |
972 | (vgpu_vreg(vgpu, offset) & sticky_mask); |
973 | vgpu_vreg(vgpu, offset) &= ~(reg_val & sticky_mask); |
974 | return 0; |
975 | } |
976 | |
977 | static int pch_adpa_mmio_write(struct intel_vgpu *vgpu, |
978 | unsigned int offset, void *p_data, unsigned int bytes) |
979 | { |
980 | u32 data; |
981 | |
982 | write_vreg(vgpu, offset, p_data, bytes); |
983 | data = vgpu_vreg(vgpu, offset); |
984 | |
985 | if (data & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) |
986 | vgpu_vreg(vgpu, offset) &= ~ADPA_CRT_HOTPLUG_FORCE_TRIGGER; |
987 | return 0; |
988 | } |
989 | |
990 | static int south_chicken2_mmio_write(struct intel_vgpu *vgpu, |
991 | unsigned int offset, void *p_data, unsigned int bytes) |
992 | { |
993 | u32 data; |
994 | |
995 | write_vreg(vgpu, offset, p_data, bytes); |
996 | data = vgpu_vreg(vgpu, offset); |
997 | |
998 | if (data & FDI_MPHY_IOSFSB_RESET_CTL) |
999 | vgpu_vreg(vgpu, offset) |= FDI_MPHY_IOSFSB_RESET_STATUS; |
1000 | else |
1001 | vgpu_vreg(vgpu, offset) &= ~FDI_MPHY_IOSFSB_RESET_STATUS; |
1002 | return 0; |
1003 | } |
1004 | |
1005 | #define DSPSURF_TO_PIPE(offset) \ |
1006 | calc_index(offset, _DSPASURF, _DSPBSURF, 0, DSPSURF(PIPE_C)) |
1007 | |
1008 | static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
1009 | void *p_data, unsigned int bytes) |
1010 | { |
1011 | struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; |
1012 | u32 pipe = DSPSURF_TO_PIPE(offset); |
1013 | int event = SKL_FLIP_EVENT(pipe, PLANE_PRIMARY); |
1014 | |
1015 | write_vreg(vgpu, offset, p_data, bytes); |
1016 | vgpu_vreg_t(vgpu, DSPSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); |
1017 | |
1018 | vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++; |
1019 | |
1020 | if (vgpu_vreg_t(vgpu, DSPCNTR(pipe)) & PLANE_CTL_ASYNC_FLIP) |
1021 | intel_vgpu_trigger_virtual_event(vgpu, event); |
1022 | else |
1023 | set_bit(nr: event, addr: vgpu->irq.flip_done_event[pipe]); |
1024 | |
1025 | return 0; |
1026 | } |
1027 | |
1028 | #define SPRSURF_TO_PIPE(offset) \ |
1029 | calc_index(offset, _SPRA_SURF, _SPRB_SURF, 0, SPRSURF(PIPE_C)) |
1030 | |
1031 | static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
1032 | void *p_data, unsigned int bytes) |
1033 | { |
1034 | u32 pipe = SPRSURF_TO_PIPE(offset); |
1035 | int event = SKL_FLIP_EVENT(pipe, PLANE_SPRITE0); |
1036 | |
1037 | write_vreg(vgpu, offset, p_data, bytes); |
1038 | vgpu_vreg_t(vgpu, SPRSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); |
1039 | |
1040 | if (vgpu_vreg_t(vgpu, SPRCTL(pipe)) & PLANE_CTL_ASYNC_FLIP) |
1041 | intel_vgpu_trigger_virtual_event(vgpu, event); |
1042 | else |
1043 | set_bit(nr: event, addr: vgpu->irq.flip_done_event[pipe]); |
1044 | |
1045 | return 0; |
1046 | } |
1047 | |
1048 | static int reg50080_mmio_write(struct intel_vgpu *vgpu, |
1049 | unsigned int offset, void *p_data, |
1050 | unsigned int bytes) |
1051 | { |
1052 | struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; |
1053 | enum pipe pipe = REG_50080_TO_PIPE(offset); |
1054 | enum plane_id plane = REG_50080_TO_PLANE(offset); |
1055 | int event = SKL_FLIP_EVENT(pipe, plane); |
1056 | |
1057 | write_vreg(vgpu, offset, p_data, bytes); |
1058 | if (plane == PLANE_PRIMARY) { |
1059 | vgpu_vreg_t(vgpu, DSPSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); |
1060 | vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++; |
1061 | } else { |
1062 | vgpu_vreg_t(vgpu, SPRSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset); |
1063 | } |
1064 | |
1065 | if ((vgpu_vreg(vgpu, offset) & REG50080_FLIP_TYPE_MASK) == REG50080_FLIP_TYPE_ASYNC) |
1066 | intel_vgpu_trigger_virtual_event(vgpu, event); |
1067 | else |
1068 | set_bit(nr: event, addr: vgpu->irq.flip_done_event[pipe]); |
1069 | |
1070 | return 0; |
1071 | } |
1072 | |
1073 | static int trigger_aux_channel_interrupt(struct intel_vgpu *vgpu, |
1074 | unsigned int reg) |
1075 | { |
1076 | struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; |
1077 | enum intel_gvt_event_type event; |
1078 | |
1079 | if (reg == i915_mmio_reg_offset(DP_AUX_CH_CTL(AUX_CH_A))) |
1080 | event = AUX_CHANNEL_A; |
1081 | else if (reg == _PCH_DPB_AUX_CH_CTL || |
1082 | reg == i915_mmio_reg_offset(DP_AUX_CH_CTL(AUX_CH_B))) |
1083 | event = AUX_CHANNEL_B; |
1084 | else if (reg == _PCH_DPC_AUX_CH_CTL || |
1085 | reg == i915_mmio_reg_offset(DP_AUX_CH_CTL(AUX_CH_C))) |
1086 | event = AUX_CHANNEL_C; |
1087 | else if (reg == _PCH_DPD_AUX_CH_CTL || |
1088 | reg == i915_mmio_reg_offset(DP_AUX_CH_CTL(AUX_CH_D))) |
1089 | event = AUX_CHANNEL_D; |
1090 | else { |
1091 | drm_WARN_ON(&dev_priv->drm, true); |
1092 | return -EINVAL; |
1093 | } |
1094 | |
1095 | intel_vgpu_trigger_virtual_event(vgpu, event); |
1096 | return 0; |
1097 | } |
1098 | |
1099 | static int dp_aux_ch_ctl_trans_done(struct intel_vgpu *vgpu, u32 value, |
1100 | unsigned int reg, int len, bool data_valid) |
1101 | { |
1102 | /* mark transaction done */ |
1103 | value |= DP_AUX_CH_CTL_DONE; |
1104 | value &= ~DP_AUX_CH_CTL_SEND_BUSY; |
1105 | value &= ~DP_AUX_CH_CTL_RECEIVE_ERROR; |
1106 | |
1107 | if (data_valid) |
1108 | value &= ~DP_AUX_CH_CTL_TIME_OUT_ERROR; |
1109 | else |
1110 | value |= DP_AUX_CH_CTL_TIME_OUT_ERROR; |
1111 | |
1112 | /* message size */ |
1113 | value &= ~(0xf << 20); |
1114 | value |= (len << 20); |
1115 | vgpu_vreg(vgpu, reg) = value; |
1116 | |
1117 | if (value & DP_AUX_CH_CTL_INTERRUPT) |
1118 | return trigger_aux_channel_interrupt(vgpu, reg); |
1119 | return 0; |
1120 | } |
1121 | |
1122 | static void dp_aux_ch_ctl_link_training(struct intel_vgpu_dpcd_data *dpcd, |
1123 | u8 t) |
1124 | { |
1125 | if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == DPCD_TRAINING_PATTERN_1) { |
1126 | /* training pattern 1 for CR */ |
1127 | /* set LANE0_CR_DONE, LANE1_CR_DONE */ |
1128 | dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_CR_DONE; |
1129 | /* set LANE2_CR_DONE, LANE3_CR_DONE */ |
1130 | dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_CR_DONE; |
1131 | } else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == |
1132 | DPCD_TRAINING_PATTERN_2) { |
1133 | /* training pattern 2 for EQ */ |
1134 | /* Set CHANNEL_EQ_DONE and SYMBOL_LOCKED for Lane0_1 */ |
1135 | dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_EQ_DONE; |
1136 | dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_SYMBOL_LOCKED; |
1137 | /* Set CHANNEL_EQ_DONE and SYMBOL_LOCKED for Lane2_3 */ |
1138 | dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_EQ_DONE; |
1139 | dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_SYMBOL_LOCKED; |
1140 | /* set INTERLANE_ALIGN_DONE */ |
1141 | dpcd->data[DPCD_LANE_ALIGN_STATUS_UPDATED] |= |
1142 | DPCD_INTERLANE_ALIGN_DONE; |
1143 | } else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == |
1144 | DPCD_LINK_TRAINING_DISABLED) { |
1145 | /* finish link training */ |
1146 | /* set sink status as synchronized */ |
1147 | dpcd->data[DPCD_SINK_STATUS] = DPCD_SINK_IN_SYNC; |
1148 | } |
1149 | } |
1150 | |
1151 | #define _REG_HSW_DP_AUX_CH_CTL(dp) \ |
1152 | ((dp) ? (_PCH_DPB_AUX_CH_CTL + ((dp)-1)*0x100) : 0x64010) |
1153 | |
1154 | #define _REG_SKL_DP_AUX_CH_CTL(dp) (0x64010 + (dp) * 0x100) |
1155 | |
1156 | #define OFFSET_TO_DP_AUX_PORT(offset) (((offset) & 0xF00) >> 8) |
1157 | |
1158 | #define dpy_is_valid_port(port) \ |
1159 | (((port) >= PORT_A) && ((port) < I915_MAX_PORTS)) |
1160 | |
1161 | static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu, |
1162 | unsigned int offset, void *p_data, unsigned int bytes) |
1163 | { |
1164 | struct intel_vgpu_display *display = &vgpu->display; |
1165 | int msg, addr, ctrl, op, len; |
1166 | int port_index = OFFSET_TO_DP_AUX_PORT(offset); |
1167 | struct intel_vgpu_dpcd_data *dpcd = NULL; |
1168 | struct intel_vgpu_port *port = NULL; |
1169 | u32 data; |
1170 | |
1171 | if (!dpy_is_valid_port(port_index)) { |
1172 | gvt_vgpu_err("Unsupported DP port access!\n" ); |
1173 | return 0; |
1174 | } |
1175 | |
1176 | write_vreg(vgpu, offset, p_data, bytes); |
1177 | data = vgpu_vreg(vgpu, offset); |
1178 | |
1179 | if ((GRAPHICS_VER(vgpu->gvt->gt->i915) >= 9) |
1180 | && offset != _REG_SKL_DP_AUX_CH_CTL(port_index)) { |
1181 | /* SKL DPB/C/D aux ctl register changed */ |
1182 | return 0; |
1183 | } else if (IS_BROADWELL(vgpu->gvt->gt->i915) && |
1184 | offset != _REG_HSW_DP_AUX_CH_CTL(port_index)) { |
1185 | /* write to the data registers */ |
1186 | return 0; |
1187 | } |
1188 | |
1189 | if (!(data & DP_AUX_CH_CTL_SEND_BUSY)) { |
1190 | /* just want to clear the sticky bits */ |
1191 | vgpu_vreg(vgpu, offset) = 0; |
1192 | return 0; |
1193 | } |
1194 | |
1195 | port = &display->ports[port_index]; |
1196 | dpcd = port->dpcd; |
1197 | |
1198 | /* read out message from DATA1 register */ |
1199 | msg = vgpu_vreg(vgpu, offset + 4); |
1200 | addr = (msg >> 8) & 0xffff; |
1201 | ctrl = (msg >> 24) & 0xff; |
1202 | len = msg & 0xff; |
1203 | op = ctrl >> 4; |
1204 | |
1205 | if (op == GVT_AUX_NATIVE_WRITE) { |
1206 | int t; |
1207 | u8 buf[16]; |
1208 | |
1209 | if ((addr + len + 1) >= DPCD_SIZE) { |
1210 | /* |
1211 | * Write request exceeds what we supported, |
1212 | * DCPD spec: When a Source Device is writing a DPCD |
1213 | * address not supported by the Sink Device, the Sink |
1214 | * Device shall reply with AUX NACK and “M” equal to |
1215 | * zero. |
1216 | */ |
1217 | |
1218 | /* NAK the write */ |
1219 | vgpu_vreg(vgpu, offset + 4) = AUX_NATIVE_REPLY_NAK; |
1220 | dp_aux_ch_ctl_trans_done(vgpu, value: data, reg: offset, len: 2, data_valid: true); |
1221 | return 0; |
1222 | } |
1223 | |
1224 | /* |
1225 | * Write request format: Headr (command + address + size) occupies |
1226 | * 4 bytes, followed by (len + 1) bytes of data. See details at |
1227 | * intel_dp_aux_transfer(). |
1228 | */ |
1229 | if ((len + 1 + 4) > AUX_BURST_SIZE) { |
1230 | gvt_vgpu_err("dp_aux_header: len %d is too large\n" , len); |
1231 | return -EINVAL; |
1232 | } |
1233 | |
1234 | /* unpack data from vreg to buf */ |
1235 | for (t = 0; t < 4; t++) { |
1236 | u32 r = vgpu_vreg(vgpu, offset + 8 + t * 4); |
1237 | |
1238 | buf[t * 4] = (r >> 24) & 0xff; |
1239 | buf[t * 4 + 1] = (r >> 16) & 0xff; |
1240 | buf[t * 4 + 2] = (r >> 8) & 0xff; |
1241 | buf[t * 4 + 3] = r & 0xff; |
1242 | } |
1243 | |
1244 | /* write to virtual DPCD */ |
1245 | if (dpcd && dpcd->data_valid) { |
1246 | for (t = 0; t <= len; t++) { |
1247 | int p = addr + t; |
1248 | |
1249 | dpcd->data[p] = buf[t]; |
1250 | /* check for link training */ |
1251 | if (p == DPCD_TRAINING_PATTERN_SET) |
1252 | dp_aux_ch_ctl_link_training(dpcd, |
1253 | t: buf[t]); |
1254 | } |
1255 | } |
1256 | |
1257 | /* ACK the write */ |
1258 | vgpu_vreg(vgpu, offset + 4) = 0; |
1259 | dp_aux_ch_ctl_trans_done(vgpu, value: data, reg: offset, len: 1, |
1260 | data_valid: dpcd && dpcd->data_valid); |
1261 | return 0; |
1262 | } |
1263 | |
1264 | if (op == GVT_AUX_NATIVE_READ) { |
1265 | int idx, i, ret = 0; |
1266 | |
1267 | if ((addr + len + 1) >= DPCD_SIZE) { |
1268 | /* |
1269 | * read request exceeds what we supported |
1270 | * DPCD spec: A Sink Device receiving a Native AUX CH |
1271 | * read request for an unsupported DPCD address must |
1272 | * reply with an AUX ACK and read data set equal to |
1273 | * zero instead of replying with AUX NACK. |
1274 | */ |
1275 | |
1276 | /* ACK the READ*/ |
1277 | vgpu_vreg(vgpu, offset + 4) = 0; |
1278 | vgpu_vreg(vgpu, offset + 8) = 0; |
1279 | vgpu_vreg(vgpu, offset + 12) = 0; |
1280 | vgpu_vreg(vgpu, offset + 16) = 0; |
1281 | vgpu_vreg(vgpu, offset + 20) = 0; |
1282 | |
1283 | dp_aux_ch_ctl_trans_done(vgpu, value: data, reg: offset, len: len + 2, |
1284 | data_valid: true); |
1285 | return 0; |
1286 | } |
1287 | |
1288 | for (idx = 1; idx <= 5; idx++) { |
1289 | /* clear the data registers */ |
1290 | vgpu_vreg(vgpu, offset + 4 * idx) = 0; |
1291 | } |
1292 | |
1293 | /* |
1294 | * Read reply format: ACK (1 byte) plus (len + 1) bytes of data. |
1295 | */ |
1296 | if ((len + 2) > AUX_BURST_SIZE) { |
1297 | gvt_vgpu_err("dp_aux_header: len %d is too large\n" , len); |
1298 | return -EINVAL; |
1299 | } |
1300 | |
1301 | /* read from virtual DPCD to vreg */ |
1302 | /* first 4 bytes: [ACK][addr][addr+1][addr+2] */ |
1303 | if (dpcd && dpcd->data_valid) { |
1304 | for (i = 1; i <= (len + 1); i++) { |
1305 | int t; |
1306 | |
1307 | t = dpcd->data[addr + i - 1]; |
1308 | t <<= (24 - 8 * (i % 4)); |
1309 | ret |= t; |
1310 | |
1311 | if ((i % 4 == 3) || (i == (len + 1))) { |
1312 | vgpu_vreg(vgpu, offset + |
1313 | (i / 4 + 1) * 4) = ret; |
1314 | ret = 0; |
1315 | } |
1316 | } |
1317 | } |
1318 | dp_aux_ch_ctl_trans_done(vgpu, value: data, reg: offset, len: len + 2, |
1319 | data_valid: dpcd && dpcd->data_valid); |
1320 | return 0; |
1321 | } |
1322 | |
1323 | /* i2c transaction starts */ |
1324 | intel_gvt_i2c_handle_aux_ch_write(vgpu, port_idx: port_index, offset, p_data); |
1325 | |
1326 | if (data & DP_AUX_CH_CTL_INTERRUPT) |
1327 | trigger_aux_channel_interrupt(vgpu, reg: offset); |
1328 | return 0; |
1329 | } |
1330 | |
1331 | static int mbctl_write(struct intel_vgpu *vgpu, unsigned int offset, |
1332 | void *p_data, unsigned int bytes) |
1333 | { |
1334 | *(u32 *)p_data &= (~GEN6_MBCTL_ENABLE_BOOT_FETCH); |
1335 | write_vreg(vgpu, offset, p_data, bytes); |
1336 | return 0; |
1337 | } |
1338 | |
1339 | static int vga_control_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
1340 | void *p_data, unsigned int bytes) |
1341 | { |
1342 | bool vga_disable; |
1343 | |
1344 | write_vreg(vgpu, offset, p_data, bytes); |
1345 | vga_disable = vgpu_vreg(vgpu, offset) & VGA_DISP_DISABLE; |
1346 | |
1347 | gvt_dbg_core("vgpu%d: %s VGA mode\n" , vgpu->id, |
1348 | vga_disable ? "Disable" : "Enable" ); |
1349 | return 0; |
1350 | } |
1351 | |
1352 | static u32 read_virtual_sbi_register(struct intel_vgpu *vgpu, |
1353 | unsigned int sbi_offset) |
1354 | { |
1355 | struct intel_vgpu_display *display = &vgpu->display; |
1356 | int num = display->sbi.number; |
1357 | int i; |
1358 | |
1359 | for (i = 0; i < num; ++i) |
1360 | if (display->sbi.registers[i].offset == sbi_offset) |
1361 | break; |
1362 | |
1363 | if (i == num) |
1364 | return 0; |
1365 | |
1366 | return display->sbi.registers[i].value; |
1367 | } |
1368 | |
1369 | static void write_virtual_sbi_register(struct intel_vgpu *vgpu, |
1370 | unsigned int offset, u32 value) |
1371 | { |
1372 | struct intel_vgpu_display *display = &vgpu->display; |
1373 | int num = display->sbi.number; |
1374 | int i; |
1375 | |
1376 | for (i = 0; i < num; ++i) { |
1377 | if (display->sbi.registers[i].offset == offset) |
1378 | break; |
1379 | } |
1380 | |
1381 | if (i == num) { |
1382 | if (num == SBI_REG_MAX) { |
1383 | gvt_vgpu_err("SBI caching meets maximum limits\n" ); |
1384 | return; |
1385 | } |
1386 | display->sbi.number++; |
1387 | } |
1388 | |
1389 | display->sbi.registers[i].offset = offset; |
1390 | display->sbi.registers[i].value = value; |
1391 | } |
1392 | |
1393 | static int sbi_data_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, |
1394 | void *p_data, unsigned int bytes) |
1395 | { |
1396 | if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> |
1397 | SBI_OPCODE_SHIFT) == SBI_CMD_CRRD) { |
1398 | unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) & |
1399 | SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT; |
1400 | vgpu_vreg(vgpu, offset) = read_virtual_sbi_register(vgpu, |
1401 | sbi_offset); |
1402 | } |
1403 | read_vreg(vgpu, offset, p_data, bytes); |
1404 | return 0; |
1405 | } |
1406 | |
1407 | static int sbi_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
1408 | void *p_data, unsigned int bytes) |
1409 | { |
1410 | u32 data; |
1411 | |
1412 | write_vreg(vgpu, offset, p_data, bytes); |
1413 | data = vgpu_vreg(vgpu, offset); |
1414 | |
1415 | data &= ~(SBI_STAT_MASK << SBI_STAT_SHIFT); |
1416 | data |= SBI_READY; |
1417 | |
1418 | data &= ~(SBI_RESPONSE_MASK << SBI_RESPONSE_SHIFT); |
1419 | data |= SBI_RESPONSE_SUCCESS; |
1420 | |
1421 | vgpu_vreg(vgpu, offset) = data; |
1422 | |
1423 | if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >> |
1424 | SBI_OPCODE_SHIFT) == SBI_CMD_CRWR) { |
1425 | unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) & |
1426 | SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT; |
1427 | |
1428 | write_virtual_sbi_register(vgpu, offset: sbi_offset, |
1429 | vgpu_vreg_t(vgpu, SBI_DATA)); |
1430 | } |
1431 | return 0; |
1432 | } |
1433 | |
1434 | #define _vgtif_reg(x) \ |
1435 | (VGT_PVINFO_PAGE + offsetof(struct vgt_if, x)) |
1436 | |
1437 | static int pvinfo_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, |
1438 | void *p_data, unsigned int bytes) |
1439 | { |
1440 | bool invalid_read = false; |
1441 | |
1442 | read_vreg(vgpu, offset, p_data, bytes); |
1443 | |
1444 | switch (offset) { |
1445 | case _vgtif_reg(magic) ... _vgtif_reg(vgt_id): |
1446 | if (offset + bytes > _vgtif_reg(vgt_id) + 4) |
1447 | invalid_read = true; |
1448 | break; |
1449 | case _vgtif_reg(avail_rs.mappable_gmadr.base) ... |
1450 | _vgtif_reg(avail_rs.fence_num): |
1451 | if (offset + bytes > |
1452 | _vgtif_reg(avail_rs.fence_num) + 4) |
1453 | invalid_read = true; |
1454 | break; |
1455 | case 0x78010: /* vgt_caps */ |
1456 | case 0x7881c: |
1457 | break; |
1458 | default: |
1459 | invalid_read = true; |
1460 | break; |
1461 | } |
1462 | if (invalid_read) |
1463 | gvt_vgpu_err("invalid pvinfo read: [%x:%x] = %x\n" , |
1464 | offset, bytes, *(u32 *)p_data); |
1465 | vgpu->pv_notified = true; |
1466 | return 0; |
1467 | } |
1468 | |
1469 | static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification) |
1470 | { |
1471 | enum intel_gvt_gtt_type root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY; |
1472 | struct intel_vgpu_mm *mm; |
1473 | u64 *pdps; |
1474 | |
1475 | pdps = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0])); |
1476 | |
1477 | switch (notification) { |
1478 | case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE: |
1479 | root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY; |
1480 | fallthrough; |
1481 | case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE: |
1482 | mm = intel_vgpu_get_ppgtt_mm(vgpu, root_entry_type, pdps); |
1483 | return PTR_ERR_OR_ZERO(ptr: mm); |
1484 | case VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY: |
1485 | case VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY: |
1486 | return intel_vgpu_put_ppgtt_mm(vgpu, pdps); |
1487 | case VGT_G2V_EXECLIST_CONTEXT_CREATE: |
1488 | case VGT_G2V_EXECLIST_CONTEXT_DESTROY: |
1489 | case 1: /* Remove this in guest driver. */ |
1490 | break; |
1491 | default: |
1492 | gvt_vgpu_err("Invalid PV notification %d\n" , notification); |
1493 | } |
1494 | return 0; |
1495 | } |
1496 | |
1497 | static int send_display_ready_uevent(struct intel_vgpu *vgpu, int ready) |
1498 | { |
1499 | struct kobject *kobj = &vgpu->gvt->gt->i915->drm.primary->kdev->kobj; |
1500 | char *env[3] = {NULL, NULL, NULL}; |
1501 | char vmid_str[20]; |
1502 | char display_ready_str[20]; |
1503 | |
1504 | snprintf(buf: display_ready_str, size: 20, fmt: "GVT_DISPLAY_READY=%d" , ready); |
1505 | env[0] = display_ready_str; |
1506 | |
1507 | snprintf(buf: vmid_str, size: 20, fmt: "VMID=%d" , vgpu->id); |
1508 | env[1] = vmid_str; |
1509 | |
1510 | return kobject_uevent_env(kobj, action: KOBJ_ADD, envp: env); |
1511 | } |
1512 | |
1513 | static int pvinfo_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
1514 | void *p_data, unsigned int bytes) |
1515 | { |
1516 | u32 data = *(u32 *)p_data; |
1517 | bool invalid_write = false; |
1518 | |
1519 | switch (offset) { |
1520 | case _vgtif_reg(display_ready): |
1521 | send_display_ready_uevent(vgpu, ready: data ? 1 : 0); |
1522 | break; |
1523 | case _vgtif_reg(g2v_notify): |
1524 | handle_g2v_notification(vgpu, notification: data); |
1525 | break; |
1526 | /* add xhot and yhot to handled list to avoid error log */ |
1527 | case _vgtif_reg(cursor_x_hot): |
1528 | case _vgtif_reg(cursor_y_hot): |
1529 | case _vgtif_reg(pdp[0].lo): |
1530 | case _vgtif_reg(pdp[0].hi): |
1531 | case _vgtif_reg(pdp[1].lo): |
1532 | case _vgtif_reg(pdp[1].hi): |
1533 | case _vgtif_reg(pdp[2].lo): |
1534 | case _vgtif_reg(pdp[2].hi): |
1535 | case _vgtif_reg(pdp[3].lo): |
1536 | case _vgtif_reg(pdp[3].hi): |
1537 | case _vgtif_reg(execlist_context_descriptor_lo): |
1538 | case _vgtif_reg(execlist_context_descriptor_hi): |
1539 | break; |
1540 | case _vgtif_reg(rsv5[0])..._vgtif_reg(rsv5[3]): |
1541 | invalid_write = true; |
1542 | enter_failsafe_mode(vgpu, reason: GVT_FAILSAFE_INSUFFICIENT_RESOURCE); |
1543 | break; |
1544 | default: |
1545 | invalid_write = true; |
1546 | gvt_vgpu_err("invalid pvinfo write offset %x bytes %x data %x\n" , |
1547 | offset, bytes, data); |
1548 | break; |
1549 | } |
1550 | |
1551 | if (!invalid_write) |
1552 | write_vreg(vgpu, offset, p_data, bytes); |
1553 | |
1554 | return 0; |
1555 | } |
1556 | |
1557 | static int pf_write(struct intel_vgpu *vgpu, |
1558 | unsigned int offset, void *p_data, unsigned int bytes) |
1559 | { |
1560 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
1561 | u32 val = *(u32 *)p_data; |
1562 | |
1563 | if ((offset == _PS_1A_CTRL || offset == _PS_2A_CTRL || |
1564 | offset == _PS_1B_CTRL || offset == _PS_2B_CTRL || |
1565 | offset == _PS_1C_CTRL) && (val & PS_BINDING_MASK) != PS_BINDING_PIPE) { |
1566 | drm_WARN_ONCE(&i915->drm, true, |
1567 | "VM(%d): guest is trying to scaling a plane\n" , |
1568 | vgpu->id); |
1569 | return 0; |
1570 | } |
1571 | |
1572 | return intel_vgpu_default_mmio_write(vgpu, offset, p_data, bytes); |
1573 | } |
1574 | |
1575 | static int power_well_ctl_mmio_write(struct intel_vgpu *vgpu, |
1576 | unsigned int offset, void *p_data, unsigned int bytes) |
1577 | { |
1578 | write_vreg(vgpu, offset, p_data, bytes); |
1579 | |
1580 | if (vgpu_vreg(vgpu, offset) & |
1581 | HSW_PWR_WELL_CTL_REQ(HSW_PW_CTL_IDX_GLOBAL)) |
1582 | vgpu_vreg(vgpu, offset) |= |
1583 | HSW_PWR_WELL_CTL_STATE(HSW_PW_CTL_IDX_GLOBAL); |
1584 | else |
1585 | vgpu_vreg(vgpu, offset) &= |
1586 | ~HSW_PWR_WELL_CTL_STATE(HSW_PW_CTL_IDX_GLOBAL); |
1587 | return 0; |
1588 | } |
1589 | |
1590 | static int gen9_dbuf_ctl_mmio_write(struct intel_vgpu *vgpu, |
1591 | unsigned int offset, void *p_data, unsigned int bytes) |
1592 | { |
1593 | write_vreg(vgpu, offset, p_data, bytes); |
1594 | |
1595 | if (vgpu_vreg(vgpu, offset) & DBUF_POWER_REQUEST) |
1596 | vgpu_vreg(vgpu, offset) |= DBUF_POWER_STATE; |
1597 | else |
1598 | vgpu_vreg(vgpu, offset) &= ~DBUF_POWER_STATE; |
1599 | |
1600 | return 0; |
1601 | } |
1602 | |
1603 | static int fpga_dbg_mmio_write(struct intel_vgpu *vgpu, |
1604 | unsigned int offset, void *p_data, unsigned int bytes) |
1605 | { |
1606 | write_vreg(vgpu, offset, p_data, bytes); |
1607 | |
1608 | if (vgpu_vreg(vgpu, offset) & FPGA_DBG_RM_NOCLAIM) |
1609 | vgpu_vreg(vgpu, offset) &= ~FPGA_DBG_RM_NOCLAIM; |
1610 | return 0; |
1611 | } |
1612 | |
1613 | static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset, |
1614 | void *p_data, unsigned int bytes) |
1615 | { |
1616 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
1617 | u32 mode; |
1618 | |
1619 | write_vreg(vgpu, offset, p_data, bytes); |
1620 | mode = vgpu_vreg(vgpu, offset); |
1621 | |
1622 | if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) { |
1623 | drm_WARN_ONCE(&i915->drm, 1, |
1624 | "VM(%d): iGVT-g doesn't support GuC\n" , |
1625 | vgpu->id); |
1626 | return 0; |
1627 | } |
1628 | |
1629 | return 0; |
1630 | } |
1631 | |
1632 | static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset, |
1633 | void *p_data, unsigned int bytes) |
1634 | { |
1635 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
1636 | u32 trtte = *(u32 *)p_data; |
1637 | |
1638 | if ((trtte & 1) && (trtte & (1 << 1)) == 0) { |
1639 | drm_WARN(&i915->drm, 1, |
1640 | "VM(%d): Use physical address for TRTT!\n" , |
1641 | vgpu->id); |
1642 | return -EINVAL; |
1643 | } |
1644 | write_vreg(vgpu, offset, p_data, bytes); |
1645 | |
1646 | return 0; |
1647 | } |
1648 | |
1649 | static int gen9_trtt_chicken_write(struct intel_vgpu *vgpu, unsigned int offset, |
1650 | void *p_data, unsigned int bytes) |
1651 | { |
1652 | write_vreg(vgpu, offset, p_data, bytes); |
1653 | return 0; |
1654 | } |
1655 | |
1656 | static int dpll_status_read(struct intel_vgpu *vgpu, unsigned int offset, |
1657 | void *p_data, unsigned int bytes) |
1658 | { |
1659 | u32 v = 0; |
1660 | |
1661 | if (vgpu_vreg(vgpu, 0x46010) & (1 << 31)) |
1662 | v |= (1 << 0); |
1663 | |
1664 | if (vgpu_vreg(vgpu, 0x46014) & (1 << 31)) |
1665 | v |= (1 << 8); |
1666 | |
1667 | if (vgpu_vreg(vgpu, 0x46040) & (1 << 31)) |
1668 | v |= (1 << 16); |
1669 | |
1670 | if (vgpu_vreg(vgpu, 0x46060) & (1 << 31)) |
1671 | v |= (1 << 24); |
1672 | |
1673 | vgpu_vreg(vgpu, offset) = v; |
1674 | |
1675 | return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes); |
1676 | } |
1677 | |
1678 | static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset, |
1679 | void *p_data, unsigned int bytes) |
1680 | { |
1681 | u32 value = *(u32 *)p_data; |
1682 | u32 cmd = value & 0xff; |
1683 | u32 *data0 = &vgpu_vreg_t(vgpu, GEN6_PCODE_DATA); |
1684 | |
1685 | switch (cmd) { |
1686 | case GEN9_PCODE_READ_MEM_LATENCY: |
1687 | if (IS_SKYLAKE(vgpu->gvt->gt->i915) || |
1688 | IS_KABYLAKE(vgpu->gvt->gt->i915) || |
1689 | IS_COFFEELAKE(vgpu->gvt->gt->i915) || |
1690 | IS_COMETLAKE(vgpu->gvt->gt->i915)) { |
1691 | /** |
1692 | * "Read memory latency" command on gen9. |
1693 | * Below memory latency values are read |
1694 | * from skylake platform. |
1695 | */ |
1696 | if (!*data0) |
1697 | *data0 = 0x1e1a1100; |
1698 | else |
1699 | *data0 = 0x61514b3d; |
1700 | } else if (IS_BROXTON(vgpu->gvt->gt->i915)) { |
1701 | /** |
1702 | * "Read memory latency" command on gen9. |
1703 | * Below memory latency values are read |
1704 | * from Broxton MRB. |
1705 | */ |
1706 | if (!*data0) |
1707 | *data0 = 0x16080707; |
1708 | else |
1709 | *data0 = 0x16161616; |
1710 | } |
1711 | break; |
1712 | case SKL_PCODE_CDCLK_CONTROL: |
1713 | if (IS_SKYLAKE(vgpu->gvt->gt->i915) || |
1714 | IS_KABYLAKE(vgpu->gvt->gt->i915) || |
1715 | IS_COFFEELAKE(vgpu->gvt->gt->i915) || |
1716 | IS_COMETLAKE(vgpu->gvt->gt->i915)) |
1717 | *data0 = SKL_CDCLK_READY_FOR_CHANGE; |
1718 | break; |
1719 | case GEN6_PCODE_READ_RC6VIDS: |
1720 | *data0 |= 0x1; |
1721 | break; |
1722 | } |
1723 | |
1724 | gvt_dbg_core("VM(%d) write %x to mailbox, return data0 %x\n" , |
1725 | vgpu->id, value, *data0); |
1726 | /** |
1727 | * PCODE_READY clear means ready for pcode read/write, |
1728 | * PCODE_ERROR_MASK clear means no error happened. In GVT-g we |
1729 | * always emulate as pcode read/write success and ready for access |
1730 | * anytime, since we don't touch real physical registers here. |
1731 | */ |
1732 | value &= ~(GEN6_PCODE_READY | GEN6_PCODE_ERROR_MASK); |
1733 | return intel_vgpu_default_mmio_write(vgpu, offset, p_data: &value, bytes); |
1734 | } |
1735 | |
1736 | static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset, |
1737 | void *p_data, unsigned int bytes) |
1738 | { |
1739 | u32 value = *(u32 *)p_data; |
1740 | const struct intel_engine_cs *engine = |
1741 | intel_gvt_render_mmio_to_engine(gvt: vgpu->gvt, offset); |
1742 | |
1743 | if (value != 0 && |
1744 | !intel_gvt_ggtt_validate_range(vgpu, addr: value, I915_GTT_PAGE_SIZE)) { |
1745 | gvt_vgpu_err("write invalid HWSP address, reg:0x%x, value:0x%x\n" , |
1746 | offset, value); |
1747 | return -EINVAL; |
1748 | } |
1749 | |
1750 | /* |
1751 | * Need to emulate all the HWSP register write to ensure host can |
1752 | * update the VM CSB status correctly. Here listed registers can |
1753 | * support BDW, SKL or other platforms with same HWSP registers. |
1754 | */ |
1755 | if (unlikely(!engine)) { |
1756 | gvt_vgpu_err("access unknown hardware status page register:0x%x\n" , |
1757 | offset); |
1758 | return -EINVAL; |
1759 | } |
1760 | vgpu->hws_pga[engine->id] = value; |
1761 | gvt_dbg_mmio("VM(%d) write: 0x%x to HWSP: 0x%x\n" , |
1762 | vgpu->id, value, offset); |
1763 | |
1764 | return intel_vgpu_default_mmio_write(vgpu, offset, p_data: &value, bytes); |
1765 | } |
1766 | |
1767 | static int skl_power_well_ctl_write(struct intel_vgpu *vgpu, |
1768 | unsigned int offset, void *p_data, unsigned int bytes) |
1769 | { |
1770 | u32 v = *(u32 *)p_data; |
1771 | |
1772 | if (IS_BROXTON(vgpu->gvt->gt->i915)) |
1773 | v &= (1 << 31) | (1 << 29); |
1774 | else |
1775 | v &= (1 << 31) | (1 << 29) | (1 << 9) | |
1776 | (1 << 7) | (1 << 5) | (1 << 3) | (1 << 1); |
1777 | v |= (v >> 1); |
1778 | |
1779 | return intel_vgpu_default_mmio_write(vgpu, offset, p_data: &v, bytes); |
1780 | } |
1781 | |
1782 | static int skl_lcpll_write(struct intel_vgpu *vgpu, unsigned int offset, |
1783 | void *p_data, unsigned int bytes) |
1784 | { |
1785 | u32 v = *(u32 *)p_data; |
1786 | |
1787 | /* other bits are MBZ. */ |
1788 | v &= (1 << 31) | (1 << 30); |
1789 | v & (1 << 31) ? (v |= (1 << 30)) : (v &= ~(1 << 30)); |
1790 | |
1791 | vgpu_vreg(vgpu, offset) = v; |
1792 | |
1793 | return 0; |
1794 | } |
1795 | |
1796 | static int bxt_de_pll_enable_write(struct intel_vgpu *vgpu, |
1797 | unsigned int offset, void *p_data, unsigned int bytes) |
1798 | { |
1799 | u32 v = *(u32 *)p_data; |
1800 | |
1801 | if (v & BXT_DE_PLL_PLL_ENABLE) |
1802 | v |= BXT_DE_PLL_LOCK; |
1803 | |
1804 | vgpu_vreg(vgpu, offset) = v; |
1805 | |
1806 | return 0; |
1807 | } |
1808 | |
1809 | static int bxt_port_pll_enable_write(struct intel_vgpu *vgpu, |
1810 | unsigned int offset, void *p_data, unsigned int bytes) |
1811 | { |
1812 | u32 v = *(u32 *)p_data; |
1813 | |
1814 | if (v & PORT_PLL_ENABLE) |
1815 | v |= PORT_PLL_LOCK; |
1816 | |
1817 | vgpu_vreg(vgpu, offset) = v; |
1818 | |
1819 | return 0; |
1820 | } |
1821 | |
1822 | static int bxt_phy_ctl_family_write(struct intel_vgpu *vgpu, |
1823 | unsigned int offset, void *p_data, unsigned int bytes) |
1824 | { |
1825 | u32 v = *(u32 *)p_data; |
1826 | u32 data = v & COMMON_RESET_DIS ? BXT_PHY_LANE_ENABLED : 0; |
1827 | |
1828 | switch (offset) { |
1829 | case _PHY_CTL_FAMILY_EDP: |
1830 | vgpu_vreg(vgpu, _BXT_PHY_CTL_DDI_A) = data; |
1831 | break; |
1832 | case _PHY_CTL_FAMILY_DDI: |
1833 | vgpu_vreg(vgpu, _BXT_PHY_CTL_DDI_B) = data; |
1834 | vgpu_vreg(vgpu, _BXT_PHY_CTL_DDI_C) = data; |
1835 | break; |
1836 | } |
1837 | |
1838 | vgpu_vreg(vgpu, offset) = v; |
1839 | |
1840 | return 0; |
1841 | } |
1842 | |
1843 | static int bxt_port_tx_dw3_read(struct intel_vgpu *vgpu, |
1844 | unsigned int offset, void *p_data, unsigned int bytes) |
1845 | { |
1846 | u32 v = vgpu_vreg(vgpu, offset); |
1847 | |
1848 | v &= ~UNIQUE_TRANGE_EN_METHOD; |
1849 | |
1850 | vgpu_vreg(vgpu, offset) = v; |
1851 | |
1852 | return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes); |
1853 | } |
1854 | |
1855 | static int bxt_pcs_dw12_grp_write(struct intel_vgpu *vgpu, |
1856 | unsigned int offset, void *p_data, unsigned int bytes) |
1857 | { |
1858 | u32 v = *(u32 *)p_data; |
1859 | |
1860 | if (offset == _PORT_PCS_DW12_GRP_A || offset == _PORT_PCS_DW12_GRP_B) { |
1861 | vgpu_vreg(vgpu, offset - 0x600) = v; |
1862 | vgpu_vreg(vgpu, offset - 0x800) = v; |
1863 | } else { |
1864 | vgpu_vreg(vgpu, offset - 0x400) = v; |
1865 | vgpu_vreg(vgpu, offset - 0x600) = v; |
1866 | } |
1867 | |
1868 | vgpu_vreg(vgpu, offset) = v; |
1869 | |
1870 | return 0; |
1871 | } |
1872 | |
1873 | static int bxt_gt_disp_pwron_write(struct intel_vgpu *vgpu, |
1874 | unsigned int offset, void *p_data, unsigned int bytes) |
1875 | { |
1876 | u32 v = *(u32 *)p_data; |
1877 | |
1878 | if (v & BIT(0)) { |
1879 | vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) &= |
1880 | ~PHY_RESERVED; |
1881 | vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY0)) |= |
1882 | PHY_POWER_GOOD; |
1883 | } |
1884 | |
1885 | if (v & BIT(1)) { |
1886 | vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) &= |
1887 | ~PHY_RESERVED; |
1888 | vgpu_vreg_t(vgpu, BXT_PORT_CL1CM_DW0(DPIO_PHY1)) |= |
1889 | PHY_POWER_GOOD; |
1890 | } |
1891 | |
1892 | |
1893 | vgpu_vreg(vgpu, offset) = v; |
1894 | |
1895 | return 0; |
1896 | } |
1897 | |
1898 | static int edp_psr_imr_iir_write(struct intel_vgpu *vgpu, |
1899 | unsigned int offset, void *p_data, unsigned int bytes) |
1900 | { |
1901 | vgpu_vreg(vgpu, offset) = 0; |
1902 | return 0; |
1903 | } |
1904 | |
1905 | /* |
1906 | * FixMe: |
1907 | * If guest fills non-priv batch buffer on ApolloLake/Broxton as Mesa i965 did: |
1908 | * 717e7539124d (i965: Use a WC map and memcpy for the batch instead of pwrite.) |
1909 | * Due to the missing flush of bb filled by VM vCPU, host GPU hangs on executing |
1910 | * these MI_BATCH_BUFFER. |
1911 | * Temporarily workaround this by setting SNOOP bit for PAT3 used by PPGTT |
1912 | * PML4 PTE: PAT(0) PCD(1) PWT(1). |
1913 | * The performance is still expected to be low, will need further improvement. |
1914 | */ |
1915 | static int bxt_ppat_low_write(struct intel_vgpu *vgpu, unsigned int offset, |
1916 | void *p_data, unsigned int bytes) |
1917 | { |
1918 | u64 pat = |
1919 | GEN8_PPAT(0, CHV_PPAT_SNOOP) | |
1920 | GEN8_PPAT(1, 0) | |
1921 | GEN8_PPAT(2, 0) | |
1922 | GEN8_PPAT(3, CHV_PPAT_SNOOP) | |
1923 | GEN8_PPAT(4, CHV_PPAT_SNOOP) | |
1924 | GEN8_PPAT(5, CHV_PPAT_SNOOP) | |
1925 | GEN8_PPAT(6, CHV_PPAT_SNOOP) | |
1926 | GEN8_PPAT(7, CHV_PPAT_SNOOP); |
1927 | |
1928 | vgpu_vreg(vgpu, offset) = lower_32_bits(pat); |
1929 | |
1930 | return 0; |
1931 | } |
1932 | |
1933 | static int guc_status_read(struct intel_vgpu *vgpu, |
1934 | unsigned int offset, void *p_data, |
1935 | unsigned int bytes) |
1936 | { |
1937 | /* keep MIA_IN_RESET before clearing */ |
1938 | read_vreg(vgpu, offset, p_data, bytes); |
1939 | vgpu_vreg(vgpu, offset) &= ~GS_MIA_IN_RESET; |
1940 | return 0; |
1941 | } |
1942 | |
1943 | static int mmio_read_from_hw(struct intel_vgpu *vgpu, |
1944 | unsigned int offset, void *p_data, unsigned int bytes) |
1945 | { |
1946 | struct intel_gvt *gvt = vgpu->gvt; |
1947 | const struct intel_engine_cs *engine = |
1948 | intel_gvt_render_mmio_to_engine(gvt, offset); |
1949 | |
1950 | /** |
1951 | * Read HW reg in following case |
1952 | * a. the offset isn't a ring mmio |
1953 | * b. the offset's ring is running on hw. |
1954 | * c. the offset is ring time stamp mmio |
1955 | */ |
1956 | |
1957 | if (!engine || |
1958 | vgpu == gvt->scheduler.engine_owner[engine->id] || |
1959 | offset == i915_mmio_reg_offset(RING_TIMESTAMP(engine->mmio_base)) || |
1960 | offset == i915_mmio_reg_offset(RING_TIMESTAMP_UDW(engine->mmio_base))) { |
1961 | mmio_hw_access_pre(gt: gvt->gt); |
1962 | vgpu_vreg(vgpu, offset) = |
1963 | intel_uncore_read(uncore: gvt->gt->uncore, _MMIO(offset)); |
1964 | mmio_hw_access_post(gt: gvt->gt); |
1965 | } |
1966 | |
1967 | return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes); |
1968 | } |
1969 | |
1970 | static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
1971 | void *p_data, unsigned int bytes) |
1972 | { |
1973 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
1974 | const struct intel_engine_cs *engine = intel_gvt_render_mmio_to_engine(gvt: vgpu->gvt, offset); |
1975 | struct intel_vgpu_execlist *execlist; |
1976 | u32 data = *(u32 *)p_data; |
1977 | int ret = 0; |
1978 | |
1979 | if (drm_WARN_ON(&i915->drm, !engine)) |
1980 | return -EINVAL; |
1981 | |
1982 | /* |
1983 | * Due to d3_entered is used to indicate skipping PPGTT invalidation on |
1984 | * vGPU reset, it's set on D0->D3 on PCI config write, and cleared after |
1985 | * vGPU reset if in resuming. |
1986 | * In S0ix exit, the device power state also transite from D3 to D0 as |
1987 | * S3 resume, but no vGPU reset (triggered by QEMU devic model). After |
1988 | * S0ix exit, all engines continue to work. However the d3_entered |
1989 | * remains set which will break next vGPU reset logic (miss the expected |
1990 | * PPGTT invalidation). |
1991 | * Engines can only work in D0. Thus the 1st elsp write gives GVT a |
1992 | * chance to clear d3_entered. |
1993 | */ |
1994 | if (vgpu->d3_entered) |
1995 | vgpu->d3_entered = false; |
1996 | |
1997 | execlist = &vgpu->submission.execlist[engine->id]; |
1998 | |
1999 | execlist->elsp_dwords.data[3 - execlist->elsp_dwords.index] = data; |
2000 | if (execlist->elsp_dwords.index == 3) { |
2001 | ret = intel_vgpu_submit_execlist(vgpu, engine); |
2002 | if(ret) |
2003 | gvt_vgpu_err("fail submit workload on ring %s\n" , |
2004 | engine->name); |
2005 | } |
2006 | |
2007 | ++execlist->elsp_dwords.index; |
2008 | execlist->elsp_dwords.index &= 0x3; |
2009 | return ret; |
2010 | } |
2011 | |
2012 | static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
2013 | void *p_data, unsigned int bytes) |
2014 | { |
2015 | u32 data = *(u32 *)p_data; |
2016 | const struct intel_engine_cs *engine = |
2017 | intel_gvt_render_mmio_to_engine(gvt: vgpu->gvt, offset); |
2018 | bool enable_execlist; |
2019 | int ret; |
2020 | |
2021 | (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(1); |
2022 | if (IS_COFFEELAKE(vgpu->gvt->gt->i915) || |
2023 | IS_COMETLAKE(vgpu->gvt->gt->i915)) |
2024 | (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(2); |
2025 | write_vreg(vgpu, offset, p_data, bytes); |
2026 | |
2027 | if (IS_MASKED_BITS_ENABLED(data, 1)) { |
2028 | enter_failsafe_mode(vgpu, reason: GVT_FAILSAFE_UNSUPPORTED_GUEST); |
2029 | return 0; |
2030 | } |
2031 | |
2032 | if ((IS_COFFEELAKE(vgpu->gvt->gt->i915) || |
2033 | IS_COMETLAKE(vgpu->gvt->gt->i915)) && |
2034 | IS_MASKED_BITS_ENABLED(data, 2)) { |
2035 | enter_failsafe_mode(vgpu, reason: GVT_FAILSAFE_UNSUPPORTED_GUEST); |
2036 | return 0; |
2037 | } |
2038 | |
2039 | /* when PPGTT mode enabled, we will check if guest has called |
2040 | * pvinfo, if not, we will treat this guest as non-gvtg-aware |
2041 | * guest, and stop emulating its cfg space, mmio, gtt, etc. |
2042 | */ |
2043 | if ((IS_MASKED_BITS_ENABLED(data, GFX_PPGTT_ENABLE) || |
2044 | IS_MASKED_BITS_ENABLED(data, GFX_RUN_LIST_ENABLE)) && |
2045 | !vgpu->pv_notified) { |
2046 | enter_failsafe_mode(vgpu, reason: GVT_FAILSAFE_UNSUPPORTED_GUEST); |
2047 | return 0; |
2048 | } |
2049 | if (IS_MASKED_BITS_ENABLED(data, GFX_RUN_LIST_ENABLE) || |
2050 | IS_MASKED_BITS_DISABLED(data, GFX_RUN_LIST_ENABLE)) { |
2051 | enable_execlist = !!(data & GFX_RUN_LIST_ENABLE); |
2052 | |
2053 | gvt_dbg_core("EXECLIST %s on ring %s\n" , |
2054 | (enable_execlist ? "enabling" : "disabling" ), |
2055 | engine->name); |
2056 | |
2057 | if (!enable_execlist) |
2058 | return 0; |
2059 | |
2060 | ret = intel_vgpu_select_submission_ops(vgpu, |
2061 | engine_mask: engine->mask, |
2062 | interface: INTEL_VGPU_EXECLIST_SUBMISSION); |
2063 | if (ret) |
2064 | return ret; |
2065 | |
2066 | intel_vgpu_start_schedule(vgpu); |
2067 | } |
2068 | return 0; |
2069 | } |
2070 | |
2071 | static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu, |
2072 | unsigned int offset, void *p_data, unsigned int bytes) |
2073 | { |
2074 | unsigned int id = 0; |
2075 | |
2076 | write_vreg(vgpu, offset, p_data, bytes); |
2077 | vgpu_vreg(vgpu, offset) = 0; |
2078 | |
2079 | switch (offset) { |
2080 | case 0x4260: |
2081 | id = RCS0; |
2082 | break; |
2083 | case 0x4264: |
2084 | id = VCS0; |
2085 | break; |
2086 | case 0x4268: |
2087 | id = VCS1; |
2088 | break; |
2089 | case 0x426c: |
2090 | id = BCS0; |
2091 | break; |
2092 | case 0x4270: |
2093 | id = VECS0; |
2094 | break; |
2095 | default: |
2096 | return -EINVAL; |
2097 | } |
2098 | set_bit(nr: id, addr: (void *)vgpu->submission.tlb_handle_pending); |
2099 | |
2100 | return 0; |
2101 | } |
2102 | |
2103 | static int ring_reset_ctl_write(struct intel_vgpu *vgpu, |
2104 | unsigned int offset, void *p_data, unsigned int bytes) |
2105 | { |
2106 | u32 data; |
2107 | |
2108 | write_vreg(vgpu, offset, p_data, bytes); |
2109 | data = vgpu_vreg(vgpu, offset); |
2110 | |
2111 | if (IS_MASKED_BITS_ENABLED(data, RESET_CTL_REQUEST_RESET)) |
2112 | data |= RESET_CTL_READY_TO_RESET; |
2113 | else if (data & _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)) |
2114 | data &= ~RESET_CTL_READY_TO_RESET; |
2115 | |
2116 | vgpu_vreg(vgpu, offset) = data; |
2117 | return 0; |
2118 | } |
2119 | |
2120 | static int csfe_chicken1_mmio_write(struct intel_vgpu *vgpu, |
2121 | unsigned int offset, void *p_data, |
2122 | unsigned int bytes) |
2123 | { |
2124 | u32 data = *(u32 *)p_data; |
2125 | |
2126 | (*(u32 *)p_data) &= ~_MASKED_BIT_ENABLE(0x18); |
2127 | write_vreg(vgpu, offset, p_data, bytes); |
2128 | |
2129 | if (IS_MASKED_BITS_ENABLED(data, 0x10) || |
2130 | IS_MASKED_BITS_ENABLED(data, 0x8)) |
2131 | enter_failsafe_mode(vgpu, reason: GVT_FAILSAFE_UNSUPPORTED_GUEST); |
2132 | |
2133 | return 0; |
2134 | } |
2135 | |
2136 | #define MMIO_F(reg, s, f, am, rm, d, r, w) do { \ |
2137 | ret = setup_mmio_info(gvt, i915_mmio_reg_offset(reg), \ |
2138 | s, f, am, rm, d, r, w); \ |
2139 | if (ret) \ |
2140 | return ret; \ |
2141 | } while (0) |
2142 | |
2143 | #define MMIO_DH(reg, d, r, w) \ |
2144 | MMIO_F(reg, 4, 0, 0, 0, d, r, w) |
2145 | |
2146 | #define MMIO_DFH(reg, d, f, r, w) \ |
2147 | MMIO_F(reg, 4, f, 0, 0, d, r, w) |
2148 | |
2149 | #define MMIO_GM(reg, d, r, w) \ |
2150 | MMIO_F(reg, 4, F_GMADR, 0xFFFFF000, 0, d, r, w) |
2151 | |
2152 | #define MMIO_GM_RDR(reg, d, r, w) \ |
2153 | MMIO_F(reg, 4, F_GMADR | F_CMD_ACCESS, 0xFFFFF000, 0, d, r, w) |
2154 | |
2155 | #define MMIO_RO(reg, d, f, rm, r, w) \ |
2156 | MMIO_F(reg, 4, F_RO | f, 0, rm, d, r, w) |
2157 | |
2158 | #define MMIO_RING_F(prefix, s, f, am, rm, d, r, w) do { \ |
2159 | MMIO_F(prefix(RENDER_RING_BASE), s, f, am, rm, d, r, w); \ |
2160 | MMIO_F(prefix(BLT_RING_BASE), s, f, am, rm, d, r, w); \ |
2161 | MMIO_F(prefix(GEN6_BSD_RING_BASE), s, f, am, rm, d, r, w); \ |
2162 | MMIO_F(prefix(VEBOX_RING_BASE), s, f, am, rm, d, r, w); \ |
2163 | if (HAS_ENGINE(gvt->gt, VCS1)) \ |
2164 | MMIO_F(prefix(GEN8_BSD2_RING_BASE), s, f, am, rm, d, r, w); \ |
2165 | } while (0) |
2166 | |
2167 | #define MMIO_RING_DFH(prefix, d, f, r, w) \ |
2168 | MMIO_RING_F(prefix, 4, f, 0, 0, d, r, w) |
2169 | |
2170 | #define MMIO_RING_GM(prefix, d, r, w) \ |
2171 | MMIO_RING_F(prefix, 4, F_GMADR, 0xFFFF0000, 0, d, r, w) |
2172 | |
2173 | #define MMIO_RING_GM_RDR(prefix, d, r, w) \ |
2174 | MMIO_RING_F(prefix, 4, F_GMADR | F_CMD_ACCESS, 0xFFFF0000, 0, d, r, w) |
2175 | |
2176 | #define MMIO_RING_RO(prefix, d, f, rm, r, w) \ |
2177 | MMIO_RING_F(prefix, 4, F_RO | f, 0, rm, d, r, w) |
2178 | |
2179 | static int init_generic_mmio_info(struct intel_gvt *gvt) |
2180 | { |
2181 | struct drm_i915_private *dev_priv = gvt->gt->i915; |
2182 | int ret; |
2183 | |
2184 | MMIO_RING_DFH(RING_IMR, D_ALL, 0, NULL, |
2185 | intel_vgpu_reg_imr_handler); |
2186 | |
2187 | MMIO_DFH(SDEIMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler); |
2188 | MMIO_DFH(SDEIER, D_ALL, 0, NULL, intel_vgpu_reg_ier_handler); |
2189 | MMIO_DFH(SDEIIR, D_ALL, 0, NULL, intel_vgpu_reg_iir_handler); |
2190 | |
2191 | MMIO_RING_DFH(RING_HWSTAM, D_ALL, 0, NULL, NULL); |
2192 | |
2193 | |
2194 | MMIO_DH(GEN8_GAMW_ECO_DEV_RW_IA, D_BDW_PLUS, NULL, |
2195 | gamw_echo_dev_rw_ia_write); |
2196 | |
2197 | MMIO_GM_RDR(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL); |
2198 | MMIO_GM_RDR(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL); |
2199 | MMIO_GM_RDR(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL); |
2200 | |
2201 | #define RING_REG(base) _MMIO((base) + 0x28) |
2202 | MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2203 | #undef RING_REG |
2204 | |
2205 | #define RING_REG(base) _MMIO((base) + 0x134) |
2206 | MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2207 | #undef RING_REG |
2208 | |
2209 | #define RING_REG(base) _MMIO((base) + 0x6c) |
2210 | MMIO_RING_DFH(RING_REG, D_ALL, 0, mmio_read_from_hw, NULL); |
2211 | #undef RING_REG |
2212 | MMIO_DH(GEN7_SC_INSTDONE, D_BDW_PLUS, mmio_read_from_hw, NULL); |
2213 | |
2214 | MMIO_GM_RDR(_MMIO(0x2148), D_ALL, NULL, NULL); |
2215 | MMIO_GM_RDR(CCID(RENDER_RING_BASE), D_ALL, NULL, NULL); |
2216 | MMIO_GM_RDR(_MMIO(0x12198), D_ALL, NULL, NULL); |
2217 | |
2218 | MMIO_RING_DFH(RING_TAIL, D_ALL, 0, NULL, NULL); |
2219 | MMIO_RING_DFH(RING_HEAD, D_ALL, 0, NULL, NULL); |
2220 | MMIO_RING_DFH(RING_CTL, D_ALL, 0, NULL, NULL); |
2221 | MMIO_RING_DFH(RING_ACTHD, D_ALL, 0, mmio_read_from_hw, NULL); |
2222 | MMIO_RING_GM(RING_START, D_ALL, NULL, NULL); |
2223 | |
2224 | /* RING MODE */ |
2225 | #define RING_REG(base) _MMIO((base) + 0x29c) |
2226 | MMIO_RING_DFH(RING_REG, D_ALL, |
2227 | F_MODE_MASK | F_CMD_ACCESS | F_CMD_WRITE_PATCH, NULL, |
2228 | ring_mode_mmio_write); |
2229 | #undef RING_REG |
2230 | |
2231 | MMIO_RING_DFH(RING_MI_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, |
2232 | NULL, NULL); |
2233 | MMIO_RING_DFH(RING_INSTPM, D_ALL, F_MODE_MASK | F_CMD_ACCESS, |
2234 | NULL, NULL); |
2235 | MMIO_RING_DFH(RING_TIMESTAMP, D_ALL, F_CMD_ACCESS, |
2236 | mmio_read_from_hw, NULL); |
2237 | MMIO_RING_DFH(RING_TIMESTAMP_UDW, D_ALL, F_CMD_ACCESS, |
2238 | mmio_read_from_hw, NULL); |
2239 | |
2240 | MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2241 | MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK | F_CMD_ACCESS, |
2242 | NULL, NULL); |
2243 | MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2244 | MMIO_DFH(CACHE_MODE_0, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2245 | MMIO_DFH(_MMIO(0x2124), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2246 | |
2247 | MMIO_DFH(_MMIO(0x20dc), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2248 | MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2249 | MMIO_DFH(_MMIO(0x2088), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2250 | MMIO_DFH(FF_SLICE_CS_CHICKEN2, D_ALL, |
2251 | F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2252 | MMIO_DFH(_MMIO(0x2470), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2253 | MMIO_DFH(GAM_ECOCHK, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2254 | MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, |
2255 | NULL, NULL); |
2256 | MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, |
2257 | NULL, NULL); |
2258 | MMIO_DFH(_MMIO(0x9030), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2259 | MMIO_DFH(_MMIO(0x20a0), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2260 | MMIO_DFH(_MMIO(0x2420), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2261 | MMIO_DFH(_MMIO(0x2430), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2262 | MMIO_DFH(_MMIO(0x2434), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2263 | MMIO_DFH(_MMIO(0x2438), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2264 | MMIO_DFH(_MMIO(0x243c), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2265 | MMIO_DFH(_MMIO(0x7018), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2266 | MMIO_DFH(HSW_HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2267 | MMIO_DFH(GEN7_HALF_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2268 | |
2269 | /* display */ |
2270 | MMIO_DH(TRANSCONF(TRANSCODER_A), D_ALL, NULL, pipeconf_mmio_write); |
2271 | MMIO_DH(TRANSCONF(TRANSCODER_B), D_ALL, NULL, pipeconf_mmio_write); |
2272 | MMIO_DH(TRANSCONF(TRANSCODER_C), D_ALL, NULL, pipeconf_mmio_write); |
2273 | MMIO_DH(TRANSCONF(TRANSCODER_EDP), D_ALL, NULL, pipeconf_mmio_write); |
2274 | MMIO_DH(DSPSURF(PIPE_A), D_ALL, NULL, pri_surf_mmio_write); |
2275 | MMIO_DH(REG_50080(PIPE_A, PLANE_PRIMARY), D_ALL, NULL, |
2276 | reg50080_mmio_write); |
2277 | MMIO_DH(DSPSURF(PIPE_B), D_ALL, NULL, pri_surf_mmio_write); |
2278 | MMIO_DH(REG_50080(PIPE_B, PLANE_PRIMARY), D_ALL, NULL, |
2279 | reg50080_mmio_write); |
2280 | MMIO_DH(DSPSURF(PIPE_C), D_ALL, NULL, pri_surf_mmio_write); |
2281 | MMIO_DH(REG_50080(PIPE_C, PLANE_PRIMARY), D_ALL, NULL, |
2282 | reg50080_mmio_write); |
2283 | MMIO_DH(SPRSURF(PIPE_A), D_ALL, NULL, spr_surf_mmio_write); |
2284 | MMIO_DH(REG_50080(PIPE_A, PLANE_SPRITE0), D_ALL, NULL, |
2285 | reg50080_mmio_write); |
2286 | MMIO_DH(SPRSURF(PIPE_B), D_ALL, NULL, spr_surf_mmio_write); |
2287 | MMIO_DH(REG_50080(PIPE_B, PLANE_SPRITE0), D_ALL, NULL, |
2288 | reg50080_mmio_write); |
2289 | MMIO_DH(SPRSURF(PIPE_C), D_ALL, NULL, spr_surf_mmio_write); |
2290 | MMIO_DH(REG_50080(PIPE_C, PLANE_SPRITE0), D_ALL, NULL, |
2291 | reg50080_mmio_write); |
2292 | |
2293 | MMIO_F(PCH_GMBUS0, 4 * 4, 0, 0, 0, D_ALL, gmbus_mmio_read, |
2294 | gmbus_mmio_write); |
2295 | MMIO_F(PCH_GPIO_BASE, 6 * 4, F_UNALIGN, 0, 0, D_ALL, NULL, NULL); |
2296 | |
2297 | MMIO_F(_MMIO(_PCH_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, |
2298 | dp_aux_ch_ctl_mmio_write); |
2299 | MMIO_F(_MMIO(_PCH_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, |
2300 | dp_aux_ch_ctl_mmio_write); |
2301 | MMIO_F(_MMIO(_PCH_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL, |
2302 | dp_aux_ch_ctl_mmio_write); |
2303 | |
2304 | MMIO_DH(PCH_ADPA, D_PRE_SKL, NULL, pch_adpa_mmio_write); |
2305 | |
2306 | MMIO_DH(_MMIO(_PCH_TRANSACONF), D_ALL, NULL, transconf_mmio_write); |
2307 | MMIO_DH(_MMIO(_PCH_TRANSBCONF), D_ALL, NULL, transconf_mmio_write); |
2308 | |
2309 | MMIO_DH(FDI_RX_IIR(PIPE_A), D_ALL, NULL, fdi_rx_iir_mmio_write); |
2310 | MMIO_DH(FDI_RX_IIR(PIPE_B), D_ALL, NULL, fdi_rx_iir_mmio_write); |
2311 | MMIO_DH(FDI_RX_IIR(PIPE_C), D_ALL, NULL, fdi_rx_iir_mmio_write); |
2312 | MMIO_DH(FDI_RX_IMR(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status); |
2313 | MMIO_DH(FDI_RX_IMR(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status); |
2314 | MMIO_DH(FDI_RX_IMR(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status); |
2315 | MMIO_DH(FDI_RX_CTL(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status); |
2316 | MMIO_DH(FDI_RX_CTL(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status); |
2317 | MMIO_DH(FDI_RX_CTL(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status); |
2318 | MMIO_DH(PCH_PP_CONTROL, D_ALL, NULL, pch_pp_control_mmio_write); |
2319 | MMIO_DH(_MMIO(0xe651c), D_ALL, dpy_reg_mmio_read, NULL); |
2320 | MMIO_DH(_MMIO(0xe661c), D_ALL, dpy_reg_mmio_read, NULL); |
2321 | MMIO_DH(_MMIO(0xe671c), D_ALL, dpy_reg_mmio_read, NULL); |
2322 | MMIO_DH(_MMIO(0xe681c), D_ALL, dpy_reg_mmio_read, NULL); |
2323 | MMIO_DH(_MMIO(0xe6c04), D_ALL, dpy_reg_mmio_read, NULL); |
2324 | MMIO_DH(_MMIO(0xe6e1c), D_ALL, dpy_reg_mmio_read, NULL); |
2325 | |
2326 | MMIO_RO(PCH_PORT_HOTPLUG, D_ALL, 0, |
2327 | PORTA_HOTPLUG_STATUS_MASK |
2328 | | PORTB_HOTPLUG_STATUS_MASK |
2329 | | PORTC_HOTPLUG_STATUS_MASK |
2330 | | PORTD_HOTPLUG_STATUS_MASK, |
2331 | NULL, NULL); |
2332 | |
2333 | MMIO_DH(LCPLL_CTL, D_ALL, NULL, lcpll_ctl_mmio_write); |
2334 | MMIO_DH(SOUTH_CHICKEN2, D_ALL, NULL, south_chicken2_mmio_write); |
2335 | MMIO_DH(SFUSE_STRAP, D_ALL, NULL, NULL); |
2336 | MMIO_DH(SBI_DATA, D_ALL, sbi_data_mmio_read, NULL); |
2337 | MMIO_DH(SBI_CTL_STAT, D_ALL, NULL, sbi_ctl_mmio_write); |
2338 | |
2339 | MMIO_F(_MMIO(_DPA_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_ALL, NULL, |
2340 | dp_aux_ch_ctl_mmio_write); |
2341 | |
2342 | MMIO_DH(DDI_BUF_CTL(PORT_A), D_ALL, NULL, ddi_buf_ctl_mmio_write); |
2343 | MMIO_DH(DDI_BUF_CTL(PORT_B), D_ALL, NULL, ddi_buf_ctl_mmio_write); |
2344 | MMIO_DH(DDI_BUF_CTL(PORT_C), D_ALL, NULL, ddi_buf_ctl_mmio_write); |
2345 | MMIO_DH(DDI_BUF_CTL(PORT_D), D_ALL, NULL, ddi_buf_ctl_mmio_write); |
2346 | MMIO_DH(DDI_BUF_CTL(PORT_E), D_ALL, NULL, ddi_buf_ctl_mmio_write); |
2347 | |
2348 | MMIO_DH(DP_TP_CTL(PORT_A), D_ALL, NULL, dp_tp_ctl_mmio_write); |
2349 | MMIO_DH(DP_TP_CTL(PORT_B), D_ALL, NULL, dp_tp_ctl_mmio_write); |
2350 | MMIO_DH(DP_TP_CTL(PORT_C), D_ALL, NULL, dp_tp_ctl_mmio_write); |
2351 | MMIO_DH(DP_TP_CTL(PORT_D), D_ALL, NULL, dp_tp_ctl_mmio_write); |
2352 | MMIO_DH(DP_TP_CTL(PORT_E), D_ALL, NULL, dp_tp_ctl_mmio_write); |
2353 | |
2354 | MMIO_DH(DP_TP_STATUS(PORT_A), D_ALL, NULL, dp_tp_status_mmio_write); |
2355 | MMIO_DH(DP_TP_STATUS(PORT_B), D_ALL, NULL, dp_tp_status_mmio_write); |
2356 | MMIO_DH(DP_TP_STATUS(PORT_C), D_ALL, NULL, dp_tp_status_mmio_write); |
2357 | MMIO_DH(DP_TP_STATUS(PORT_D), D_ALL, NULL, dp_tp_status_mmio_write); |
2358 | MMIO_DH(DP_TP_STATUS(PORT_E), D_ALL, NULL, NULL); |
2359 | |
2360 | MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_A), D_ALL, NULL, NULL); |
2361 | MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_B), D_ALL, NULL, NULL); |
2362 | MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_C), D_ALL, NULL, NULL); |
2363 | MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_EDP), D_ALL, NULL, NULL); |
2364 | |
2365 | MMIO_DH(FORCEWAKE, D_ALL, NULL, NULL); |
2366 | MMIO_DFH(GTFIFODBG, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2367 | MMIO_DFH(GTFIFOCTL, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2368 | MMIO_DH(FORCEWAKE_MT, D_PRE_SKL, NULL, mul_force_wake_write); |
2369 | MMIO_DH(FORCEWAKE_ACK_HSW, D_BDW, NULL, NULL); |
2370 | MMIO_DH(GEN6_RC_CONTROL, D_ALL, NULL, NULL); |
2371 | MMIO_DH(GEN6_RC_STATE, D_ALL, NULL, NULL); |
2372 | MMIO_DH(HSW_PWR_WELL_CTL1, D_BDW, NULL, power_well_ctl_mmio_write); |
2373 | MMIO_DH(HSW_PWR_WELL_CTL2, D_BDW, NULL, power_well_ctl_mmio_write); |
2374 | MMIO_DH(HSW_PWR_WELL_CTL3, D_BDW, NULL, power_well_ctl_mmio_write); |
2375 | MMIO_DH(HSW_PWR_WELL_CTL4, D_BDW, NULL, power_well_ctl_mmio_write); |
2376 | MMIO_DH(HSW_PWR_WELL_CTL5, D_BDW, NULL, power_well_ctl_mmio_write); |
2377 | MMIO_DH(HSW_PWR_WELL_CTL6, D_BDW, NULL, power_well_ctl_mmio_write); |
2378 | |
2379 | MMIO_DH(GEN6_GDRST, D_ALL, NULL, gdrst_mmio_write); |
2380 | MMIO_F(FENCE_REG_GEN6_LO(0), 0x80, 0, 0, 0, D_ALL, fence_mmio_read, fence_mmio_write); |
2381 | MMIO_DH(CPU_VGACNTRL, D_ALL, NULL, vga_control_mmio_write); |
2382 | |
2383 | MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL); |
2384 | MMIO_DH(GFX_FLSH_CNTL_GEN6, D_ALL, NULL, NULL); |
2385 | |
2386 | MMIO_DH(GEN6_MBCTL, D_ALL, NULL, mbctl_write); |
2387 | MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2388 | |
2389 | MMIO_DH(FPGA_DBG, D_ALL, NULL, fpga_dbg_mmio_write); |
2390 | MMIO_DFH(_MMIO(0x215c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2391 | MMIO_DFH(_MMIO(0x2178), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2392 | MMIO_DFH(_MMIO(0x217c), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2393 | MMIO_DFH(_MMIO(0x12178), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2394 | MMIO_DFH(_MMIO(0x1217c), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2395 | |
2396 | MMIO_F(_MMIO(0x2290), 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL); |
2397 | MMIO_F(_MMIO(0x5200), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2398 | MMIO_F(_MMIO(0x5240), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2399 | MMIO_F(_MMIO(0x5280), 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2400 | |
2401 | MMIO_DFH(_MMIO(0x1c17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2402 | MMIO_DFH(_MMIO(0x1c178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2403 | MMIO_DFH(BCS_SWCTRL, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2404 | |
2405 | MMIO_F(HS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2406 | MMIO_F(DS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2407 | MMIO_F(IA_VERTICES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2408 | MMIO_F(IA_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2409 | MMIO_F(VS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2410 | MMIO_F(GS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2411 | MMIO_F(GS_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2412 | MMIO_F(CL_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2413 | MMIO_F(CL_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2414 | MMIO_F(PS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2415 | MMIO_F(PS_DEPTH_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL); |
2416 | MMIO_DH(_MMIO(0x4260), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); |
2417 | MMIO_DH(_MMIO(0x4264), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); |
2418 | MMIO_DH(_MMIO(0x4268), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); |
2419 | MMIO_DH(_MMIO(0x426c), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); |
2420 | MMIO_DH(_MMIO(0x4270), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler); |
2421 | MMIO_DFH(_MMIO(0x4094), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2422 | |
2423 | MMIO_DFH(ARB_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2424 | MMIO_RING_GM(RING_BBADDR, D_ALL, NULL, NULL); |
2425 | MMIO_DFH(_MMIO(0x2220), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2426 | MMIO_DFH(_MMIO(0x12220), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2427 | MMIO_DFH(_MMIO(0x22220), D_ALL, F_CMD_ACCESS, NULL, NULL); |
2428 | MMIO_RING_DFH(RING_SYNC_1, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2429 | MMIO_RING_DFH(RING_SYNC_0, D_ALL, F_CMD_ACCESS, NULL, NULL); |
2430 | MMIO_DFH(_MMIO(0x22178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2431 | MMIO_DFH(_MMIO(0x1a178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2432 | MMIO_DFH(_MMIO(0x1a17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2433 | MMIO_DFH(_MMIO(0x2217c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2434 | |
2435 | MMIO_DH(EDP_PSR_IMR, D_BDW_PLUS, NULL, edp_psr_imr_iir_write); |
2436 | MMIO_DH(EDP_PSR_IIR, D_BDW_PLUS, NULL, edp_psr_imr_iir_write); |
2437 | MMIO_DH(GUC_STATUS, D_ALL, guc_status_read, NULL); |
2438 | |
2439 | return 0; |
2440 | } |
2441 | |
2442 | static int init_bdw_mmio_info(struct intel_gvt *gvt) |
2443 | { |
2444 | int ret; |
2445 | |
2446 | MMIO_DH(GEN8_GT_IMR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2447 | MMIO_DH(GEN8_GT_IER(0), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2448 | MMIO_DH(GEN8_GT_IIR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2449 | |
2450 | MMIO_DH(GEN8_GT_IMR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2451 | MMIO_DH(GEN8_GT_IER(1), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2452 | MMIO_DH(GEN8_GT_IIR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2453 | |
2454 | MMIO_DH(GEN8_GT_IMR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2455 | MMIO_DH(GEN8_GT_IER(2), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2456 | MMIO_DH(GEN8_GT_IIR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2457 | |
2458 | MMIO_DH(GEN8_GT_IMR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2459 | MMIO_DH(GEN8_GT_IER(3), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2460 | MMIO_DH(GEN8_GT_IIR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2461 | |
2462 | MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_A), D_BDW_PLUS, NULL, |
2463 | intel_vgpu_reg_imr_handler); |
2464 | MMIO_DH(GEN8_DE_PIPE_IER(PIPE_A), D_BDW_PLUS, NULL, |
2465 | intel_vgpu_reg_ier_handler); |
2466 | MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_A), D_BDW_PLUS, NULL, |
2467 | intel_vgpu_reg_iir_handler); |
2468 | |
2469 | MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_B), D_BDW_PLUS, NULL, |
2470 | intel_vgpu_reg_imr_handler); |
2471 | MMIO_DH(GEN8_DE_PIPE_IER(PIPE_B), D_BDW_PLUS, NULL, |
2472 | intel_vgpu_reg_ier_handler); |
2473 | MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_B), D_BDW_PLUS, NULL, |
2474 | intel_vgpu_reg_iir_handler); |
2475 | |
2476 | MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_C), D_BDW_PLUS, NULL, |
2477 | intel_vgpu_reg_imr_handler); |
2478 | MMIO_DH(GEN8_DE_PIPE_IER(PIPE_C), D_BDW_PLUS, NULL, |
2479 | intel_vgpu_reg_ier_handler); |
2480 | MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_C), D_BDW_PLUS, NULL, |
2481 | intel_vgpu_reg_iir_handler); |
2482 | |
2483 | MMIO_DH(GEN8_DE_PORT_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2484 | MMIO_DH(GEN8_DE_PORT_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2485 | MMIO_DH(GEN8_DE_PORT_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2486 | |
2487 | MMIO_DH(GEN8_DE_MISC_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2488 | MMIO_DH(GEN8_DE_MISC_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2489 | MMIO_DH(GEN8_DE_MISC_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2490 | |
2491 | MMIO_DH(GEN8_PCU_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler); |
2492 | MMIO_DH(GEN8_PCU_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler); |
2493 | MMIO_DH(GEN8_PCU_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler); |
2494 | |
2495 | MMIO_DH(GEN8_MASTER_IRQ, D_BDW_PLUS, NULL, |
2496 | intel_vgpu_reg_master_irq_handler); |
2497 | |
2498 | MMIO_RING_DFH(RING_ACTHD_UDW, D_BDW_PLUS, 0, |
2499 | mmio_read_from_hw, NULL); |
2500 | |
2501 | #define RING_REG(base) _MMIO((base) + 0xd0) |
2502 | MMIO_RING_F(RING_REG, 4, F_RO, 0, |
2503 | ~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL, |
2504 | ring_reset_ctl_write); |
2505 | #undef RING_REG |
2506 | |
2507 | #define RING_REG(base) _MMIO((base) + 0x230) |
2508 | MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write); |
2509 | #undef RING_REG |
2510 | |
2511 | #define RING_REG(base) _MMIO((base) + 0x234) |
2512 | MMIO_RING_F(RING_REG, 8, F_RO, 0, ~0, D_BDW_PLUS, |
2513 | NULL, NULL); |
2514 | #undef RING_REG |
2515 | |
2516 | #define RING_REG(base) _MMIO((base) + 0x244) |
2517 | MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2518 | #undef RING_REG |
2519 | |
2520 | #define RING_REG(base) _MMIO((base) + 0x370) |
2521 | MMIO_RING_F(RING_REG, 48, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL); |
2522 | #undef RING_REG |
2523 | |
2524 | #define RING_REG(base) _MMIO((base) + 0x3a0) |
2525 | MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_MODE_MASK, NULL, NULL); |
2526 | #undef RING_REG |
2527 | |
2528 | MMIO_DH(GEN6_PCODE_MAILBOX, D_BDW_PLUS, NULL, mailbox_write); |
2529 | |
2530 | #define RING_REG(base) _MMIO((base) + 0x270) |
2531 | MMIO_RING_F(RING_REG, 32, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL); |
2532 | #undef RING_REG |
2533 | |
2534 | MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, hws_pga_write); |
2535 | |
2536 | MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2537 | |
2538 | MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, |
2539 | NULL, NULL); |
2540 | MMIO_DFH(GEN7_ROW_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, |
2541 | NULL, NULL); |
2542 | MMIO_DFH(GEN8_UCGCTL6, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2543 | |
2544 | MMIO_DFH(_MMIO(0xb1f0), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2545 | MMIO_DFH(_MMIO(0xb1c0), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2546 | MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2547 | MMIO_DFH(_MMIO(0xb100), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2548 | MMIO_DFH(_MMIO(0xb10c), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2549 | |
2550 | MMIO_F(_MMIO(0x24d0), 48, F_CMD_ACCESS | F_CMD_WRITE_PATCH, 0, 0, |
2551 | D_BDW_PLUS, NULL, force_nonpriv_write); |
2552 | |
2553 | MMIO_DFH(_MMIO(0x83a4), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2554 | |
2555 | MMIO_DFH(_MMIO(0x8430), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2556 | |
2557 | MMIO_DFH(_MMIO(0xe194), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2558 | MMIO_DFH(_MMIO(0xe188), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2559 | MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2560 | MMIO_DFH(_MMIO(0x2580), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2561 | |
2562 | MMIO_DFH(_MMIO(0x2248), D_BDW, F_CMD_ACCESS, NULL, NULL); |
2563 | |
2564 | MMIO_DFH(_MMIO(0xe220), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2565 | MMIO_DFH(_MMIO(0xe230), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2566 | MMIO_DFH(_MMIO(0xe240), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2567 | MMIO_DFH(_MMIO(0xe260), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2568 | MMIO_DFH(_MMIO(0xe270), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2569 | MMIO_DFH(_MMIO(0xe280), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2570 | MMIO_DFH(_MMIO(0xe2a0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2571 | MMIO_DFH(_MMIO(0xe2b0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2572 | MMIO_DFH(_MMIO(0xe2c0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2573 | MMIO_DFH(_MMIO(0x21f0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2574 | return 0; |
2575 | } |
2576 | |
2577 | static int init_skl_mmio_info(struct intel_gvt *gvt) |
2578 | { |
2579 | int ret; |
2580 | |
2581 | MMIO_DH(FORCEWAKE_RENDER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); |
2582 | MMIO_DH(FORCEWAKE_ACK_RENDER_GEN9, D_SKL_PLUS, NULL, NULL); |
2583 | MMIO_DH(FORCEWAKE_GT_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); |
2584 | MMIO_DH(FORCEWAKE_ACK_GT_GEN9, D_SKL_PLUS, NULL, NULL); |
2585 | MMIO_DH(FORCEWAKE_MEDIA_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write); |
2586 | MMIO_DH(FORCEWAKE_ACK_MEDIA_GEN9, D_SKL_PLUS, NULL, NULL); |
2587 | |
2588 | MMIO_F(DP_AUX_CH_CTL(AUX_CH_B), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, |
2589 | dp_aux_ch_ctl_mmio_write); |
2590 | MMIO_F(DP_AUX_CH_CTL(AUX_CH_C), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, |
2591 | dp_aux_ch_ctl_mmio_write); |
2592 | MMIO_F(DP_AUX_CH_CTL(AUX_CH_D), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL, |
2593 | dp_aux_ch_ctl_mmio_write); |
2594 | |
2595 | MMIO_DH(HSW_PWR_WELL_CTL2, D_SKL_PLUS, NULL, skl_power_well_ctl_write); |
2596 | |
2597 | MMIO_DH(DBUF_CTL_S(0), D_SKL_PLUS, NULL, gen9_dbuf_ctl_mmio_write); |
2598 | |
2599 | MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2600 | MMIO_DFH(MMCD_MISC_CTRL, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2601 | MMIO_DH(CHICKEN_PAR1_1, D_SKL_PLUS, NULL, NULL); |
2602 | MMIO_DH(LCPLL1_CTL, D_SKL_PLUS, NULL, skl_lcpll_write); |
2603 | MMIO_DH(LCPLL2_CTL, D_SKL_PLUS, NULL, skl_lcpll_write); |
2604 | MMIO_DH(DPLL_STATUS, D_SKL_PLUS, dpll_status_read, NULL); |
2605 | |
2606 | MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); |
2607 | MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); |
2608 | MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 0), D_SKL_PLUS, NULL, pf_write); |
2609 | MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 1), D_SKL_PLUS, NULL, pf_write); |
2610 | MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 0), D_SKL_PLUS, NULL, pf_write); |
2611 | MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 1), D_SKL_PLUS, NULL, pf_write); |
2612 | |
2613 | MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); |
2614 | MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); |
2615 | MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 0), D_SKL_PLUS, NULL, pf_write); |
2616 | MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 1), D_SKL_PLUS, NULL, pf_write); |
2617 | MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 0), D_SKL_PLUS, NULL, pf_write); |
2618 | MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 1), D_SKL_PLUS, NULL, pf_write); |
2619 | |
2620 | MMIO_DH(SKL_PS_CTRL(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write); |
2621 | MMIO_DH(SKL_PS_CTRL(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write); |
2622 | MMIO_DH(SKL_PS_CTRL(PIPE_B, 0), D_SKL_PLUS, NULL, pf_write); |
2623 | MMIO_DH(SKL_PS_CTRL(PIPE_B, 1), D_SKL_PLUS, NULL, pf_write); |
2624 | MMIO_DH(SKL_PS_CTRL(PIPE_C, 0), D_SKL_PLUS, NULL, pf_write); |
2625 | MMIO_DH(SKL_PS_CTRL(PIPE_C, 1), D_SKL_PLUS, NULL, pf_write); |
2626 | |
2627 | MMIO_DH(PLANE_BUF_CFG(PIPE_A, 0), D_SKL_PLUS, NULL, NULL); |
2628 | MMIO_DH(PLANE_BUF_CFG(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); |
2629 | MMIO_DH(PLANE_BUF_CFG(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); |
2630 | MMIO_DH(PLANE_BUF_CFG(PIPE_A, 3), D_SKL_PLUS, NULL, NULL); |
2631 | |
2632 | MMIO_DH(PLANE_BUF_CFG(PIPE_B, 0), D_SKL_PLUS, NULL, NULL); |
2633 | MMIO_DH(PLANE_BUF_CFG(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); |
2634 | MMIO_DH(PLANE_BUF_CFG(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); |
2635 | MMIO_DH(PLANE_BUF_CFG(PIPE_B, 3), D_SKL_PLUS, NULL, NULL); |
2636 | |
2637 | MMIO_DH(PLANE_BUF_CFG(PIPE_C, 0), D_SKL_PLUS, NULL, NULL); |
2638 | MMIO_DH(PLANE_BUF_CFG(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); |
2639 | MMIO_DH(PLANE_BUF_CFG(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); |
2640 | MMIO_DH(PLANE_BUF_CFG(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); |
2641 | |
2642 | MMIO_DH(CUR_BUF_CFG(PIPE_A), D_SKL_PLUS, NULL, NULL); |
2643 | MMIO_DH(CUR_BUF_CFG(PIPE_B), D_SKL_PLUS, NULL, NULL); |
2644 | MMIO_DH(CUR_BUF_CFG(PIPE_C), D_SKL_PLUS, NULL, NULL); |
2645 | |
2646 | MMIO_DH(PLANE_WM_TRANS(PIPE_A, 0), D_SKL_PLUS, NULL, NULL); |
2647 | MMIO_DH(PLANE_WM_TRANS(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); |
2648 | MMIO_DH(PLANE_WM_TRANS(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); |
2649 | |
2650 | MMIO_DH(PLANE_WM_TRANS(PIPE_B, 0), D_SKL_PLUS, NULL, NULL); |
2651 | MMIO_DH(PLANE_WM_TRANS(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); |
2652 | MMIO_DH(PLANE_WM_TRANS(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); |
2653 | |
2654 | MMIO_DH(PLANE_WM_TRANS(PIPE_C, 0), D_SKL_PLUS, NULL, NULL); |
2655 | MMIO_DH(PLANE_WM_TRANS(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); |
2656 | MMIO_DH(PLANE_WM_TRANS(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); |
2657 | |
2658 | MMIO_DH(CUR_WM_TRANS(PIPE_A), D_SKL_PLUS, NULL, NULL); |
2659 | MMIO_DH(CUR_WM_TRANS(PIPE_B), D_SKL_PLUS, NULL, NULL); |
2660 | MMIO_DH(CUR_WM_TRANS(PIPE_C), D_SKL_PLUS, NULL, NULL); |
2661 | |
2662 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 0), D_SKL_PLUS, NULL, NULL); |
2663 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 1), D_SKL_PLUS, NULL, NULL); |
2664 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 2), D_SKL_PLUS, NULL, NULL); |
2665 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 3), D_SKL_PLUS, NULL, NULL); |
2666 | |
2667 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 0), D_SKL_PLUS, NULL, NULL); |
2668 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 1), D_SKL_PLUS, NULL, NULL); |
2669 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 2), D_SKL_PLUS, NULL, NULL); |
2670 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 3), D_SKL_PLUS, NULL, NULL); |
2671 | |
2672 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 0), D_SKL_PLUS, NULL, NULL); |
2673 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 1), D_SKL_PLUS, NULL, NULL); |
2674 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 2), D_SKL_PLUS, NULL, NULL); |
2675 | MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 3), D_SKL_PLUS, NULL, NULL); |
2676 | |
2677 | MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL); |
2678 | MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL); |
2679 | MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL); |
2680 | MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL); |
2681 | |
2682 | MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL); |
2683 | MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL); |
2684 | MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL); |
2685 | MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL); |
2686 | |
2687 | MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL); |
2688 | MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL); |
2689 | MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL); |
2690 | MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL); |
2691 | |
2692 | MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL); |
2693 | MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL); |
2694 | MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL); |
2695 | MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL); |
2696 | |
2697 | MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL); |
2698 | MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL); |
2699 | MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL); |
2700 | MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL); |
2701 | |
2702 | MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL); |
2703 | MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL); |
2704 | MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL); |
2705 | MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL); |
2706 | |
2707 | MMIO_DFH(BDW_SCRATCH1, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2708 | |
2709 | MMIO_F(GEN9_GFX_MOCS(0), 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL_PLUS, |
2710 | NULL, NULL); |
2711 | MMIO_F(GEN7_L3CNTLREG2, 0x80, F_CMD_ACCESS, 0, 0, D_SKL_PLUS, |
2712 | NULL, NULL); |
2713 | |
2714 | MMIO_DFH(GEN7_FF_SLICE_CS_CHICKEN1, D_SKL_PLUS, |
2715 | F_MODE_MASK | F_CMD_ACCESS, NULL, NULL); |
2716 | MMIO_DFH(GEN9_CS_DEBUG_MODE1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, |
2717 | NULL, NULL); |
2718 | |
2719 | /* TRTT */ |
2720 | MMIO_DFH(TRVATTL3PTRDW(0), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2721 | MMIO_DFH(TRVATTL3PTRDW(1), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2722 | MMIO_DFH(TRVATTL3PTRDW(2), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2723 | MMIO_DFH(TRVATTL3PTRDW(3), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2724 | MMIO_DFH(TRVADR, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2725 | MMIO_DFH(TRTTE, D_SKL_PLUS, F_CMD_ACCESS | F_PM_SAVE, |
2726 | NULL, gen9_trtte_write); |
2727 | MMIO_DFH(_MMIO(0x4dfc), D_SKL_PLUS, F_PM_SAVE, |
2728 | NULL, gen9_trtt_chicken_write); |
2729 | |
2730 | MMIO_DFH(GEN8_GARBCNTL, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL); |
2731 | MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write); |
2732 | |
2733 | #define CSFE_CHICKEN1_REG(base) _MMIO((base) + 0xD4) |
2734 | MMIO_RING_DFH(CSFE_CHICKEN1_REG, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, |
2735 | NULL, csfe_chicken1_mmio_write); |
2736 | #undef CSFE_CHICKEN1_REG |
2737 | MMIO_DFH(GEN8_HDC_CHICKEN1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, |
2738 | NULL, NULL); |
2739 | MMIO_DFH(GEN9_WM_CHICKEN3, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, |
2740 | NULL, NULL); |
2741 | |
2742 | MMIO_DFH(GAMT_CHKN_BIT_REG, D_KBL | D_CFL, F_CMD_ACCESS, NULL, NULL); |
2743 | MMIO_DFH(_MMIO(0xe4cc), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL); |
2744 | |
2745 | return 0; |
2746 | } |
2747 | |
2748 | static int init_bxt_mmio_info(struct intel_gvt *gvt) |
2749 | { |
2750 | int ret; |
2751 | |
2752 | MMIO_DH(BXT_P_CR_GT_DISP_PWRON, D_BXT, NULL, bxt_gt_disp_pwron_write); |
2753 | MMIO_DH(BXT_PHY_CTL_FAMILY(DPIO_PHY0), D_BXT, |
2754 | NULL, bxt_phy_ctl_family_write); |
2755 | MMIO_DH(BXT_PHY_CTL_FAMILY(DPIO_PHY1), D_BXT, |
2756 | NULL, bxt_phy_ctl_family_write); |
2757 | MMIO_DH(BXT_PORT_PLL_ENABLE(PORT_A), D_BXT, |
2758 | NULL, bxt_port_pll_enable_write); |
2759 | MMIO_DH(BXT_PORT_PLL_ENABLE(PORT_B), D_BXT, |
2760 | NULL, bxt_port_pll_enable_write); |
2761 | MMIO_DH(BXT_PORT_PLL_ENABLE(PORT_C), D_BXT, NULL, |
2762 | bxt_port_pll_enable_write); |
2763 | |
2764 | MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH0), D_BXT, |
2765 | NULL, bxt_pcs_dw12_grp_write); |
2766 | MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH0), D_BXT, |
2767 | bxt_port_tx_dw3_read, NULL); |
2768 | MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY0, DPIO_CH1), D_BXT, |
2769 | NULL, bxt_pcs_dw12_grp_write); |
2770 | MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY0, DPIO_CH1), D_BXT, |
2771 | bxt_port_tx_dw3_read, NULL); |
2772 | MMIO_DH(BXT_PORT_PCS_DW12_GRP(DPIO_PHY1, DPIO_CH0), D_BXT, |
2773 | NULL, bxt_pcs_dw12_grp_write); |
2774 | MMIO_DH(BXT_PORT_TX_DW3_LN0(DPIO_PHY1, DPIO_CH0), D_BXT, |
2775 | bxt_port_tx_dw3_read, NULL); |
2776 | MMIO_DH(BXT_DE_PLL_ENABLE, D_BXT, NULL, bxt_de_pll_enable_write); |
2777 | MMIO_DFH(GEN8_L3SQCREG1, D_BXT, F_CMD_ACCESS, NULL, NULL); |
2778 | MMIO_DFH(GEN8_L3CNTLREG, D_BXT, F_CMD_ACCESS, NULL, NULL); |
2779 | MMIO_DFH(_MMIO(0x20D8), D_BXT, F_CMD_ACCESS, NULL, NULL); |
2780 | MMIO_F(GEN8_RING_CS_GPR(RENDER_RING_BASE, 0), 0x40, F_CMD_ACCESS, |
2781 | 0, 0, D_BXT, NULL, NULL); |
2782 | MMIO_F(GEN8_RING_CS_GPR(GEN6_BSD_RING_BASE, 0), 0x40, F_CMD_ACCESS, |
2783 | 0, 0, D_BXT, NULL, NULL); |
2784 | MMIO_F(GEN8_RING_CS_GPR(BLT_RING_BASE, 0), 0x40, F_CMD_ACCESS, |
2785 | 0, 0, D_BXT, NULL, NULL); |
2786 | MMIO_F(GEN8_RING_CS_GPR(VEBOX_RING_BASE, 0), 0x40, F_CMD_ACCESS, |
2787 | 0, 0, D_BXT, NULL, NULL); |
2788 | |
2789 | MMIO_DFH(GEN9_CTX_PREEMPT_REG, D_BXT, F_CMD_ACCESS, NULL, NULL); |
2790 | |
2791 | MMIO_DH(GEN8_PRIVATE_PAT_LO, D_BXT, NULL, bxt_ppat_low_write); |
2792 | |
2793 | return 0; |
2794 | } |
2795 | |
2796 | static struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt, |
2797 | unsigned int offset) |
2798 | { |
2799 | struct gvt_mmio_block *block = gvt->mmio.mmio_block; |
2800 | int num = gvt->mmio.num_mmio_block; |
2801 | int i; |
2802 | |
2803 | for (i = 0; i < num; i++, block++) { |
2804 | if (offset >= i915_mmio_reg_offset(block->offset) && |
2805 | offset < i915_mmio_reg_offset(block->offset) + block->size) |
2806 | return block; |
2807 | } |
2808 | return NULL; |
2809 | } |
2810 | |
2811 | /** |
2812 | * intel_gvt_clean_mmio_info - clean up MMIO information table for GVT device |
2813 | * @gvt: GVT device |
2814 | * |
2815 | * This function is called at the driver unloading stage, to clean up the MMIO |
2816 | * information table of GVT device |
2817 | * |
2818 | */ |
2819 | void intel_gvt_clean_mmio_info(struct intel_gvt *gvt) |
2820 | { |
2821 | struct hlist_node *tmp; |
2822 | struct intel_gvt_mmio_info *e; |
2823 | int i; |
2824 | |
2825 | hash_for_each_safe(gvt->mmio.mmio_info_table, i, tmp, e, node) |
2826 | kfree(objp: e); |
2827 | |
2828 | kfree(objp: gvt->mmio.mmio_block); |
2829 | gvt->mmio.mmio_block = NULL; |
2830 | gvt->mmio.num_mmio_block = 0; |
2831 | |
2832 | vfree(addr: gvt->mmio.mmio_attribute); |
2833 | gvt->mmio.mmio_attribute = NULL; |
2834 | } |
2835 | |
2836 | static int handle_mmio(struct intel_gvt_mmio_table_iter *iter, u32 offset, |
2837 | u32 size) |
2838 | { |
2839 | struct intel_gvt *gvt = iter->data; |
2840 | struct intel_gvt_mmio_info *info, *p; |
2841 | u32 start, end, i; |
2842 | |
2843 | if (WARN_ON(!IS_ALIGNED(offset, 4))) |
2844 | return -EINVAL; |
2845 | |
2846 | start = offset; |
2847 | end = offset + size; |
2848 | |
2849 | for (i = start; i < end; i += 4) { |
2850 | p = intel_gvt_find_mmio_info(gvt, offset: i); |
2851 | if (p) { |
2852 | WARN(1, "dup mmio definition offset %x\n" , i); |
2853 | |
2854 | /* We return -EEXIST here to make GVT-g load fail. |
2855 | * So duplicated MMIO can be found as soon as |
2856 | * possible. |
2857 | */ |
2858 | return -EEXIST; |
2859 | } |
2860 | |
2861 | info = kzalloc(size: sizeof(*info), GFP_KERNEL); |
2862 | if (!info) |
2863 | return -ENOMEM; |
2864 | |
2865 | info->offset = i; |
2866 | info->read = intel_vgpu_default_mmio_read; |
2867 | info->write = intel_vgpu_default_mmio_write; |
2868 | INIT_HLIST_NODE(h: &info->node); |
2869 | hash_add(gvt->mmio.mmio_info_table, &info->node, info->offset); |
2870 | gvt->mmio.num_tracked_mmio++; |
2871 | } |
2872 | return 0; |
2873 | } |
2874 | |
2875 | static int handle_mmio_block(struct intel_gvt_mmio_table_iter *iter, |
2876 | u32 offset, u32 size) |
2877 | { |
2878 | struct intel_gvt *gvt = iter->data; |
2879 | struct gvt_mmio_block *block = gvt->mmio.mmio_block; |
2880 | void *ret; |
2881 | |
2882 | ret = krealloc(objp: block, |
2883 | new_size: (gvt->mmio.num_mmio_block + 1) * sizeof(*block), |
2884 | GFP_KERNEL); |
2885 | if (!ret) |
2886 | return -ENOMEM; |
2887 | |
2888 | gvt->mmio.mmio_block = block = ret; |
2889 | |
2890 | block += gvt->mmio.num_mmio_block; |
2891 | |
2892 | memset(block, 0, sizeof(*block)); |
2893 | |
2894 | block->offset = _MMIO(offset); |
2895 | block->size = size; |
2896 | |
2897 | gvt->mmio.num_mmio_block++; |
2898 | |
2899 | return 0; |
2900 | } |
2901 | |
2902 | static int handle_mmio_cb(struct intel_gvt_mmio_table_iter *iter, u32 offset, |
2903 | u32 size) |
2904 | { |
2905 | if (size < 1024 || offset == i915_mmio_reg_offset(GEN9_GFX_MOCS(0))) |
2906 | return handle_mmio(iter, offset, size); |
2907 | else |
2908 | return handle_mmio_block(iter, offset, size); |
2909 | } |
2910 | |
2911 | static int init_mmio_info(struct intel_gvt *gvt) |
2912 | { |
2913 | struct intel_gvt_mmio_table_iter iter = { |
2914 | .i915 = gvt->gt->i915, |
2915 | .data = gvt, |
2916 | .handle_mmio_cb = handle_mmio_cb, |
2917 | }; |
2918 | |
2919 | return intel_gvt_iterate_mmio_table(iter: &iter); |
2920 | } |
2921 | |
2922 | static int init_mmio_block_handlers(struct intel_gvt *gvt) |
2923 | { |
2924 | struct gvt_mmio_block *block; |
2925 | |
2926 | block = find_mmio_block(gvt, VGT_PVINFO_PAGE); |
2927 | if (!block) { |
2928 | WARN(1, "fail to assign handlers to mmio block %x\n" , |
2929 | i915_mmio_reg_offset(gvt->mmio.mmio_block->offset)); |
2930 | return -ENODEV; |
2931 | } |
2932 | |
2933 | block->read = pvinfo_mmio_read; |
2934 | block->write = pvinfo_mmio_write; |
2935 | |
2936 | return 0; |
2937 | } |
2938 | |
2939 | /** |
2940 | * intel_gvt_setup_mmio_info - setup MMIO information table for GVT device |
2941 | * @gvt: GVT device |
2942 | * |
2943 | * This function is called at the initialization stage, to setup the MMIO |
2944 | * information table for GVT device |
2945 | * |
2946 | * Returns: |
2947 | * zero on success, negative if failed. |
2948 | */ |
2949 | int intel_gvt_setup_mmio_info(struct intel_gvt *gvt) |
2950 | { |
2951 | struct intel_gvt_device_info *info = &gvt->device_info; |
2952 | struct drm_i915_private *i915 = gvt->gt->i915; |
2953 | int size = info->mmio_size / 4 * sizeof(*gvt->mmio.mmio_attribute); |
2954 | int ret; |
2955 | |
2956 | gvt->mmio.mmio_attribute = vzalloc(size); |
2957 | if (!gvt->mmio.mmio_attribute) |
2958 | return -ENOMEM; |
2959 | |
2960 | ret = init_mmio_info(gvt); |
2961 | if (ret) |
2962 | goto err; |
2963 | |
2964 | ret = init_mmio_block_handlers(gvt); |
2965 | if (ret) |
2966 | goto err; |
2967 | |
2968 | ret = init_generic_mmio_info(gvt); |
2969 | if (ret) |
2970 | goto err; |
2971 | |
2972 | if (IS_BROADWELL(i915)) { |
2973 | ret = init_bdw_mmio_info(gvt); |
2974 | if (ret) |
2975 | goto err; |
2976 | } else if (IS_SKYLAKE(i915) || |
2977 | IS_KABYLAKE(i915) || |
2978 | IS_COFFEELAKE(i915) || |
2979 | IS_COMETLAKE(i915)) { |
2980 | ret = init_bdw_mmio_info(gvt); |
2981 | if (ret) |
2982 | goto err; |
2983 | ret = init_skl_mmio_info(gvt); |
2984 | if (ret) |
2985 | goto err; |
2986 | } else if (IS_BROXTON(i915)) { |
2987 | ret = init_bdw_mmio_info(gvt); |
2988 | if (ret) |
2989 | goto err; |
2990 | ret = init_skl_mmio_info(gvt); |
2991 | if (ret) |
2992 | goto err; |
2993 | ret = init_bxt_mmio_info(gvt); |
2994 | if (ret) |
2995 | goto err; |
2996 | } |
2997 | |
2998 | return 0; |
2999 | err: |
3000 | intel_gvt_clean_mmio_info(gvt); |
3001 | return ret; |
3002 | } |
3003 | |
3004 | /** |
3005 | * intel_gvt_for_each_tracked_mmio - iterate each tracked mmio |
3006 | * @gvt: a GVT device |
3007 | * @handler: the handler |
3008 | * @data: private data given to handler |
3009 | * |
3010 | * Returns: |
3011 | * Zero on success, negative error code if failed. |
3012 | */ |
3013 | int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt, |
3014 | int (*handler)(struct intel_gvt *gvt, u32 offset, void *data), |
3015 | void *data) |
3016 | { |
3017 | struct gvt_mmio_block *block = gvt->mmio.mmio_block; |
3018 | struct intel_gvt_mmio_info *e; |
3019 | int i, j, ret; |
3020 | |
3021 | hash_for_each(gvt->mmio.mmio_info_table, i, e, node) { |
3022 | ret = handler(gvt, e->offset, data); |
3023 | if (ret) |
3024 | return ret; |
3025 | } |
3026 | |
3027 | for (i = 0; i < gvt->mmio.num_mmio_block; i++, block++) { |
3028 | /* pvinfo data doesn't come from hw mmio */ |
3029 | if (i915_mmio_reg_offset(block->offset) == VGT_PVINFO_PAGE) |
3030 | continue; |
3031 | |
3032 | for (j = 0; j < block->size; j += 4) { |
3033 | ret = handler(gvt, i915_mmio_reg_offset(block->offset) + j, data); |
3034 | if (ret) |
3035 | return ret; |
3036 | } |
3037 | } |
3038 | return 0; |
3039 | } |
3040 | |
3041 | /** |
3042 | * intel_vgpu_default_mmio_read - default MMIO read handler |
3043 | * @vgpu: a vGPU |
3044 | * @offset: access offset |
3045 | * @p_data: data return buffer |
3046 | * @bytes: access data length |
3047 | * |
3048 | * Returns: |
3049 | * Zero on success, negative error code if failed. |
3050 | */ |
3051 | int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, |
3052 | void *p_data, unsigned int bytes) |
3053 | { |
3054 | read_vreg(vgpu, offset, p_data, bytes); |
3055 | return 0; |
3056 | } |
3057 | |
3058 | /** |
3059 | * intel_vgpu_default_mmio_write() - default MMIO write handler |
3060 | * @vgpu: a vGPU |
3061 | * @offset: access offset |
3062 | * @p_data: write data buffer |
3063 | * @bytes: access data length |
3064 | * |
3065 | * Returns: |
3066 | * Zero on success, negative error code if failed. |
3067 | */ |
3068 | int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
3069 | void *p_data, unsigned int bytes) |
3070 | { |
3071 | write_vreg(vgpu, offset, p_data, bytes); |
3072 | return 0; |
3073 | } |
3074 | |
3075 | /** |
3076 | * intel_vgpu_mask_mmio_write - write mask register |
3077 | * @vgpu: a vGPU |
3078 | * @offset: access offset |
3079 | * @p_data: write data buffer |
3080 | * @bytes: access data length |
3081 | * |
3082 | * Returns: |
3083 | * Zero on success, negative error code if failed. |
3084 | */ |
3085 | int intel_vgpu_mask_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, |
3086 | void *p_data, unsigned int bytes) |
3087 | { |
3088 | u32 mask, old_vreg; |
3089 | |
3090 | old_vreg = vgpu_vreg(vgpu, offset); |
3091 | write_vreg(vgpu, offset, p_data, bytes); |
3092 | mask = vgpu_vreg(vgpu, offset) >> 16; |
3093 | vgpu_vreg(vgpu, offset) = (old_vreg & ~mask) | |
3094 | (vgpu_vreg(vgpu, offset) & mask); |
3095 | |
3096 | return 0; |
3097 | } |
3098 | |
3099 | /** |
3100 | * intel_gvt_in_force_nonpriv_whitelist - if a mmio is in whitelist to be |
3101 | * force-nopriv register |
3102 | * |
3103 | * @gvt: a GVT device |
3104 | * @offset: register offset |
3105 | * |
3106 | * Returns: |
3107 | * True if the register is in force-nonpriv whitelist; |
3108 | * False if outside; |
3109 | */ |
3110 | bool intel_gvt_in_force_nonpriv_whitelist(struct intel_gvt *gvt, |
3111 | unsigned int offset) |
3112 | { |
3113 | return in_whitelist(reg: offset); |
3114 | } |
3115 | |
3116 | /** |
3117 | * intel_vgpu_mmio_reg_rw - emulate tracked mmio registers |
3118 | * @vgpu: a vGPU |
3119 | * @offset: register offset |
3120 | * @pdata: data buffer |
3121 | * @bytes: data length |
3122 | * @is_read: read or write |
3123 | * |
3124 | * Returns: |
3125 | * Zero on success, negative error code if failed. |
3126 | */ |
3127 | int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset, |
3128 | void *pdata, unsigned int bytes, bool is_read) |
3129 | { |
3130 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
3131 | struct intel_gvt *gvt = vgpu->gvt; |
3132 | struct intel_gvt_mmio_info *mmio_info; |
3133 | struct gvt_mmio_block *mmio_block; |
3134 | gvt_mmio_func func; |
3135 | int ret; |
3136 | |
3137 | if (drm_WARN_ON(&i915->drm, bytes > 8)) |
3138 | return -EINVAL; |
3139 | |
3140 | /* |
3141 | * Handle special MMIO blocks. |
3142 | */ |
3143 | mmio_block = find_mmio_block(gvt, offset); |
3144 | if (mmio_block) { |
3145 | func = is_read ? mmio_block->read : mmio_block->write; |
3146 | if (func) |
3147 | return func(vgpu, offset, pdata, bytes); |
3148 | goto default_rw; |
3149 | } |
3150 | |
3151 | /* |
3152 | * Normal tracked MMIOs. |
3153 | */ |
3154 | mmio_info = intel_gvt_find_mmio_info(gvt, offset); |
3155 | if (!mmio_info) { |
3156 | gvt_dbg_mmio("untracked MMIO %08x len %d\n" , offset, bytes); |
3157 | goto default_rw; |
3158 | } |
3159 | |
3160 | if (is_read) |
3161 | return mmio_info->read(vgpu, offset, pdata, bytes); |
3162 | else { |
3163 | u64 ro_mask = mmio_info->ro_mask; |
3164 | u32 old_vreg = 0; |
3165 | u64 data = 0; |
3166 | |
3167 | if (intel_gvt_mmio_has_mode_mask(gvt, offset: mmio_info->offset)) { |
3168 | old_vreg = vgpu_vreg(vgpu, offset); |
3169 | } |
3170 | |
3171 | if (likely(!ro_mask)) |
3172 | ret = mmio_info->write(vgpu, offset, pdata, bytes); |
3173 | else if (!~ro_mask) { |
3174 | gvt_vgpu_err("try to write RO reg %x\n" , offset); |
3175 | return 0; |
3176 | } else { |
3177 | /* keep the RO bits in the virtual register */ |
3178 | memcpy(&data, pdata, bytes); |
3179 | data &= ~ro_mask; |
3180 | data |= vgpu_vreg(vgpu, offset) & ro_mask; |
3181 | ret = mmio_info->write(vgpu, offset, &data, bytes); |
3182 | } |
3183 | |
3184 | /* higher 16bits of mode ctl regs are mask bits for change */ |
3185 | if (intel_gvt_mmio_has_mode_mask(gvt, offset: mmio_info->offset)) { |
3186 | u32 mask = vgpu_vreg(vgpu, offset) >> 16; |
3187 | |
3188 | vgpu_vreg(vgpu, offset) = (old_vreg & ~mask) |
3189 | | (vgpu_vreg(vgpu, offset) & mask); |
3190 | } |
3191 | } |
3192 | |
3193 | return ret; |
3194 | |
3195 | default_rw: |
3196 | return is_read ? |
3197 | intel_vgpu_default_mmio_read(vgpu, offset, p_data: pdata, bytes) : |
3198 | intel_vgpu_default_mmio_write(vgpu, offset, p_data: pdata, bytes); |
3199 | } |
3200 | |
3201 | void intel_gvt_restore_fence(struct intel_gvt *gvt) |
3202 | { |
3203 | struct intel_vgpu *vgpu; |
3204 | int i, id; |
3205 | |
3206 | idr_for_each_entry(&(gvt)->vgpu_idr, vgpu, id) { |
3207 | mmio_hw_access_pre(gt: gvt->gt); |
3208 | for (i = 0; i < vgpu_fence_sz(vgpu); i++) |
3209 | intel_vgpu_write_fence(vgpu, fence: i, vgpu_vreg64(vgpu, fence_num_to_offset(i))); |
3210 | mmio_hw_access_post(gt: gvt->gt); |
3211 | } |
3212 | } |
3213 | |
3214 | static int mmio_pm_restore_handler(struct intel_gvt *gvt, u32 offset, void *data) |
3215 | { |
3216 | struct intel_vgpu *vgpu = data; |
3217 | struct drm_i915_private *dev_priv = gvt->gt->i915; |
3218 | |
3219 | if (gvt->mmio.mmio_attribute[offset >> 2] & F_PM_SAVE) |
3220 | intel_uncore_write(uncore: &dev_priv->uncore, _MMIO(offset), vgpu_vreg(vgpu, offset)); |
3221 | |
3222 | return 0; |
3223 | } |
3224 | |
3225 | void intel_gvt_restore_mmio(struct intel_gvt *gvt) |
3226 | { |
3227 | struct intel_vgpu *vgpu; |
3228 | int id; |
3229 | |
3230 | idr_for_each_entry(&(gvt)->vgpu_idr, vgpu, id) { |
3231 | mmio_hw_access_pre(gt: gvt->gt); |
3232 | intel_gvt_for_each_tracked_mmio(gvt, handler: mmio_pm_restore_handler, data: vgpu); |
3233 | mmio_hw_access_post(gt: gvt->gt); |
3234 | } |
3235 | } |
3236 | |