1 | /* |
2 | * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
21 | * SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Ke Yu |
25 | * Kevin Tian <kevin.tian@intel.com> |
26 | * Zhiyuan Lv <zhiyuan.lv@intel.com> |
27 | * |
28 | * Contributors: |
29 | * Min He <min.he@intel.com> |
30 | * Ping Gao <ping.a.gao@intel.com> |
31 | * Tina Zhang <tina.zhang@intel.com> |
32 | * Yulei Zhang <yulei.zhang@intel.com> |
33 | * Zhi Wang <zhi.a.wang@intel.com> |
34 | * |
35 | */ |
36 | |
37 | #include <linux/slab.h> |
38 | |
39 | #include "i915_drv.h" |
40 | #include "i915_reg.h" |
41 | #include "gt/intel_engine_regs.h" |
42 | #include "gt/intel_gpu_commands.h" |
43 | #include "gt/intel_gt_regs.h" |
44 | #include "gt/intel_lrc.h" |
45 | #include "gt/intel_ring.h" |
46 | #include "gt/intel_gt_requests.h" |
47 | #include "gt/shmem_utils.h" |
48 | #include "gvt.h" |
49 | #include "i915_pvinfo.h" |
50 | #include "trace.h" |
51 | |
52 | #include "display/intel_display.h" |
53 | #include "gem/i915_gem_context.h" |
54 | #include "gem/i915_gem_pm.h" |
55 | #include "gt/intel_context.h" |
56 | |
57 | #define INVALID_OP (~0U) |
58 | |
59 | #define OP_LEN_MI 9 |
60 | #define OP_LEN_2D 10 |
61 | #define OP_LEN_3D_MEDIA 16 |
62 | #define OP_LEN_MFX_VC 16 |
63 | #define OP_LEN_VEBOX 16 |
64 | |
65 | #define CMD_TYPE(cmd) (((cmd) >> 29) & 7) |
66 | |
67 | struct sub_op_bits { |
68 | int hi; |
69 | int low; |
70 | }; |
71 | struct decode_info { |
72 | const char *name; |
73 | int op_len; |
74 | int nr_sub_op; |
75 | const struct sub_op_bits *sub_op; |
76 | }; |
77 | |
78 | #define MAX_CMD_BUDGET 0x7fffffff |
79 | #define MI_WAIT_FOR_PLANE_C_FLIP_PENDING (1<<15) |
80 | #define MI_WAIT_FOR_PLANE_B_FLIP_PENDING (1<<9) |
81 | #define MI_WAIT_FOR_PLANE_A_FLIP_PENDING (1<<1) |
82 | |
83 | #define MI_WAIT_FOR_SPRITE_C_FLIP_PENDING (1<<20) |
84 | #define MI_WAIT_FOR_SPRITE_B_FLIP_PENDING (1<<10) |
85 | #define MI_WAIT_FOR_SPRITE_A_FLIP_PENDING (1<<2) |
86 | |
87 | /* Render Command Map */ |
88 | |
89 | /* MI_* command Opcode (28:23) */ |
90 | #define OP_MI_NOOP 0x0 |
91 | #define OP_MI_SET_PREDICATE 0x1 /* HSW+ */ |
92 | #define OP_MI_USER_INTERRUPT 0x2 |
93 | #define OP_MI_WAIT_FOR_EVENT 0x3 |
94 | #define OP_MI_FLUSH 0x4 |
95 | #define OP_MI_ARB_CHECK 0x5 |
96 | #define OP_MI_RS_CONTROL 0x6 /* HSW+ */ |
97 | #define OP_MI_REPORT_HEAD 0x7 |
98 | #define OP_MI_ARB_ON_OFF 0x8 |
99 | #define OP_MI_URB_ATOMIC_ALLOC 0x9 /* HSW+ */ |
100 | #define OP_MI_BATCH_BUFFER_END 0xA |
101 | #define OP_MI_SUSPEND_FLUSH 0xB |
102 | #define OP_MI_PREDICATE 0xC /* IVB+ */ |
103 | #define OP_MI_TOPOLOGY_FILTER 0xD /* IVB+ */ |
104 | #define OP_MI_SET_APPID 0xE /* IVB+ */ |
105 | #define OP_MI_RS_CONTEXT 0xF /* HSW+ */ |
106 | #define OP_MI_LOAD_SCAN_LINES_INCL 0x12 /* HSW+ */ |
107 | #define OP_MI_DISPLAY_FLIP 0x14 |
108 | #define OP_MI_SEMAPHORE_MBOX 0x16 |
109 | #define OP_MI_SET_CONTEXT 0x18 |
110 | #define OP_MI_MATH 0x1A |
111 | #define OP_MI_URB_CLEAR 0x19 |
112 | #define OP_MI_SEMAPHORE_SIGNAL 0x1B /* BDW+ */ |
113 | #define OP_MI_SEMAPHORE_WAIT 0x1C /* BDW+ */ |
114 | |
115 | #define OP_MI_STORE_DATA_IMM 0x20 |
116 | #define OP_MI_STORE_DATA_INDEX 0x21 |
117 | #define OP_MI_LOAD_REGISTER_IMM 0x22 |
118 | #define OP_MI_UPDATE_GTT 0x23 |
119 | #define OP_MI_STORE_REGISTER_MEM 0x24 |
120 | #define OP_MI_FLUSH_DW 0x26 |
121 | #define OP_MI_CLFLUSH 0x27 |
122 | #define OP_MI_REPORT_PERF_COUNT 0x28 |
123 | #define OP_MI_LOAD_REGISTER_MEM 0x29 /* HSW+ */ |
124 | #define OP_MI_LOAD_REGISTER_REG 0x2A /* HSW+ */ |
125 | #define OP_MI_RS_STORE_DATA_IMM 0x2B /* HSW+ */ |
126 | #define OP_MI_LOAD_URB_MEM 0x2C /* HSW+ */ |
127 | #define OP_MI_STORE_URM_MEM 0x2D /* HSW+ */ |
128 | #define OP_MI_2E 0x2E /* BDW+ */ |
129 | #define OP_MI_2F 0x2F /* BDW+ */ |
130 | #define OP_MI_BATCH_BUFFER_START 0x31 |
131 | |
132 | /* Bit definition for dword 0 */ |
133 | #define _CMDBIT_BB_START_IN_PPGTT (1UL << 8) |
134 | |
135 | #define OP_MI_CONDITIONAL_BATCH_BUFFER_END 0x36 |
136 | |
137 | #define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2)) |
138 | #define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U)) |
139 | #define BATCH_BUFFER_ADR_SPACE_BIT(x) (((x) >> 8) & 1U) |
140 | #define BATCH_BUFFER_2ND_LEVEL_BIT(x) ((x) >> 22 & 1U) |
141 | |
142 | /* 2D command: Opcode (28:22) */ |
143 | #define OP_2D(x) ((2<<7) | x) |
144 | |
145 | #define OP_XY_SETUP_BLT OP_2D(0x1) |
146 | #define OP_XY_SETUP_CLIP_BLT OP_2D(0x3) |
147 | #define OP_XY_SETUP_MONO_PATTERN_SL_BLT OP_2D(0x11) |
148 | #define OP_XY_PIXEL_BLT OP_2D(0x24) |
149 | #define OP_XY_SCANLINES_BLT OP_2D(0x25) |
150 | #define OP_XY_TEXT_BLT OP_2D(0x26) |
151 | #define OP_XY_TEXT_IMMEDIATE_BLT OP_2D(0x31) |
152 | #define OP_XY_COLOR_BLT OP_2D(0x50) |
153 | #define OP_XY_PAT_BLT OP_2D(0x51) |
154 | #define OP_XY_MONO_PAT_BLT OP_2D(0x52) |
155 | #define OP_XY_SRC_COPY_BLT OP_2D(0x53) |
156 | #define OP_XY_MONO_SRC_COPY_BLT OP_2D(0x54) |
157 | #define OP_XY_FULL_BLT OP_2D(0x55) |
158 | #define OP_XY_FULL_MONO_SRC_BLT OP_2D(0x56) |
159 | #define OP_XY_FULL_MONO_PATTERN_BLT OP_2D(0x57) |
160 | #define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT OP_2D(0x58) |
161 | #define OP_XY_MONO_PAT_FIXED_BLT OP_2D(0x59) |
162 | #define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT OP_2D(0x71) |
163 | #define OP_XY_PAT_BLT_IMMEDIATE OP_2D(0x72) |
164 | #define OP_XY_SRC_COPY_CHROMA_BLT OP_2D(0x73) |
165 | #define OP_XY_FULL_IMMEDIATE_PATTERN_BLT OP_2D(0x74) |
166 | #define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT OP_2D(0x75) |
167 | #define OP_XY_PAT_CHROMA_BLT OP_2D(0x76) |
168 | #define OP_XY_PAT_CHROMA_BLT_IMMEDIATE OP_2D(0x77) |
169 | |
170 | /* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */ |
171 | #define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \ |
172 | ((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode)) |
173 | |
174 | #define OP_STATE_PREFETCH OP_3D_MEDIA(0x0, 0x0, 0x03) |
175 | |
176 | #define OP_STATE_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x01) |
177 | #define OP_STATE_SIP OP_3D_MEDIA(0x0, 0x1, 0x02) |
178 | #define OP_3D_MEDIA_0_1_4 OP_3D_MEDIA(0x0, 0x1, 0x04) |
179 | #define OP_SWTESS_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x03) |
180 | |
181 | #define OP_3DSTATE_VF_STATISTICS_GM45 OP_3D_MEDIA(0x1, 0x0, 0x0B) |
182 | |
183 | #define OP_PIPELINE_SELECT OP_3D_MEDIA(0x1, 0x1, 0x04) |
184 | |
185 | #define OP_MEDIA_VFE_STATE OP_3D_MEDIA(0x2, 0x0, 0x0) |
186 | #define OP_MEDIA_CURBE_LOAD OP_3D_MEDIA(0x2, 0x0, 0x1) |
187 | #define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD OP_3D_MEDIA(0x2, 0x0, 0x2) |
188 | #define OP_MEDIA_GATEWAY_STATE OP_3D_MEDIA(0x2, 0x0, 0x3) |
189 | #define OP_MEDIA_STATE_FLUSH OP_3D_MEDIA(0x2, 0x0, 0x4) |
190 | #define OP_MEDIA_POOL_STATE OP_3D_MEDIA(0x2, 0x0, 0x5) |
191 | |
192 | #define OP_MEDIA_OBJECT OP_3D_MEDIA(0x2, 0x1, 0x0) |
193 | #define OP_MEDIA_OBJECT_PRT OP_3D_MEDIA(0x2, 0x1, 0x2) |
194 | #define OP_MEDIA_OBJECT_WALKER OP_3D_MEDIA(0x2, 0x1, 0x3) |
195 | #define OP_GPGPU_WALKER OP_3D_MEDIA(0x2, 0x1, 0x5) |
196 | |
197 | #define OP_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */ |
198 | #define OP_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */ |
199 | #define OP_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */ |
200 | #define OP_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */ |
201 | #define OP_3DSTATE_VERTEX_BUFFERS OP_3D_MEDIA(0x3, 0x0, 0x08) |
202 | #define OP_3DSTATE_VERTEX_ELEMENTS OP_3D_MEDIA(0x3, 0x0, 0x09) |
203 | #define OP_3DSTATE_INDEX_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x0A) |
204 | #define OP_3DSTATE_VF_STATISTICS OP_3D_MEDIA(0x3, 0x0, 0x0B) |
205 | #define OP_3DSTATE_VF OP_3D_MEDIA(0x3, 0x0, 0x0C) /* HSW+ */ |
206 | #define OP_3DSTATE_CC_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0E) |
207 | #define OP_3DSTATE_SCISSOR_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0F) |
208 | #define OP_3DSTATE_VS OP_3D_MEDIA(0x3, 0x0, 0x10) |
209 | #define OP_3DSTATE_GS OP_3D_MEDIA(0x3, 0x0, 0x11) |
210 | #define OP_3DSTATE_CLIP OP_3D_MEDIA(0x3, 0x0, 0x12) |
211 | #define OP_3DSTATE_SF OP_3D_MEDIA(0x3, 0x0, 0x13) |
212 | #define OP_3DSTATE_WM OP_3D_MEDIA(0x3, 0x0, 0x14) |
213 | #define OP_3DSTATE_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x15) |
214 | #define OP_3DSTATE_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x16) |
215 | #define OP_3DSTATE_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x17) |
216 | #define OP_3DSTATE_SAMPLE_MASK OP_3D_MEDIA(0x3, 0x0, 0x18) |
217 | #define OP_3DSTATE_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */ |
218 | #define OP_3DSTATE_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */ |
219 | #define OP_3DSTATE_HS OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */ |
220 | #define OP_3DSTATE_TE OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */ |
221 | #define OP_3DSTATE_DS OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */ |
222 | #define OP_3DSTATE_STREAMOUT OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */ |
223 | #define OP_3DSTATE_SBE OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */ |
224 | #define OP_3DSTATE_PS OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */ |
225 | #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */ |
226 | #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */ |
227 | #define OP_3DSTATE_BLEND_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */ |
228 | #define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */ |
229 | #define OP_3DSTATE_BINDING_TABLE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */ |
230 | #define OP_3DSTATE_BINDING_TABLE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */ |
231 | #define OP_3DSTATE_BINDING_TABLE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */ |
232 | #define OP_3DSTATE_BINDING_TABLE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */ |
233 | #define OP_3DSTATE_BINDING_TABLE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */ |
234 | #define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */ |
235 | #define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */ |
236 | #define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */ |
237 | #define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */ |
238 | #define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */ |
239 | #define OP_3DSTATE_URB_VS OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */ |
240 | #define OP_3DSTATE_URB_HS OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */ |
241 | #define OP_3DSTATE_URB_DS OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */ |
242 | #define OP_3DSTATE_URB_GS OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */ |
243 | #define OP_3DSTATE_GATHER_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */ |
244 | #define OP_3DSTATE_GATHER_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */ |
245 | #define OP_3DSTATE_GATHER_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */ |
246 | #define OP_3DSTATE_GATHER_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */ |
247 | #define OP_3DSTATE_GATHER_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */ |
248 | #define OP_3DSTATE_DX9_CONSTANTF_VS OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */ |
249 | #define OP_3DSTATE_DX9_CONSTANTF_PS OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */ |
250 | #define OP_3DSTATE_DX9_CONSTANTI_VS OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */ |
251 | #define OP_3DSTATE_DX9_CONSTANTI_PS OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */ |
252 | #define OP_3DSTATE_DX9_CONSTANTB_VS OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */ |
253 | #define OP_3DSTATE_DX9_CONSTANTB_PS OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */ |
254 | #define OP_3DSTATE_DX9_LOCAL_VALID_VS OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */ |
255 | #define OP_3DSTATE_DX9_LOCAL_VALID_PS OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */ |
256 | #define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */ |
257 | #define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */ |
258 | #define OP_3DSTATE_BINDING_TABLE_EDIT_VS OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */ |
259 | #define OP_3DSTATE_BINDING_TABLE_EDIT_GS OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */ |
260 | #define OP_3DSTATE_BINDING_TABLE_EDIT_HS OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */ |
261 | #define OP_3DSTATE_BINDING_TABLE_EDIT_DS OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */ |
262 | #define OP_3DSTATE_BINDING_TABLE_EDIT_PS OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */ |
263 | |
264 | #define OP_3DSTATE_VF_INSTANCING OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */ |
265 | #define OP_3DSTATE_VF_SGVS OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */ |
266 | #define OP_3DSTATE_VF_TOPOLOGY OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */ |
267 | #define OP_3DSTATE_WM_CHROMAKEY OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */ |
268 | #define OP_3DSTATE_PS_BLEND OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */ |
269 | #define OP_3DSTATE_WM_DEPTH_STENCIL OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */ |
270 | #define OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */ |
271 | #define OP_3DSTATE_RASTER OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */ |
272 | #define OP_3DSTATE_SBE_SWIZ OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */ |
273 | #define OP_3DSTATE_WM_HZ_OP OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */ |
274 | #define OP_3DSTATE_COMPONENT_PACKING OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */ |
275 | |
276 | #define OP_3DSTATE_DRAWING_RECTANGLE OP_3D_MEDIA(0x3, 0x1, 0x00) |
277 | #define OP_3DSTATE_SAMPLER_PALETTE_LOAD0 OP_3D_MEDIA(0x3, 0x1, 0x02) |
278 | #define OP_3DSTATE_CHROMA_KEY OP_3D_MEDIA(0x3, 0x1, 0x04) |
279 | #define OP_SNB_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x05) |
280 | #define OP_3DSTATE_POLY_STIPPLE_OFFSET OP_3D_MEDIA(0x3, 0x1, 0x06) |
281 | #define OP_3DSTATE_POLY_STIPPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x07) |
282 | #define OP_3DSTATE_LINE_STIPPLE OP_3D_MEDIA(0x3, 0x1, 0x08) |
283 | #define OP_3DSTATE_AA_LINE_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x0A) |
284 | #define OP_3DSTATE_GS_SVB_INDEX OP_3D_MEDIA(0x3, 0x1, 0x0B) |
285 | #define OP_3DSTATE_SAMPLER_PALETTE_LOAD1 OP_3D_MEDIA(0x3, 0x1, 0x0C) |
286 | #define OP_3DSTATE_MULTISAMPLE_BDW OP_3D_MEDIA(0x3, 0x0, 0x0D) |
287 | #define OP_SNB_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0E) |
288 | #define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0F) |
289 | #define OP_SNB_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x10) |
290 | #define OP_3DSTATE_MONOFILTER_SIZE OP_3D_MEDIA(0x3, 0x1, 0x11) |
291 | #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */ |
292 | #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */ |
293 | #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */ |
294 | #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */ |
295 | #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */ |
296 | #define OP_3DSTATE_SO_DECL_LIST OP_3D_MEDIA(0x3, 0x1, 0x17) |
297 | #define OP_3DSTATE_SO_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x18) |
298 | #define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */ |
299 | #define OP_3DSTATE_GATHER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */ |
300 | #define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */ |
301 | #define OP_3DSTATE_SAMPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x1C) |
302 | #define OP_PIPE_CONTROL OP_3D_MEDIA(0x3, 0x2, 0x00) |
303 | #define OP_3DPRIMITIVE OP_3D_MEDIA(0x3, 0x3, 0x00) |
304 | |
305 | /* VCCP Command Parser */ |
306 | |
307 | /* |
308 | * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License) |
309 | * git://anongit.freedesktop.org/vaapi/intel-driver |
310 | * src/i965_defines.h |
311 | * |
312 | */ |
313 | |
314 | #define OP_MFX(pipeline, op, sub_opa, sub_opb) \ |
315 | (3 << 13 | \ |
316 | (pipeline) << 11 | \ |
317 | (op) << 8 | \ |
318 | (sub_opa) << 5 | \ |
319 | (sub_opb)) |
320 | |
321 | #define OP_MFX_PIPE_MODE_SELECT OP_MFX(2, 0, 0, 0) /* ALL */ |
322 | #define OP_MFX_SURFACE_STATE OP_MFX(2, 0, 0, 1) /* ALL */ |
323 | #define OP_MFX_PIPE_BUF_ADDR_STATE OP_MFX(2, 0, 0, 2) /* ALL */ |
324 | #define OP_MFX_IND_OBJ_BASE_ADDR_STATE OP_MFX(2, 0, 0, 3) /* ALL */ |
325 | #define OP_MFX_BSP_BUF_BASE_ADDR_STATE OP_MFX(2, 0, 0, 4) /* ALL */ |
326 | #define OP_2_0_0_5 OP_MFX(2, 0, 0, 5) /* ALL */ |
327 | #define OP_MFX_STATE_POINTER OP_MFX(2, 0, 0, 6) /* ALL */ |
328 | #define OP_MFX_QM_STATE OP_MFX(2, 0, 0, 7) /* IVB+ */ |
329 | #define OP_MFX_FQM_STATE OP_MFX(2, 0, 0, 8) /* IVB+ */ |
330 | #define OP_MFX_PAK_INSERT_OBJECT OP_MFX(2, 0, 2, 8) /* IVB+ */ |
331 | #define OP_MFX_STITCH_OBJECT OP_MFX(2, 0, 2, 0xA) /* IVB+ */ |
332 | |
333 | #define OP_MFD_IT_OBJECT OP_MFX(2, 0, 1, 9) /* ALL */ |
334 | |
335 | #define OP_MFX_WAIT OP_MFX(1, 0, 0, 0) /* IVB+ */ |
336 | #define OP_MFX_AVC_IMG_STATE OP_MFX(2, 1, 0, 0) /* ALL */ |
337 | #define OP_MFX_AVC_QM_STATE OP_MFX(2, 1, 0, 1) /* ALL */ |
338 | #define OP_MFX_AVC_DIRECTMODE_STATE OP_MFX(2, 1, 0, 2) /* ALL */ |
339 | #define OP_MFX_AVC_SLICE_STATE OP_MFX(2, 1, 0, 3) /* ALL */ |
340 | #define OP_MFX_AVC_REF_IDX_STATE OP_MFX(2, 1, 0, 4) /* ALL */ |
341 | #define OP_MFX_AVC_WEIGHTOFFSET_STATE OP_MFX(2, 1, 0, 5) /* ALL */ |
342 | #define OP_MFD_AVC_PICID_STATE OP_MFX(2, 1, 1, 5) /* HSW+ */ |
343 | #define OP_MFD_AVC_DPB_STATE OP_MFX(2, 1, 1, 6) /* IVB+ */ |
344 | #define OP_MFD_AVC_SLICEADDR OP_MFX(2, 1, 1, 7) /* IVB+ */ |
345 | #define OP_MFD_AVC_BSD_OBJECT OP_MFX(2, 1, 1, 8) /* ALL */ |
346 | #define OP_MFC_AVC_PAK_OBJECT OP_MFX(2, 1, 2, 9) /* ALL */ |
347 | |
348 | #define OP_MFX_VC1_PRED_PIPE_STATE OP_MFX(2, 2, 0, 1) /* ALL */ |
349 | #define OP_MFX_VC1_DIRECTMODE_STATE OP_MFX(2, 2, 0, 2) /* ALL */ |
350 | #define OP_MFD_VC1_SHORT_PIC_STATE OP_MFX(2, 2, 1, 0) /* IVB+ */ |
351 | #define OP_MFD_VC1_LONG_PIC_STATE OP_MFX(2, 2, 1, 1) /* IVB+ */ |
352 | #define OP_MFD_VC1_BSD_OBJECT OP_MFX(2, 2, 1, 8) /* ALL */ |
353 | |
354 | #define OP_MFX_MPEG2_PIC_STATE OP_MFX(2, 3, 0, 0) /* ALL */ |
355 | #define OP_MFX_MPEG2_QM_STATE OP_MFX(2, 3, 0, 1) /* ALL */ |
356 | #define OP_MFD_MPEG2_BSD_OBJECT OP_MFX(2, 3, 1, 8) /* ALL */ |
357 | #define OP_MFC_MPEG2_SLICEGROUP_STATE OP_MFX(2, 3, 2, 3) /* ALL */ |
358 | #define OP_MFC_MPEG2_PAK_OBJECT OP_MFX(2, 3, 2, 9) /* ALL */ |
359 | |
360 | #define OP_MFX_2_6_0_0 OP_MFX(2, 6, 0, 0) /* IVB+ */ |
361 | #define OP_MFX_2_6_0_8 OP_MFX(2, 6, 0, 8) /* IVB+ */ |
362 | #define OP_MFX_2_6_0_9 OP_MFX(2, 6, 0, 9) /* IVB+ */ |
363 | |
364 | #define OP_MFX_JPEG_PIC_STATE OP_MFX(2, 7, 0, 0) |
365 | #define OP_MFX_JPEG_HUFF_TABLE_STATE OP_MFX(2, 7, 0, 2) |
366 | #define OP_MFD_JPEG_BSD_OBJECT OP_MFX(2, 7, 1, 8) |
367 | |
368 | #define OP_VEB(pipeline, op, sub_opa, sub_opb) \ |
369 | (3 << 13 | \ |
370 | (pipeline) << 11 | \ |
371 | (op) << 8 | \ |
372 | (sub_opa) << 5 | \ |
373 | (sub_opb)) |
374 | |
375 | #define OP_VEB_SURFACE_STATE OP_VEB(2, 4, 0, 0) |
376 | #define OP_VEB_STATE OP_VEB(2, 4, 0, 2) |
377 | #define OP_VEB_DNDI_IECP_STATE OP_VEB(2, 4, 0, 3) |
378 | |
379 | struct parser_exec_state; |
380 | |
381 | typedef int (*parser_cmd_handler)(struct parser_exec_state *s); |
382 | |
383 | #define GVT_CMD_HASH_BITS 7 |
384 | |
385 | /* which DWords need address fix */ |
386 | #define ADDR_FIX_1(x1) (1 << (x1)) |
387 | #define ADDR_FIX_2(x1, x2) (ADDR_FIX_1(x1) | ADDR_FIX_1(x2)) |
388 | #define ADDR_FIX_3(x1, x2, x3) (ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3)) |
389 | #define ADDR_FIX_4(x1, x2, x3, x4) (ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4)) |
390 | #define ADDR_FIX_5(x1, x2, x3, x4, x5) (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5)) |
391 | |
392 | #define DWORD_FIELD(dword, end, start) \ |
393 | FIELD_GET(GENMASK(end, start), cmd_val(s, dword)) |
394 | |
395 | #define OP_LENGTH_BIAS 2 |
396 | #define CMD_LEN(value) (value + OP_LENGTH_BIAS) |
397 | |
398 | static int gvt_check_valid_cmd_length(int len, int valid_len) |
399 | { |
400 | if (valid_len != len) { |
401 | gvt_err("len is not valid: len=%u valid_len=%u\n" , |
402 | len, valid_len); |
403 | return -EFAULT; |
404 | } |
405 | return 0; |
406 | } |
407 | |
408 | struct cmd_info { |
409 | const char *name; |
410 | u32 opcode; |
411 | |
412 | #define F_LEN_MASK 3U |
413 | #define F_LEN_CONST 1U |
414 | #define F_LEN_VAR 0U |
415 | /* value is const although LEN maybe variable */ |
416 | #define F_LEN_VAR_FIXED (1<<1) |
417 | |
418 | /* |
419 | * command has its own ip advance logic |
420 | * e.g. MI_BATCH_START, MI_BATCH_END |
421 | */ |
422 | #define F_IP_ADVANCE_CUSTOM (1<<2) |
423 | u32 flag; |
424 | |
425 | #define R_RCS BIT(RCS0) |
426 | #define R_VCS1 BIT(VCS0) |
427 | #define R_VCS2 BIT(VCS1) |
428 | #define R_VCS (R_VCS1 | R_VCS2) |
429 | #define R_BCS BIT(BCS0) |
430 | #define R_VECS BIT(VECS0) |
431 | #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS) |
432 | /* rings that support this cmd: BLT/RCS/VCS/VECS */ |
433 | intel_engine_mask_t rings; |
434 | |
435 | /* devices that support this cmd: SNB/IVB/HSW/... */ |
436 | u16 devices; |
437 | |
438 | /* which DWords are address that need fix up. |
439 | * bit 0 means a 32-bit non address operand in command |
440 | * bit 1 means address operand, which could be 32-bit |
441 | * or 64-bit depending on different architectures.( |
442 | * defined by "gmadr_bytes_in_cmd" in intel_gvt. |
443 | * No matter the address length, each address only takes |
444 | * one bit in the bitmap. |
445 | */ |
446 | u16 addr_bitmap; |
447 | |
448 | /* flag == F_LEN_CONST : command length |
449 | * flag == F_LEN_VAR : length bias bits |
450 | * Note: length is in DWord |
451 | */ |
452 | u32 len; |
453 | |
454 | parser_cmd_handler handler; |
455 | |
456 | /* valid length in DWord */ |
457 | u32 valid_len; |
458 | }; |
459 | |
460 | struct cmd_entry { |
461 | struct hlist_node hlist; |
462 | const struct cmd_info *info; |
463 | }; |
464 | |
465 | enum { |
466 | RING_BUFFER_INSTRUCTION, |
467 | BATCH_BUFFER_INSTRUCTION, |
468 | BATCH_BUFFER_2ND_LEVEL, |
469 | RING_BUFFER_CTX, |
470 | }; |
471 | |
472 | enum { |
473 | GTT_BUFFER, |
474 | PPGTT_BUFFER |
475 | }; |
476 | |
477 | struct parser_exec_state { |
478 | struct intel_vgpu *vgpu; |
479 | const struct intel_engine_cs *engine; |
480 | |
481 | int buf_type; |
482 | |
483 | /* batch buffer address type */ |
484 | int buf_addr_type; |
485 | |
486 | /* graphics memory address of ring buffer start */ |
487 | unsigned long ring_start; |
488 | unsigned long ring_size; |
489 | unsigned long ring_head; |
490 | unsigned long ring_tail; |
491 | |
492 | /* instruction graphics memory address */ |
493 | unsigned long ip_gma; |
494 | |
495 | /* mapped va of the instr_gma */ |
496 | void *ip_va; |
497 | void *rb_va; |
498 | |
499 | void *ret_bb_va; |
500 | /* next instruction when return from batch buffer to ring buffer */ |
501 | unsigned long ret_ip_gma_ring; |
502 | |
503 | /* next instruction when return from 2nd batch buffer to batch buffer */ |
504 | unsigned long ret_ip_gma_bb; |
505 | |
506 | /* batch buffer address type (GTT or PPGTT) |
507 | * used when ret from 2nd level batch buffer |
508 | */ |
509 | int saved_buf_addr_type; |
510 | bool is_ctx_wa; |
511 | bool is_init_ctx; |
512 | |
513 | const struct cmd_info *info; |
514 | |
515 | struct intel_vgpu_workload *workload; |
516 | }; |
517 | |
518 | #define gmadr_dw_number(s) \ |
519 | (s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2) |
520 | |
521 | static unsigned long bypass_scan_mask = 0; |
522 | |
523 | /* ring ALL, type = 0 */ |
524 | static const struct sub_op_bits sub_op_mi[] = { |
525 | {31, 29}, |
526 | {28, 23}, |
527 | }; |
528 | |
529 | static const struct decode_info decode_info_mi = { |
530 | "MI" , |
531 | OP_LEN_MI, |
532 | ARRAY_SIZE(sub_op_mi), |
533 | sub_op_mi, |
534 | }; |
535 | |
536 | /* ring RCS, command type 2 */ |
537 | static const struct sub_op_bits sub_op_2d[] = { |
538 | {31, 29}, |
539 | {28, 22}, |
540 | }; |
541 | |
542 | static const struct decode_info decode_info_2d = { |
543 | "2D" , |
544 | OP_LEN_2D, |
545 | ARRAY_SIZE(sub_op_2d), |
546 | sub_op_2d, |
547 | }; |
548 | |
549 | /* ring RCS, command type 3 */ |
550 | static const struct sub_op_bits sub_op_3d_media[] = { |
551 | {31, 29}, |
552 | {28, 27}, |
553 | {26, 24}, |
554 | {23, 16}, |
555 | }; |
556 | |
557 | static const struct decode_info decode_info_3d_media = { |
558 | "3D_Media" , |
559 | OP_LEN_3D_MEDIA, |
560 | ARRAY_SIZE(sub_op_3d_media), |
561 | sub_op_3d_media, |
562 | }; |
563 | |
564 | /* ring VCS, command type 3 */ |
565 | static const struct sub_op_bits sub_op_mfx_vc[] = { |
566 | {31, 29}, |
567 | {28, 27}, |
568 | {26, 24}, |
569 | {23, 21}, |
570 | {20, 16}, |
571 | }; |
572 | |
573 | static const struct decode_info decode_info_mfx_vc = { |
574 | "MFX_VC" , |
575 | OP_LEN_MFX_VC, |
576 | ARRAY_SIZE(sub_op_mfx_vc), |
577 | sub_op_mfx_vc, |
578 | }; |
579 | |
580 | /* ring VECS, command type 3 */ |
581 | static const struct sub_op_bits sub_op_vebox[] = { |
582 | {31, 29}, |
583 | {28, 27}, |
584 | {26, 24}, |
585 | {23, 21}, |
586 | {20, 16}, |
587 | }; |
588 | |
589 | static const struct decode_info decode_info_vebox = { |
590 | "VEBOX" , |
591 | OP_LEN_VEBOX, |
592 | ARRAY_SIZE(sub_op_vebox), |
593 | sub_op_vebox, |
594 | }; |
595 | |
596 | static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = { |
597 | [RCS0] = { |
598 | &decode_info_mi, |
599 | NULL, |
600 | NULL, |
601 | &decode_info_3d_media, |
602 | NULL, |
603 | NULL, |
604 | NULL, |
605 | NULL, |
606 | }, |
607 | |
608 | [VCS0] = { |
609 | &decode_info_mi, |
610 | NULL, |
611 | NULL, |
612 | &decode_info_mfx_vc, |
613 | NULL, |
614 | NULL, |
615 | NULL, |
616 | NULL, |
617 | }, |
618 | |
619 | [BCS0] = { |
620 | &decode_info_mi, |
621 | NULL, |
622 | &decode_info_2d, |
623 | NULL, |
624 | NULL, |
625 | NULL, |
626 | NULL, |
627 | NULL, |
628 | }, |
629 | |
630 | [VECS0] = { |
631 | &decode_info_mi, |
632 | NULL, |
633 | NULL, |
634 | &decode_info_vebox, |
635 | NULL, |
636 | NULL, |
637 | NULL, |
638 | NULL, |
639 | }, |
640 | |
641 | [VCS1] = { |
642 | &decode_info_mi, |
643 | NULL, |
644 | NULL, |
645 | &decode_info_mfx_vc, |
646 | NULL, |
647 | NULL, |
648 | NULL, |
649 | NULL, |
650 | }, |
651 | }; |
652 | |
653 | static inline u32 get_opcode(u32 cmd, const struct intel_engine_cs *engine) |
654 | { |
655 | const struct decode_info *d_info; |
656 | |
657 | d_info = ring_decode_info[engine->id][CMD_TYPE(cmd)]; |
658 | if (d_info == NULL) |
659 | return INVALID_OP; |
660 | |
661 | return cmd >> (32 - d_info->op_len); |
662 | } |
663 | |
664 | static inline const struct cmd_info * |
665 | find_cmd_entry(struct intel_gvt *gvt, unsigned int opcode, |
666 | const struct intel_engine_cs *engine) |
667 | { |
668 | struct cmd_entry *e; |
669 | |
670 | hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) { |
671 | if (opcode == e->info->opcode && |
672 | e->info->rings & engine->mask) |
673 | return e->info; |
674 | } |
675 | return NULL; |
676 | } |
677 | |
678 | static inline const struct cmd_info * |
679 | get_cmd_info(struct intel_gvt *gvt, u32 cmd, |
680 | const struct intel_engine_cs *engine) |
681 | { |
682 | u32 opcode; |
683 | |
684 | opcode = get_opcode(cmd, engine); |
685 | if (opcode == INVALID_OP) |
686 | return NULL; |
687 | |
688 | return find_cmd_entry(gvt, opcode, engine); |
689 | } |
690 | |
691 | static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low) |
692 | { |
693 | return (cmd >> low) & ((1U << (hi - low + 1)) - 1); |
694 | } |
695 | |
696 | static inline void print_opcode(u32 cmd, const struct intel_engine_cs *engine) |
697 | { |
698 | const struct decode_info *d_info; |
699 | int i; |
700 | |
701 | d_info = ring_decode_info[engine->id][CMD_TYPE(cmd)]; |
702 | if (d_info == NULL) |
703 | return; |
704 | |
705 | gvt_dbg_cmd("opcode=0x%x %s sub_ops:" , |
706 | cmd >> (32 - d_info->op_len), d_info->name); |
707 | |
708 | for (i = 0; i < d_info->nr_sub_op; i++) |
709 | pr_err("0x%x " , sub_op_val(cmd, d_info->sub_op[i].hi, |
710 | d_info->sub_op[i].low)); |
711 | |
712 | pr_err("\n" ); |
713 | } |
714 | |
715 | static inline u32 *cmd_ptr(struct parser_exec_state *s, int index) |
716 | { |
717 | return s->ip_va + (index << 2); |
718 | } |
719 | |
720 | static inline u32 cmd_val(struct parser_exec_state *s, int index) |
721 | { |
722 | return *cmd_ptr(s, index); |
723 | } |
724 | |
725 | static inline bool is_init_ctx(struct parser_exec_state *s) |
726 | { |
727 | return (s->buf_type == RING_BUFFER_CTX && s->is_init_ctx); |
728 | } |
729 | |
730 | static void parser_exec_state_dump(struct parser_exec_state *s) |
731 | { |
732 | int cnt = 0; |
733 | int i; |
734 | |
735 | gvt_dbg_cmd(" vgpu%d RING%s: ring_start(%08lx) ring_end(%08lx)" |
736 | " ring_head(%08lx) ring_tail(%08lx)\n" , |
737 | s->vgpu->id, s->engine->name, |
738 | s->ring_start, s->ring_start + s->ring_size, |
739 | s->ring_head, s->ring_tail); |
740 | |
741 | gvt_dbg_cmd(" %s %s ip_gma(%08lx) " , |
742 | s->buf_type == RING_BUFFER_INSTRUCTION ? |
743 | "RING_BUFFER" : ((s->buf_type == RING_BUFFER_CTX) ? |
744 | "CTX_BUFFER" : "BATCH_BUFFER" ), |
745 | s->buf_addr_type == GTT_BUFFER ? |
746 | "GTT" : "PPGTT" , s->ip_gma); |
747 | |
748 | if (s->ip_va == NULL) { |
749 | gvt_dbg_cmd(" ip_va(NULL)" ); |
750 | return; |
751 | } |
752 | |
753 | gvt_dbg_cmd(" ip_va=%p: %08x %08x %08x %08x\n" , |
754 | s->ip_va, cmd_val(s, 0), cmd_val(s, 1), |
755 | cmd_val(s, 2), cmd_val(s, 3)); |
756 | |
757 | print_opcode(cmd: cmd_val(s, index: 0), engine: s->engine); |
758 | |
759 | s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12); |
760 | |
761 | while (cnt < 1024) { |
762 | gvt_dbg_cmd("ip_va=%p: " , s->ip_va); |
763 | for (i = 0; i < 8; i++) |
764 | gvt_dbg_cmd("%08x " , cmd_val(s, i)); |
765 | gvt_dbg_cmd("\n" ); |
766 | |
767 | s->ip_va += 8 * sizeof(u32); |
768 | cnt += 8; |
769 | } |
770 | } |
771 | |
772 | static inline void update_ip_va(struct parser_exec_state *s) |
773 | { |
774 | unsigned long len = 0; |
775 | |
776 | if (WARN_ON(s->ring_head == s->ring_tail)) |
777 | return; |
778 | |
779 | if (s->buf_type == RING_BUFFER_INSTRUCTION || |
780 | s->buf_type == RING_BUFFER_CTX) { |
781 | unsigned long ring_top = s->ring_start + s->ring_size; |
782 | |
783 | if (s->ring_head > s->ring_tail) { |
784 | if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top) |
785 | len = (s->ip_gma - s->ring_head); |
786 | else if (s->ip_gma >= s->ring_start && |
787 | s->ip_gma <= s->ring_tail) |
788 | len = (ring_top - s->ring_head) + |
789 | (s->ip_gma - s->ring_start); |
790 | } else |
791 | len = (s->ip_gma - s->ring_head); |
792 | |
793 | s->ip_va = s->rb_va + len; |
794 | } else {/* shadow batch buffer */ |
795 | s->ip_va = s->ret_bb_va; |
796 | } |
797 | } |
798 | |
799 | static inline int ip_gma_set(struct parser_exec_state *s, |
800 | unsigned long ip_gma) |
801 | { |
802 | WARN_ON(!IS_ALIGNED(ip_gma, 4)); |
803 | |
804 | s->ip_gma = ip_gma; |
805 | update_ip_va(s); |
806 | return 0; |
807 | } |
808 | |
809 | static inline int ip_gma_advance(struct parser_exec_state *s, |
810 | unsigned int dw_len) |
811 | { |
812 | s->ip_gma += (dw_len << 2); |
813 | |
814 | if (s->buf_type == RING_BUFFER_INSTRUCTION) { |
815 | if (s->ip_gma >= s->ring_start + s->ring_size) |
816 | s->ip_gma -= s->ring_size; |
817 | update_ip_va(s); |
818 | } else { |
819 | s->ip_va += (dw_len << 2); |
820 | } |
821 | |
822 | return 0; |
823 | } |
824 | |
825 | static inline int get_cmd_length(const struct cmd_info *info, u32 cmd) |
826 | { |
827 | if ((info->flag & F_LEN_MASK) == F_LEN_CONST) |
828 | return info->len; |
829 | else |
830 | return (cmd & ((1U << info->len) - 1)) + 2; |
831 | return 0; |
832 | } |
833 | |
834 | static inline int cmd_length(struct parser_exec_state *s) |
835 | { |
836 | return get_cmd_length(info: s->info, cmd: cmd_val(s, index: 0)); |
837 | } |
838 | |
839 | /* do not remove this, some platform may need clflush here */ |
840 | #define patch_value(s, addr, val) do { \ |
841 | *addr = val; \ |
842 | } while (0) |
843 | |
844 | static inline bool is_mocs_mmio(unsigned int offset) |
845 | { |
846 | return ((offset >= 0xc800) && (offset <= 0xcff8)) || |
847 | ((offset >= 0xb020) && (offset <= 0xb0a0)); |
848 | } |
849 | |
850 | static int is_cmd_update_pdps(unsigned int offset, |
851 | struct parser_exec_state *s) |
852 | { |
853 | u32 base = s->workload->engine->mmio_base; |
854 | return i915_mmio_reg_equal(_MMIO(offset), GEN8_RING_PDP_UDW(base, 0)); |
855 | } |
856 | |
857 | static int cmd_pdp_mmio_update_handler(struct parser_exec_state *s, |
858 | unsigned int offset, unsigned int index) |
859 | { |
860 | struct intel_vgpu *vgpu = s->vgpu; |
861 | struct intel_vgpu_mm *shadow_mm = s->workload->shadow_mm; |
862 | struct intel_vgpu_mm *mm; |
863 | u64 pdps[GEN8_3LVL_PDPES]; |
864 | |
865 | if (shadow_mm->ppgtt_mm.root_entry_type == |
866 | GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { |
867 | pdps[0] = (u64)cmd_val(s, index: 2) << 32; |
868 | pdps[0] |= cmd_val(s, index: 4); |
869 | |
870 | mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps); |
871 | if (!mm) { |
872 | gvt_vgpu_err("failed to get the 4-level shadow vm\n" ); |
873 | return -EINVAL; |
874 | } |
875 | intel_vgpu_mm_get(mm); |
876 | list_add_tail(new: &mm->ppgtt_mm.link, |
877 | head: &s->workload->lri_shadow_mm); |
878 | *cmd_ptr(s, index: 2) = upper_32_bits(mm->ppgtt_mm.shadow_pdps[0]); |
879 | *cmd_ptr(s, index: 4) = lower_32_bits(mm->ppgtt_mm.shadow_pdps[0]); |
880 | } else { |
881 | /* Currently all guests use PML4 table and now can't |
882 | * have a guest with 3-level table but uses LRI for |
883 | * PPGTT update. So this is simply un-testable. */ |
884 | GEM_BUG_ON(1); |
885 | gvt_vgpu_err("invalid shared shadow vm type\n" ); |
886 | return -EINVAL; |
887 | } |
888 | return 0; |
889 | } |
890 | |
891 | static int cmd_reg_handler(struct parser_exec_state *s, |
892 | unsigned int offset, unsigned int index, char *cmd) |
893 | { |
894 | struct intel_vgpu *vgpu = s->vgpu; |
895 | struct intel_gvt *gvt = vgpu->gvt; |
896 | u32 ctx_sr_ctl; |
897 | u32 *vreg, vreg_old; |
898 | |
899 | if (offset + 4 > gvt->device_info.mmio_size) { |
900 | gvt_vgpu_err("%s access to (%x) outside of MMIO range\n" , |
901 | cmd, offset); |
902 | return -EFAULT; |
903 | } |
904 | |
905 | if (is_init_ctx(s)) { |
906 | struct intel_gvt_mmio_info *mmio_info; |
907 | |
908 | intel_gvt_mmio_set_cmd_accessible(gvt, offset); |
909 | mmio_info = intel_gvt_find_mmio_info(gvt, offset); |
910 | if (mmio_info && mmio_info->write) |
911 | intel_gvt_mmio_set_cmd_write_patch(gvt, offset); |
912 | return 0; |
913 | } |
914 | |
915 | if (!intel_gvt_mmio_is_cmd_accessible(gvt, offset)) { |
916 | gvt_vgpu_err("%s access to non-render register (%x)\n" , |
917 | cmd, offset); |
918 | return -EBADRQC; |
919 | } |
920 | |
921 | if (!strncmp(cmd, "srm" , 3) || |
922 | !strncmp(cmd, "lrm" , 3)) { |
923 | if (offset == i915_mmio_reg_offset(GEN8_L3SQCREG4) || |
924 | offset == 0x21f0 || |
925 | (IS_BROADWELL(gvt->gt->i915) && |
926 | offset == i915_mmio_reg_offset(INSTPM))) |
927 | return 0; |
928 | else { |
929 | gvt_vgpu_err("%s access to register (%x)\n" , |
930 | cmd, offset); |
931 | return -EPERM; |
932 | } |
933 | } |
934 | |
935 | if (!strncmp(cmd, "lrr-src" , 7) || |
936 | !strncmp(cmd, "lrr-dst" , 7)) { |
937 | if (IS_BROADWELL(gvt->gt->i915) && offset == 0x215c) |
938 | return 0; |
939 | else { |
940 | gvt_vgpu_err("not allowed cmd %s reg (%x)\n" , cmd, offset); |
941 | return -EPERM; |
942 | } |
943 | } |
944 | |
945 | if (!strncmp(cmd, "pipe_ctrl" , 9)) { |
946 | /* TODO: add LRI POST logic here */ |
947 | return 0; |
948 | } |
949 | |
950 | if (strncmp(cmd, "lri" , 3)) |
951 | return -EPERM; |
952 | |
953 | /* below are all lri handlers */ |
954 | vreg = &vgpu_vreg(s->vgpu, offset); |
955 | |
956 | if (is_cmd_update_pdps(offset, s) && |
957 | cmd_pdp_mmio_update_handler(s, offset, index)) |
958 | return -EINVAL; |
959 | |
960 | if (offset == i915_mmio_reg_offset(DERRMR) || |
961 | offset == i915_mmio_reg_offset(FORCEWAKE_MT)) { |
962 | /* Writing to HW VGT_PVINFO_PAGE offset will be discarded */ |
963 | patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE); |
964 | } |
965 | |
966 | if (is_mocs_mmio(offset)) |
967 | *vreg = cmd_val(s, index: index + 1); |
968 | |
969 | vreg_old = *vreg; |
970 | |
971 | if (intel_gvt_mmio_is_cmd_write_patch(gvt, offset)) { |
972 | u32 cmdval_new, cmdval; |
973 | struct intel_gvt_mmio_info *mmio_info; |
974 | |
975 | cmdval = cmd_val(s, index: index + 1); |
976 | |
977 | mmio_info = intel_gvt_find_mmio_info(gvt, offset); |
978 | if (!mmio_info) { |
979 | cmdval_new = cmdval; |
980 | } else { |
981 | u64 ro_mask = mmio_info->ro_mask; |
982 | int ret; |
983 | |
984 | if (likely(!ro_mask)) |
985 | ret = mmio_info->write(s->vgpu, offset, |
986 | &cmdval, 4); |
987 | else { |
988 | gvt_vgpu_err("try to write RO reg %x\n" , |
989 | offset); |
990 | ret = -EBADRQC; |
991 | } |
992 | if (ret) |
993 | return ret; |
994 | cmdval_new = *vreg; |
995 | } |
996 | if (cmdval_new != cmdval) |
997 | patch_value(s, cmd_ptr(s, index+1), cmdval_new); |
998 | } |
999 | |
1000 | /* only patch cmd. restore vreg value if changed in mmio write handler*/ |
1001 | *vreg = vreg_old; |
1002 | |
1003 | /* TODO |
1004 | * In order to let workload with inhibit context to generate |
1005 | * correct image data into memory, vregs values will be loaded to |
1006 | * hw via LRIs in the workload with inhibit context. But as |
1007 | * indirect context is loaded prior to LRIs in workload, we don't |
1008 | * want reg values specified in indirect context overwritten by |
1009 | * LRIs in workloads. So, when scanning an indirect context, we |
1010 | * update reg values in it into vregs, so LRIs in workload with |
1011 | * inhibit context will restore with correct values |
1012 | */ |
1013 | if (GRAPHICS_VER(s->engine->i915) == 9 && |
1014 | intel_gvt_mmio_is_sr_in_ctx(gvt, offset) && |
1015 | !strncmp(cmd, "lri" , 3)) { |
1016 | intel_gvt_read_gpa(vgpu: s->vgpu, |
1017 | gpa: s->workload->ring_context_gpa + 12, buf: &ctx_sr_ctl, len: 4); |
1018 | /* check inhibit context */ |
1019 | if (ctx_sr_ctl & 1) { |
1020 | u32 data = cmd_val(s, index: index + 1); |
1021 | |
1022 | if (intel_gvt_mmio_has_mode_mask(gvt: s->vgpu->gvt, offset)) |
1023 | intel_vgpu_mask_mmio_write(vgpu, |
1024 | offset, p_data: &data, bytes: 4); |
1025 | else |
1026 | vgpu_vreg(vgpu, offset) = data; |
1027 | } |
1028 | } |
1029 | |
1030 | return 0; |
1031 | } |
1032 | |
1033 | #define cmd_reg(s, i) \ |
1034 | (cmd_val(s, i) & GENMASK(22, 2)) |
1035 | |
1036 | #define cmd_reg_inhibit(s, i) \ |
1037 | (cmd_val(s, i) & GENMASK(22, 18)) |
1038 | |
1039 | #define cmd_gma(s, i) \ |
1040 | (cmd_val(s, i) & GENMASK(31, 2)) |
1041 | |
1042 | #define cmd_gma_hi(s, i) \ |
1043 | (cmd_val(s, i) & GENMASK(15, 0)) |
1044 | |
1045 | static int cmd_handler_lri(struct parser_exec_state *s) |
1046 | { |
1047 | int i, ret = 0; |
1048 | int cmd_len = cmd_length(s); |
1049 | |
1050 | for (i = 1; i < cmd_len; i += 2) { |
1051 | if (IS_BROADWELL(s->engine->i915) && s->engine->id != RCS0) { |
1052 | if (s->engine->id == BCS0 && |
1053 | cmd_reg(s, i) == i915_mmio_reg_offset(DERRMR)) |
1054 | ret |= 0; |
1055 | else |
1056 | ret |= cmd_reg_inhibit(s, i) ? -EBADRQC : 0; |
1057 | } |
1058 | if (ret) |
1059 | break; |
1060 | ret |= cmd_reg_handler(s, cmd_reg(s, i), index: i, cmd: "lri" ); |
1061 | if (ret) |
1062 | break; |
1063 | } |
1064 | return ret; |
1065 | } |
1066 | |
1067 | static int cmd_handler_lrr(struct parser_exec_state *s) |
1068 | { |
1069 | int i, ret = 0; |
1070 | int cmd_len = cmd_length(s); |
1071 | |
1072 | for (i = 1; i < cmd_len; i += 2) { |
1073 | if (IS_BROADWELL(s->engine->i915)) |
1074 | ret |= ((cmd_reg_inhibit(s, i) || |
1075 | (cmd_reg_inhibit(s, i + 1)))) ? |
1076 | -EBADRQC : 0; |
1077 | if (ret) |
1078 | break; |
1079 | ret |= cmd_reg_handler(s, cmd_reg(s, i), index: i, cmd: "lrr-src" ); |
1080 | if (ret) |
1081 | break; |
1082 | ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), index: i, cmd: "lrr-dst" ); |
1083 | if (ret) |
1084 | break; |
1085 | } |
1086 | return ret; |
1087 | } |
1088 | |
1089 | static inline int cmd_address_audit(struct parser_exec_state *s, |
1090 | unsigned long guest_gma, int op_size, bool index_mode); |
1091 | |
1092 | static int cmd_handler_lrm(struct parser_exec_state *s) |
1093 | { |
1094 | struct intel_gvt *gvt = s->vgpu->gvt; |
1095 | int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; |
1096 | unsigned long gma; |
1097 | int i, ret = 0; |
1098 | int cmd_len = cmd_length(s); |
1099 | |
1100 | for (i = 1; i < cmd_len;) { |
1101 | if (IS_BROADWELL(s->engine->i915)) |
1102 | ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0; |
1103 | if (ret) |
1104 | break; |
1105 | ret |= cmd_reg_handler(s, cmd_reg(s, i), index: i, cmd: "lrm" ); |
1106 | if (ret) |
1107 | break; |
1108 | if (cmd_val(s, index: 0) & (1 << 22)) { |
1109 | gma = cmd_gma(s, i + 1); |
1110 | if (gmadr_bytes == 8) |
1111 | gma |= (cmd_gma_hi(s, i + 2)) << 32; |
1112 | ret |= cmd_address_audit(s, guest_gma: gma, op_size: sizeof(u32), index_mode: false); |
1113 | if (ret) |
1114 | break; |
1115 | } |
1116 | i += gmadr_dw_number(s) + 1; |
1117 | } |
1118 | return ret; |
1119 | } |
1120 | |
1121 | static int cmd_handler_srm(struct parser_exec_state *s) |
1122 | { |
1123 | int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
1124 | unsigned long gma; |
1125 | int i, ret = 0; |
1126 | int cmd_len = cmd_length(s); |
1127 | |
1128 | for (i = 1; i < cmd_len;) { |
1129 | ret |= cmd_reg_handler(s, cmd_reg(s, i), index: i, cmd: "srm" ); |
1130 | if (ret) |
1131 | break; |
1132 | if (cmd_val(s, index: 0) & (1 << 22)) { |
1133 | gma = cmd_gma(s, i + 1); |
1134 | if (gmadr_bytes == 8) |
1135 | gma |= (cmd_gma_hi(s, i + 2)) << 32; |
1136 | ret |= cmd_address_audit(s, guest_gma: gma, op_size: sizeof(u32), index_mode: false); |
1137 | if (ret) |
1138 | break; |
1139 | } |
1140 | i += gmadr_dw_number(s) + 1; |
1141 | } |
1142 | return ret; |
1143 | } |
1144 | |
1145 | struct cmd_interrupt_event { |
1146 | int pipe_control_notify; |
1147 | int mi_flush_dw; |
1148 | int mi_user_interrupt; |
1149 | }; |
1150 | |
1151 | static const struct cmd_interrupt_event cmd_interrupt_events[] = { |
1152 | [RCS0] = { |
1153 | .pipe_control_notify = RCS_PIPE_CONTROL, |
1154 | .mi_flush_dw = INTEL_GVT_EVENT_RESERVED, |
1155 | .mi_user_interrupt = RCS_MI_USER_INTERRUPT, |
1156 | }, |
1157 | [BCS0] = { |
1158 | .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, |
1159 | .mi_flush_dw = BCS_MI_FLUSH_DW, |
1160 | .mi_user_interrupt = BCS_MI_USER_INTERRUPT, |
1161 | }, |
1162 | [VCS0] = { |
1163 | .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, |
1164 | .mi_flush_dw = VCS_MI_FLUSH_DW, |
1165 | .mi_user_interrupt = VCS_MI_USER_INTERRUPT, |
1166 | }, |
1167 | [VCS1] = { |
1168 | .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, |
1169 | .mi_flush_dw = VCS2_MI_FLUSH_DW, |
1170 | .mi_user_interrupt = VCS2_MI_USER_INTERRUPT, |
1171 | }, |
1172 | [VECS0] = { |
1173 | .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, |
1174 | .mi_flush_dw = VECS_MI_FLUSH_DW, |
1175 | .mi_user_interrupt = VECS_MI_USER_INTERRUPT, |
1176 | }, |
1177 | }; |
1178 | |
1179 | static int cmd_handler_pipe_control(struct parser_exec_state *s) |
1180 | { |
1181 | int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
1182 | unsigned long gma; |
1183 | bool index_mode = false; |
1184 | unsigned int post_sync; |
1185 | int ret = 0; |
1186 | u32 hws_pga, val; |
1187 | |
1188 | post_sync = (cmd_val(s, index: 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14; |
1189 | |
1190 | /* LRI post sync */ |
1191 | if (cmd_val(s, index: 1) & PIPE_CONTROL_MMIO_WRITE) |
1192 | ret = cmd_reg_handler(s, cmd_reg(s, 2), index: 1, cmd: "pipe_ctrl" ); |
1193 | /* post sync */ |
1194 | else if (post_sync) { |
1195 | if (post_sync == 2) |
1196 | ret = cmd_reg_handler(s, offset: 0x2350, index: 1, cmd: "pipe_ctrl" ); |
1197 | else if (post_sync == 3) |
1198 | ret = cmd_reg_handler(s, offset: 0x2358, index: 1, cmd: "pipe_ctrl" ); |
1199 | else if (post_sync == 1) { |
1200 | /* check ggtt*/ |
1201 | if ((cmd_val(s, index: 1) & PIPE_CONTROL_GLOBAL_GTT_IVB)) { |
1202 | gma = cmd_val(s, index: 2) & GENMASK(31, 3); |
1203 | if (gmadr_bytes == 8) |
1204 | gma |= (cmd_gma_hi(s, 3)) << 32; |
1205 | /* Store Data Index */ |
1206 | if (cmd_val(s, index: 1) & (1 << 21)) |
1207 | index_mode = true; |
1208 | ret |= cmd_address_audit(s, guest_gma: gma, op_size: sizeof(u64), |
1209 | index_mode); |
1210 | if (ret) |
1211 | return ret; |
1212 | if (index_mode) { |
1213 | hws_pga = s->vgpu->hws_pga[s->engine->id]; |
1214 | gma = hws_pga + gma; |
1215 | patch_value(s, cmd_ptr(s, 2), gma); |
1216 | val = cmd_val(s, index: 1) & (~(1 << 21)); |
1217 | patch_value(s, cmd_ptr(s, 1), val); |
1218 | } |
1219 | } |
1220 | } |
1221 | } |
1222 | |
1223 | if (ret) |
1224 | return ret; |
1225 | |
1226 | if (cmd_val(s, index: 1) & PIPE_CONTROL_NOTIFY) |
1227 | set_bit(nr: cmd_interrupt_events[s->engine->id].pipe_control_notify, |
1228 | addr: s->workload->pending_events); |
1229 | return 0; |
1230 | } |
1231 | |
1232 | static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s) |
1233 | { |
1234 | set_bit(nr: cmd_interrupt_events[s->engine->id].mi_user_interrupt, |
1235 | addr: s->workload->pending_events); |
1236 | patch_value(s, cmd_ptr(s, 0), MI_NOOP); |
1237 | return 0; |
1238 | } |
1239 | |
1240 | static int cmd_advance_default(struct parser_exec_state *s) |
1241 | { |
1242 | return ip_gma_advance(s, dw_len: cmd_length(s)); |
1243 | } |
1244 | |
1245 | static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s) |
1246 | { |
1247 | int ret; |
1248 | |
1249 | if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { |
1250 | s->buf_type = BATCH_BUFFER_INSTRUCTION; |
1251 | ret = ip_gma_set(s, ip_gma: s->ret_ip_gma_bb); |
1252 | s->buf_addr_type = s->saved_buf_addr_type; |
1253 | } else if (s->buf_type == RING_BUFFER_CTX) { |
1254 | ret = ip_gma_set(s, ip_gma: s->ring_tail); |
1255 | } else { |
1256 | s->buf_type = RING_BUFFER_INSTRUCTION; |
1257 | s->buf_addr_type = GTT_BUFFER; |
1258 | if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size) |
1259 | s->ret_ip_gma_ring -= s->ring_size; |
1260 | ret = ip_gma_set(s, ip_gma: s->ret_ip_gma_ring); |
1261 | } |
1262 | return ret; |
1263 | } |
1264 | |
1265 | struct mi_display_flip_command_info { |
1266 | int pipe; |
1267 | int plane; |
1268 | int event; |
1269 | i915_reg_t stride_reg; |
1270 | i915_reg_t ctrl_reg; |
1271 | i915_reg_t surf_reg; |
1272 | u64 stride_val; |
1273 | u64 tile_val; |
1274 | u64 surf_val; |
1275 | bool async_flip; |
1276 | }; |
1277 | |
1278 | struct plane_code_mapping { |
1279 | int pipe; |
1280 | int plane; |
1281 | int event; |
1282 | }; |
1283 | |
1284 | static int gen8_decode_mi_display_flip(struct parser_exec_state *s, |
1285 | struct mi_display_flip_command_info *info) |
1286 | { |
1287 | struct drm_i915_private *dev_priv = s->engine->i915; |
1288 | struct plane_code_mapping gen8_plane_code[] = { |
1289 | [0] = {.pipe: PIPE_A, .plane: PLANE_A, .event: PRIMARY_A_FLIP_DONE}, |
1290 | [1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE}, |
1291 | [2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE}, |
1292 | [3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE}, |
1293 | [4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE}, |
1294 | [5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE}, |
1295 | }; |
1296 | u32 dword0, dword1, dword2; |
1297 | u32 v; |
1298 | |
1299 | dword0 = cmd_val(s, index: 0); |
1300 | dword1 = cmd_val(s, index: 1); |
1301 | dword2 = cmd_val(s, index: 2); |
1302 | |
1303 | v = (dword0 & GENMASK(21, 19)) >> 19; |
1304 | if (drm_WARN_ON(&dev_priv->drm, v >= ARRAY_SIZE(gen8_plane_code))) |
1305 | return -EBADRQC; |
1306 | |
1307 | info->pipe = gen8_plane_code[v].pipe; |
1308 | info->plane = gen8_plane_code[v].plane; |
1309 | info->event = gen8_plane_code[v].event; |
1310 | info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; |
1311 | info->tile_val = (dword1 & 0x1); |
1312 | info->surf_val = (dword2 & GENMASK(31, 12)) >> 12; |
1313 | info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1); |
1314 | |
1315 | if (info->plane == PLANE_A) { |
1316 | info->ctrl_reg = DSPCNTR(info->pipe); |
1317 | info->stride_reg = DSPSTRIDE(info->pipe); |
1318 | info->surf_reg = DSPSURF(info->pipe); |
1319 | } else if (info->plane == PLANE_B) { |
1320 | info->ctrl_reg = SPRCTL(info->pipe); |
1321 | info->stride_reg = SPRSTRIDE(info->pipe); |
1322 | info->surf_reg = SPRSURF(info->pipe); |
1323 | } else { |
1324 | drm_WARN_ON(&dev_priv->drm, 1); |
1325 | return -EBADRQC; |
1326 | } |
1327 | return 0; |
1328 | } |
1329 | |
1330 | static int skl_decode_mi_display_flip(struct parser_exec_state *s, |
1331 | struct mi_display_flip_command_info *info) |
1332 | { |
1333 | struct drm_i915_private *dev_priv = s->engine->i915; |
1334 | struct intel_vgpu *vgpu = s->vgpu; |
1335 | u32 dword0 = cmd_val(s, index: 0); |
1336 | u32 dword1 = cmd_val(s, index: 1); |
1337 | u32 dword2 = cmd_val(s, index: 2); |
1338 | u32 plane = (dword0 & GENMASK(12, 8)) >> 8; |
1339 | |
1340 | info->plane = PRIMARY_PLANE; |
1341 | |
1342 | switch (plane) { |
1343 | case MI_DISPLAY_FLIP_SKL_PLANE_1_A: |
1344 | info->pipe = PIPE_A; |
1345 | info->event = PRIMARY_A_FLIP_DONE; |
1346 | break; |
1347 | case MI_DISPLAY_FLIP_SKL_PLANE_1_B: |
1348 | info->pipe = PIPE_B; |
1349 | info->event = PRIMARY_B_FLIP_DONE; |
1350 | break; |
1351 | case MI_DISPLAY_FLIP_SKL_PLANE_1_C: |
1352 | info->pipe = PIPE_C; |
1353 | info->event = PRIMARY_C_FLIP_DONE; |
1354 | break; |
1355 | |
1356 | case MI_DISPLAY_FLIP_SKL_PLANE_2_A: |
1357 | info->pipe = PIPE_A; |
1358 | info->event = SPRITE_A_FLIP_DONE; |
1359 | info->plane = SPRITE_PLANE; |
1360 | break; |
1361 | case MI_DISPLAY_FLIP_SKL_PLANE_2_B: |
1362 | info->pipe = PIPE_B; |
1363 | info->event = SPRITE_B_FLIP_DONE; |
1364 | info->plane = SPRITE_PLANE; |
1365 | break; |
1366 | case MI_DISPLAY_FLIP_SKL_PLANE_2_C: |
1367 | info->pipe = PIPE_C; |
1368 | info->event = SPRITE_C_FLIP_DONE; |
1369 | info->plane = SPRITE_PLANE; |
1370 | break; |
1371 | |
1372 | default: |
1373 | gvt_vgpu_err("unknown plane code %d\n" , plane); |
1374 | return -EBADRQC; |
1375 | } |
1376 | |
1377 | info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; |
1378 | info->tile_val = (dword1 & GENMASK(2, 0)); |
1379 | info->surf_val = (dword2 & GENMASK(31, 12)) >> 12; |
1380 | info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1); |
1381 | |
1382 | info->ctrl_reg = DSPCNTR(info->pipe); |
1383 | info->stride_reg = DSPSTRIDE(info->pipe); |
1384 | info->surf_reg = DSPSURF(info->pipe); |
1385 | |
1386 | return 0; |
1387 | } |
1388 | |
1389 | static int gen8_check_mi_display_flip(struct parser_exec_state *s, |
1390 | struct mi_display_flip_command_info *info) |
1391 | { |
1392 | u32 stride, tile; |
1393 | |
1394 | if (!info->async_flip) |
1395 | return 0; |
1396 | |
1397 | if (GRAPHICS_VER(s->engine->i915) >= 9) { |
1398 | stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0); |
1399 | tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & |
1400 | GENMASK(12, 10)) >> 10; |
1401 | } else { |
1402 | stride = (vgpu_vreg_t(s->vgpu, info->stride_reg) & |
1403 | GENMASK(15, 6)) >> 6; |
1404 | tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10; |
1405 | } |
1406 | |
1407 | if (stride != info->stride_val) |
1408 | gvt_dbg_cmd("cannot change stride during async flip\n" ); |
1409 | |
1410 | if (tile != info->tile_val) |
1411 | gvt_dbg_cmd("cannot change tile during async flip\n" ); |
1412 | |
1413 | return 0; |
1414 | } |
1415 | |
1416 | static int gen8_update_plane_mmio_from_mi_display_flip( |
1417 | struct parser_exec_state *s, |
1418 | struct mi_display_flip_command_info *info) |
1419 | { |
1420 | struct drm_i915_private *dev_priv = s->engine->i915; |
1421 | struct intel_vgpu *vgpu = s->vgpu; |
1422 | |
1423 | set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12), |
1424 | info->surf_val << 12); |
1425 | if (GRAPHICS_VER(dev_priv) >= 9) { |
1426 | set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0), |
1427 | info->stride_val); |
1428 | set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10), |
1429 | info->tile_val << 10); |
1430 | } else { |
1431 | set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(15, 6), |
1432 | info->stride_val << 6); |
1433 | set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(10, 10), |
1434 | info->tile_val << 10); |
1435 | } |
1436 | |
1437 | if (info->plane == PLANE_PRIMARY) |
1438 | vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(info->pipe))++; |
1439 | |
1440 | if (info->async_flip) |
1441 | intel_vgpu_trigger_virtual_event(vgpu, event: info->event); |
1442 | else |
1443 | set_bit(nr: info->event, addr: vgpu->irq.flip_done_event[info->pipe]); |
1444 | |
1445 | return 0; |
1446 | } |
1447 | |
1448 | static int decode_mi_display_flip(struct parser_exec_state *s, |
1449 | struct mi_display_flip_command_info *info) |
1450 | { |
1451 | if (IS_BROADWELL(s->engine->i915)) |
1452 | return gen8_decode_mi_display_flip(s, info); |
1453 | if (GRAPHICS_VER(s->engine->i915) >= 9) |
1454 | return skl_decode_mi_display_flip(s, info); |
1455 | |
1456 | return -ENODEV; |
1457 | } |
1458 | |
1459 | static int check_mi_display_flip(struct parser_exec_state *s, |
1460 | struct mi_display_flip_command_info *info) |
1461 | { |
1462 | return gen8_check_mi_display_flip(s, info); |
1463 | } |
1464 | |
1465 | static int update_plane_mmio_from_mi_display_flip( |
1466 | struct parser_exec_state *s, |
1467 | struct mi_display_flip_command_info *info) |
1468 | { |
1469 | return gen8_update_plane_mmio_from_mi_display_flip(s, info); |
1470 | } |
1471 | |
1472 | static int cmd_handler_mi_display_flip(struct parser_exec_state *s) |
1473 | { |
1474 | struct mi_display_flip_command_info info; |
1475 | struct intel_vgpu *vgpu = s->vgpu; |
1476 | int ret; |
1477 | int i; |
1478 | int len = cmd_length(s); |
1479 | u32 valid_len = CMD_LEN(1); |
1480 | |
1481 | /* Flip Type == Stereo 3D Flip */ |
1482 | if (DWORD_FIELD(2, 1, 0) == 2) |
1483 | valid_len++; |
1484 | ret = gvt_check_valid_cmd_length(len: cmd_length(s), |
1485 | valid_len); |
1486 | if (ret) |
1487 | return ret; |
1488 | |
1489 | ret = decode_mi_display_flip(s, info: &info); |
1490 | if (ret) { |
1491 | gvt_vgpu_err("fail to decode MI display flip command\n" ); |
1492 | return ret; |
1493 | } |
1494 | |
1495 | ret = check_mi_display_flip(s, info: &info); |
1496 | if (ret) { |
1497 | gvt_vgpu_err("invalid MI display flip command\n" ); |
1498 | return ret; |
1499 | } |
1500 | |
1501 | ret = update_plane_mmio_from_mi_display_flip(s, info: &info); |
1502 | if (ret) { |
1503 | gvt_vgpu_err("fail to update plane mmio\n" ); |
1504 | return ret; |
1505 | } |
1506 | |
1507 | for (i = 0; i < len; i++) |
1508 | patch_value(s, cmd_ptr(s, i), MI_NOOP); |
1509 | return 0; |
1510 | } |
1511 | |
1512 | static bool is_wait_for_flip_pending(u32 cmd) |
1513 | { |
1514 | return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING | |
1515 | MI_WAIT_FOR_PLANE_B_FLIP_PENDING | |
1516 | MI_WAIT_FOR_PLANE_C_FLIP_PENDING | |
1517 | MI_WAIT_FOR_SPRITE_A_FLIP_PENDING | |
1518 | MI_WAIT_FOR_SPRITE_B_FLIP_PENDING | |
1519 | MI_WAIT_FOR_SPRITE_C_FLIP_PENDING); |
1520 | } |
1521 | |
1522 | static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s) |
1523 | { |
1524 | u32 cmd = cmd_val(s, index: 0); |
1525 | |
1526 | if (!is_wait_for_flip_pending(cmd)) |
1527 | return 0; |
1528 | |
1529 | patch_value(s, cmd_ptr(s, 0), MI_NOOP); |
1530 | return 0; |
1531 | } |
1532 | |
1533 | static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index) |
1534 | { |
1535 | unsigned long addr; |
1536 | unsigned long gma_high, gma_low; |
1537 | struct intel_vgpu *vgpu = s->vgpu; |
1538 | int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
1539 | |
1540 | if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) { |
1541 | gvt_vgpu_err("invalid gma bytes %d\n" , gmadr_bytes); |
1542 | return INTEL_GVT_INVALID_ADDR; |
1543 | } |
1544 | |
1545 | gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK; |
1546 | if (gmadr_bytes == 4) { |
1547 | addr = gma_low; |
1548 | } else { |
1549 | gma_high = cmd_val(s, index: index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK; |
1550 | addr = (((unsigned long)gma_high) << 32) | gma_low; |
1551 | } |
1552 | return addr; |
1553 | } |
1554 | |
1555 | static inline int cmd_address_audit(struct parser_exec_state *s, |
1556 | unsigned long guest_gma, int op_size, bool index_mode) |
1557 | { |
1558 | struct intel_vgpu *vgpu = s->vgpu; |
1559 | u32 max_surface_size = vgpu->gvt->device_info.max_surface_size; |
1560 | int i; |
1561 | int ret; |
1562 | |
1563 | if (op_size > max_surface_size) { |
1564 | gvt_vgpu_err("command address audit fail name %s\n" , |
1565 | s->info->name); |
1566 | return -EFAULT; |
1567 | } |
1568 | |
1569 | if (index_mode) { |
1570 | if (guest_gma >= I915_GTT_PAGE_SIZE) { |
1571 | ret = -EFAULT; |
1572 | goto err; |
1573 | } |
1574 | } else if (!intel_gvt_ggtt_validate_range(vgpu, addr: guest_gma, size: op_size)) { |
1575 | ret = -EFAULT; |
1576 | goto err; |
1577 | } |
1578 | |
1579 | return 0; |
1580 | |
1581 | err: |
1582 | gvt_vgpu_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n" , |
1583 | s->info->name, guest_gma, op_size); |
1584 | |
1585 | pr_err("cmd dump: " ); |
1586 | for (i = 0; i < cmd_length(s); i++) { |
1587 | if (!(i % 4)) |
1588 | pr_err("\n%08x " , cmd_val(s, i)); |
1589 | else |
1590 | pr_err("%08x " , cmd_val(s, i)); |
1591 | } |
1592 | pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n" , |
1593 | vgpu->id, |
1594 | vgpu_aperture_gmadr_base(vgpu), |
1595 | vgpu_aperture_gmadr_end(vgpu), |
1596 | vgpu_hidden_gmadr_base(vgpu), |
1597 | vgpu_hidden_gmadr_end(vgpu)); |
1598 | return ret; |
1599 | } |
1600 | |
1601 | static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s) |
1602 | { |
1603 | int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
1604 | int op_size = (cmd_length(s) - 3) * sizeof(u32); |
1605 | int core_id = (cmd_val(s, index: 2) & (1 << 0)) ? 1 : 0; |
1606 | unsigned long gma, gma_low, gma_high; |
1607 | u32 valid_len = CMD_LEN(2); |
1608 | int ret = 0; |
1609 | |
1610 | /* check ppggt */ |
1611 | if (!(cmd_val(s, index: 0) & (1 << 22))) |
1612 | return 0; |
1613 | |
1614 | /* check if QWORD */ |
1615 | if (DWORD_FIELD(0, 21, 21)) |
1616 | valid_len++; |
1617 | ret = gvt_check_valid_cmd_length(len: cmd_length(s), |
1618 | valid_len); |
1619 | if (ret) |
1620 | return ret; |
1621 | |
1622 | gma = cmd_val(s, index: 2) & GENMASK(31, 2); |
1623 | |
1624 | if (gmadr_bytes == 8) { |
1625 | gma_low = cmd_val(s, index: 1) & GENMASK(31, 2); |
1626 | gma_high = cmd_val(s, index: 2) & GENMASK(15, 0); |
1627 | gma = (gma_high << 32) | gma_low; |
1628 | core_id = (cmd_val(s, index: 1) & (1 << 0)) ? 1 : 0; |
1629 | } |
1630 | ret = cmd_address_audit(s, guest_gma: gma + op_size * core_id, op_size, index_mode: false); |
1631 | return ret; |
1632 | } |
1633 | |
1634 | static inline int unexpected_cmd(struct parser_exec_state *s) |
1635 | { |
1636 | struct intel_vgpu *vgpu = s->vgpu; |
1637 | |
1638 | gvt_vgpu_err("Unexpected %s in command buffer!\n" , s->info->name); |
1639 | |
1640 | return -EBADRQC; |
1641 | } |
1642 | |
1643 | static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s) |
1644 | { |
1645 | return unexpected_cmd(s); |
1646 | } |
1647 | |
1648 | static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s) |
1649 | { |
1650 | return unexpected_cmd(s); |
1651 | } |
1652 | |
1653 | static int cmd_handler_mi_op_2e(struct parser_exec_state *s) |
1654 | { |
1655 | return unexpected_cmd(s); |
1656 | } |
1657 | |
1658 | static int cmd_handler_mi_op_2f(struct parser_exec_state *s) |
1659 | { |
1660 | int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
1661 | int op_size = (1 << ((cmd_val(s, index: 0) & GENMASK(20, 19)) >> 19)) * |
1662 | sizeof(u32); |
1663 | unsigned long gma, gma_high; |
1664 | u32 valid_len = CMD_LEN(1); |
1665 | int ret = 0; |
1666 | |
1667 | if (!(cmd_val(s, index: 0) & (1 << 22))) |
1668 | return ret; |
1669 | |
1670 | /* check inline data */ |
1671 | if (cmd_val(s, index: 0) & BIT(18)) |
1672 | valid_len = CMD_LEN(9); |
1673 | ret = gvt_check_valid_cmd_length(len: cmd_length(s), |
1674 | valid_len); |
1675 | if (ret) |
1676 | return ret; |
1677 | |
1678 | gma = cmd_val(s, index: 1) & GENMASK(31, 2); |
1679 | if (gmadr_bytes == 8) { |
1680 | gma_high = cmd_val(s, index: 2) & GENMASK(15, 0); |
1681 | gma = (gma_high << 32) | gma; |
1682 | } |
1683 | ret = cmd_address_audit(s, guest_gma: gma, op_size, index_mode: false); |
1684 | return ret; |
1685 | } |
1686 | |
1687 | static int cmd_handler_mi_store_data_index(struct parser_exec_state *s) |
1688 | { |
1689 | return unexpected_cmd(s); |
1690 | } |
1691 | |
1692 | static int cmd_handler_mi_clflush(struct parser_exec_state *s) |
1693 | { |
1694 | return unexpected_cmd(s); |
1695 | } |
1696 | |
1697 | static int cmd_handler_mi_conditional_batch_buffer_end( |
1698 | struct parser_exec_state *s) |
1699 | { |
1700 | return unexpected_cmd(s); |
1701 | } |
1702 | |
1703 | static int cmd_handler_mi_update_gtt(struct parser_exec_state *s) |
1704 | { |
1705 | return unexpected_cmd(s); |
1706 | } |
1707 | |
1708 | static int cmd_handler_mi_flush_dw(struct parser_exec_state *s) |
1709 | { |
1710 | int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
1711 | unsigned long gma; |
1712 | bool index_mode = false; |
1713 | int ret = 0; |
1714 | u32 hws_pga, val; |
1715 | u32 valid_len = CMD_LEN(2); |
1716 | |
1717 | ret = gvt_check_valid_cmd_length(len: cmd_length(s), |
1718 | valid_len); |
1719 | if (ret) { |
1720 | /* Check again for Qword */ |
1721 | ret = gvt_check_valid_cmd_length(len: cmd_length(s), |
1722 | valid_len: ++valid_len); |
1723 | return ret; |
1724 | } |
1725 | |
1726 | /* Check post-sync and ppgtt bit */ |
1727 | if (((cmd_val(s, index: 0) >> 14) & 0x3) && (cmd_val(s, index: 1) & (1 << 2))) { |
1728 | gma = cmd_val(s, index: 1) & GENMASK(31, 3); |
1729 | if (gmadr_bytes == 8) |
1730 | gma |= (cmd_val(s, index: 2) & GENMASK(15, 0)) << 32; |
1731 | /* Store Data Index */ |
1732 | if (cmd_val(s, index: 0) & (1 << 21)) |
1733 | index_mode = true; |
1734 | ret = cmd_address_audit(s, guest_gma: gma, op_size: sizeof(u64), index_mode); |
1735 | if (ret) |
1736 | return ret; |
1737 | if (index_mode) { |
1738 | hws_pga = s->vgpu->hws_pga[s->engine->id]; |
1739 | gma = hws_pga + gma; |
1740 | patch_value(s, cmd_ptr(s, 1), gma); |
1741 | val = cmd_val(s, index: 0) & (~(1 << 21)); |
1742 | patch_value(s, cmd_ptr(s, 0), val); |
1743 | } |
1744 | } |
1745 | /* Check notify bit */ |
1746 | if ((cmd_val(s, index: 0) & (1 << 8))) |
1747 | set_bit(nr: cmd_interrupt_events[s->engine->id].mi_flush_dw, |
1748 | addr: s->workload->pending_events); |
1749 | return ret; |
1750 | } |
1751 | |
1752 | static void addr_type_update_snb(struct parser_exec_state *s) |
1753 | { |
1754 | if ((s->buf_type == RING_BUFFER_INSTRUCTION) && |
1755 | (BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) { |
1756 | s->buf_addr_type = PPGTT_BUFFER; |
1757 | } |
1758 | } |
1759 | |
1760 | |
1761 | static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm, |
1762 | unsigned long gma, unsigned long end_gma, void *va) |
1763 | { |
1764 | unsigned long copy_len, offset; |
1765 | unsigned long len = 0; |
1766 | unsigned long gpa; |
1767 | |
1768 | while (gma != end_gma) { |
1769 | gpa = intel_vgpu_gma_to_gpa(mm, gma); |
1770 | if (gpa == INTEL_GVT_INVALID_ADDR) { |
1771 | gvt_vgpu_err("invalid gma address: %lx\n" , gma); |
1772 | return -EFAULT; |
1773 | } |
1774 | |
1775 | offset = gma & (I915_GTT_PAGE_SIZE - 1); |
1776 | |
1777 | copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ? |
1778 | I915_GTT_PAGE_SIZE - offset : end_gma - gma; |
1779 | |
1780 | intel_gvt_read_gpa(vgpu, gpa, buf: va + len, len: copy_len); |
1781 | |
1782 | len += copy_len; |
1783 | gma += copy_len; |
1784 | } |
1785 | return len; |
1786 | } |
1787 | |
1788 | |
1789 | /* |
1790 | * Check whether a batch buffer needs to be scanned. Currently |
1791 | * the only criteria is based on privilege. |
1792 | */ |
1793 | static int batch_buffer_needs_scan(struct parser_exec_state *s) |
1794 | { |
1795 | /* Decide privilege based on address space */ |
1796 | if (cmd_val(s, index: 0) & BIT(8) && |
1797 | !(s->vgpu->scan_nonprivbb & s->engine->mask)) |
1798 | return 0; |
1799 | |
1800 | return 1; |
1801 | } |
1802 | |
1803 | static const char *repr_addr_type(unsigned int type) |
1804 | { |
1805 | return type == PPGTT_BUFFER ? "ppgtt" : "ggtt" ; |
1806 | } |
1807 | |
1808 | static int find_bb_size(struct parser_exec_state *s, |
1809 | unsigned long *bb_size, |
1810 | unsigned long *bb_end_cmd_offset) |
1811 | { |
1812 | unsigned long gma = 0; |
1813 | const struct cmd_info *info; |
1814 | u32 cmd_len = 0; |
1815 | bool bb_end = false; |
1816 | struct intel_vgpu *vgpu = s->vgpu; |
1817 | u32 cmd; |
1818 | struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ? |
1819 | s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm; |
1820 | |
1821 | *bb_size = 0; |
1822 | *bb_end_cmd_offset = 0; |
1823 | |
1824 | /* get the start gm address of the batch buffer */ |
1825 | gma = get_gma_bb_from_cmd(s, index: 1); |
1826 | if (gma == INTEL_GVT_INVALID_ADDR) |
1827 | return -EFAULT; |
1828 | |
1829 | cmd = cmd_val(s, index: 0); |
1830 | info = get_cmd_info(gvt: s->vgpu->gvt, cmd, engine: s->engine); |
1831 | if (info == NULL) { |
1832 | gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n" , |
1833 | cmd, get_opcode(cmd, s->engine), |
1834 | repr_addr_type(s->buf_addr_type), |
1835 | s->engine->name, s->workload); |
1836 | return -EBADRQC; |
1837 | } |
1838 | do { |
1839 | if (copy_gma_to_hva(vgpu: s->vgpu, mm, |
1840 | gma, end_gma: gma + 4, va: &cmd) < 0) |
1841 | return -EFAULT; |
1842 | info = get_cmd_info(gvt: s->vgpu->gvt, cmd, engine: s->engine); |
1843 | if (info == NULL) { |
1844 | gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n" , |
1845 | cmd, get_opcode(cmd, s->engine), |
1846 | repr_addr_type(s->buf_addr_type), |
1847 | s->engine->name, s->workload); |
1848 | return -EBADRQC; |
1849 | } |
1850 | |
1851 | if (info->opcode == OP_MI_BATCH_BUFFER_END) { |
1852 | bb_end = true; |
1853 | } else if (info->opcode == OP_MI_BATCH_BUFFER_START) { |
1854 | if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) |
1855 | /* chained batch buffer */ |
1856 | bb_end = true; |
1857 | } |
1858 | |
1859 | if (bb_end) |
1860 | *bb_end_cmd_offset = *bb_size; |
1861 | |
1862 | cmd_len = get_cmd_length(info, cmd) << 2; |
1863 | *bb_size += cmd_len; |
1864 | gma += cmd_len; |
1865 | } while (!bb_end); |
1866 | |
1867 | return 0; |
1868 | } |
1869 | |
1870 | static int audit_bb_end(struct parser_exec_state *s, void *va) |
1871 | { |
1872 | struct intel_vgpu *vgpu = s->vgpu; |
1873 | u32 cmd = *(u32 *)va; |
1874 | const struct cmd_info *info; |
1875 | |
1876 | info = get_cmd_info(gvt: s->vgpu->gvt, cmd, engine: s->engine); |
1877 | if (info == NULL) { |
1878 | gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n" , |
1879 | cmd, get_opcode(cmd, s->engine), |
1880 | repr_addr_type(s->buf_addr_type), |
1881 | s->engine->name, s->workload); |
1882 | return -EBADRQC; |
1883 | } |
1884 | |
1885 | if ((info->opcode == OP_MI_BATCH_BUFFER_END) || |
1886 | ((info->opcode == OP_MI_BATCH_BUFFER_START) && |
1887 | (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0))) |
1888 | return 0; |
1889 | |
1890 | return -EBADRQC; |
1891 | } |
1892 | |
1893 | static int perform_bb_shadow(struct parser_exec_state *s) |
1894 | { |
1895 | struct intel_vgpu *vgpu = s->vgpu; |
1896 | struct intel_vgpu_shadow_bb *bb; |
1897 | unsigned long gma = 0; |
1898 | unsigned long bb_size; |
1899 | unsigned long bb_end_cmd_offset; |
1900 | int ret = 0; |
1901 | struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ? |
1902 | s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm; |
1903 | unsigned long start_offset = 0; |
1904 | |
1905 | /* get the start gm address of the batch buffer */ |
1906 | gma = get_gma_bb_from_cmd(s, index: 1); |
1907 | if (gma == INTEL_GVT_INVALID_ADDR) |
1908 | return -EFAULT; |
1909 | |
1910 | ret = find_bb_size(s, bb_size: &bb_size, bb_end_cmd_offset: &bb_end_cmd_offset); |
1911 | if (ret) |
1912 | return ret; |
1913 | |
1914 | bb = kzalloc(size: sizeof(*bb), GFP_KERNEL); |
1915 | if (!bb) |
1916 | return -ENOMEM; |
1917 | |
1918 | bb->ppgtt = (s->buf_addr_type == GTT_BUFFER) ? false : true; |
1919 | |
1920 | /* the start_offset stores the batch buffer's start gma's |
1921 | * offset relative to page boundary. so for non-privileged batch |
1922 | * buffer, the shadowed gem object holds exactly the same page |
1923 | * layout as original gem object. This is for the convience of |
1924 | * replacing the whole non-privilged batch buffer page to this |
1925 | * shadowed one in PPGTT at the same gma address. (this replacing |
1926 | * action is not implemented yet now, but may be necessary in |
1927 | * future). |
1928 | * for prileged batch buffer, we just change start gma address to |
1929 | * that of shadowed page. |
1930 | */ |
1931 | if (bb->ppgtt) |
1932 | start_offset = gma & ~I915_GTT_PAGE_MASK; |
1933 | |
1934 | bb->obj = i915_gem_object_create_shmem(i915: s->engine->i915, |
1935 | round_up(bb_size + start_offset, |
1936 | PAGE_SIZE)); |
1937 | if (IS_ERR(ptr: bb->obj)) { |
1938 | ret = PTR_ERR(ptr: bb->obj); |
1939 | goto err_free_bb; |
1940 | } |
1941 | |
1942 | bb->va = i915_gem_object_pin_map(obj: bb->obj, type: I915_MAP_WB); |
1943 | if (IS_ERR(ptr: bb->va)) { |
1944 | ret = PTR_ERR(ptr: bb->va); |
1945 | goto err_free_obj; |
1946 | } |
1947 | |
1948 | ret = copy_gma_to_hva(vgpu: s->vgpu, mm, |
1949 | gma, end_gma: gma + bb_size, |
1950 | va: bb->va + start_offset); |
1951 | if (ret < 0) { |
1952 | gvt_vgpu_err("fail to copy guest ring buffer\n" ); |
1953 | ret = -EFAULT; |
1954 | goto err_unmap; |
1955 | } |
1956 | |
1957 | ret = audit_bb_end(s, va: bb->va + start_offset + bb_end_cmd_offset); |
1958 | if (ret) |
1959 | goto err_unmap; |
1960 | |
1961 | i915_gem_object_unlock(obj: bb->obj); |
1962 | INIT_LIST_HEAD(list: &bb->list); |
1963 | list_add(new: &bb->list, head: &s->workload->shadow_bb); |
1964 | |
1965 | bb->bb_start_cmd_va = s->ip_va; |
1966 | |
1967 | if ((s->buf_type == BATCH_BUFFER_INSTRUCTION) && (!s->is_ctx_wa)) |
1968 | bb->bb_offset = s->ip_va - s->rb_va; |
1969 | else |
1970 | bb->bb_offset = 0; |
1971 | |
1972 | /* |
1973 | * ip_va saves the virtual address of the shadow batch buffer, while |
1974 | * ip_gma saves the graphics address of the original batch buffer. |
1975 | * As the shadow batch buffer is just a copy from the originial one, |
1976 | * it should be right to use shadow batch buffer'va and original batch |
1977 | * buffer's gma in pair. After all, we don't want to pin the shadow |
1978 | * buffer here (too early). |
1979 | */ |
1980 | s->ip_va = bb->va + start_offset; |
1981 | s->ip_gma = gma; |
1982 | return 0; |
1983 | err_unmap: |
1984 | i915_gem_object_unpin_map(obj: bb->obj); |
1985 | err_free_obj: |
1986 | i915_gem_object_put(obj: bb->obj); |
1987 | err_free_bb: |
1988 | kfree(objp: bb); |
1989 | return ret; |
1990 | } |
1991 | |
1992 | static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s) |
1993 | { |
1994 | bool second_level; |
1995 | int ret = 0; |
1996 | struct intel_vgpu *vgpu = s->vgpu; |
1997 | |
1998 | if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { |
1999 | gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n" ); |
2000 | return -EFAULT; |
2001 | } |
2002 | |
2003 | second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1; |
2004 | if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) { |
2005 | gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n" ); |
2006 | return -EFAULT; |
2007 | } |
2008 | |
2009 | s->saved_buf_addr_type = s->buf_addr_type; |
2010 | addr_type_update_snb(s); |
2011 | if (s->buf_type == RING_BUFFER_INSTRUCTION) { |
2012 | s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32); |
2013 | s->buf_type = BATCH_BUFFER_INSTRUCTION; |
2014 | } else if (second_level) { |
2015 | s->buf_type = BATCH_BUFFER_2ND_LEVEL; |
2016 | s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32); |
2017 | s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32); |
2018 | } |
2019 | |
2020 | if (batch_buffer_needs_scan(s)) { |
2021 | ret = perform_bb_shadow(s); |
2022 | if (ret < 0) |
2023 | gvt_vgpu_err("invalid shadow batch buffer\n" ); |
2024 | } else { |
2025 | /* emulate a batch buffer end to do return right */ |
2026 | ret = cmd_handler_mi_batch_buffer_end(s); |
2027 | if (ret < 0) |
2028 | return ret; |
2029 | } |
2030 | return ret; |
2031 | } |
2032 | |
2033 | static int mi_noop_index; |
2034 | |
2035 | static const struct cmd_info cmd_info[] = { |
2036 | {"MI_NOOP" , OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, |
2037 | |
2038 | {"MI_SET_PREDICATE" , OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL, |
2039 | 0, 1, NULL}, |
2040 | |
2041 | {"MI_USER_INTERRUPT" , OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL, |
2042 | 0, 1, cmd_handler_mi_user_interrupt}, |
2043 | |
2044 | {"MI_WAIT_FOR_EVENT" , OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS, |
2045 | D_ALL, 0, 1, cmd_handler_mi_wait_for_event}, |
2046 | |
2047 | {"MI_FLUSH" , OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, |
2048 | |
2049 | {"MI_ARB_CHECK" , OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1, |
2050 | NULL}, |
2051 | |
2052 | {"MI_RS_CONTROL" , OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1, |
2053 | NULL}, |
2054 | |
2055 | {"MI_REPORT_HEAD" , OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1, |
2056 | NULL}, |
2057 | |
2058 | {"MI_ARB_ON_OFF" , OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1, |
2059 | NULL}, |
2060 | |
2061 | {"MI_URB_ATOMIC_ALLOC" , OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS, |
2062 | D_ALL, 0, 1, NULL}, |
2063 | |
2064 | {"MI_BATCH_BUFFER_END" , OP_MI_BATCH_BUFFER_END, |
2065 | F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1, |
2066 | cmd_handler_mi_batch_buffer_end}, |
2067 | |
2068 | {"MI_SUSPEND_FLUSH" , OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL, |
2069 | 0, 1, NULL}, |
2070 | |
2071 | {"MI_PREDICATE" , OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1, |
2072 | NULL}, |
2073 | |
2074 | {"MI_TOPOLOGY_FILTER" , OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL, |
2075 | D_ALL, 0, 1, NULL}, |
2076 | |
2077 | {"MI_SET_APPID" , OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1, |
2078 | NULL}, |
2079 | |
2080 | {"MI_RS_CONTEXT" , OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1, |
2081 | NULL}, |
2082 | |
2083 | {"MI_DISPLAY_FLIP" , OP_MI_DISPLAY_FLIP, F_LEN_VAR, |
2084 | R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip}, |
2085 | |
2086 | {"MI_SEMAPHORE_MBOX" , OP_MI_SEMAPHORE_MBOX, F_LEN_VAR | F_LEN_VAR_FIXED, |
2087 | R_ALL, D_ALL, 0, 8, NULL, CMD_LEN(1)}, |
2088 | |
2089 | {"MI_MATH" , OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL}, |
2090 | |
2091 | {"MI_URB_CLEAR" , OP_MI_URB_CLEAR, F_LEN_VAR | F_LEN_VAR_FIXED, R_RCS, |
2092 | D_ALL, 0, 8, NULL, CMD_LEN(0)}, |
2093 | |
2094 | {"MI_SEMAPHORE_SIGNAL" , OP_MI_SEMAPHORE_SIGNAL, |
2095 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, 0, 8, |
2096 | NULL, CMD_LEN(0)}, |
2097 | |
2098 | {"MI_SEMAPHORE_WAIT" , OP_MI_SEMAPHORE_WAIT, |
2099 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, ADDR_FIX_1(2), |
2100 | 8, cmd_handler_mi_semaphore_wait, CMD_LEN(2)}, |
2101 | |
2102 | {"MI_STORE_DATA_IMM" , OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS, |
2103 | ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm}, |
2104 | |
2105 | {"MI_STORE_DATA_INDEX" , OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL, |
2106 | 0, 8, cmd_handler_mi_store_data_index}, |
2107 | |
2108 | {"MI_LOAD_REGISTER_IMM" , OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL, |
2109 | D_ALL, 0, 8, cmd_handler_lri}, |
2110 | |
2111 | {"MI_UPDATE_GTT" , OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10, |
2112 | cmd_handler_mi_update_gtt}, |
2113 | |
2114 | {"MI_STORE_REGISTER_MEM" , OP_MI_STORE_REGISTER_MEM, |
2115 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8, |
2116 | cmd_handler_srm, CMD_LEN(2)}, |
2117 | |
2118 | {"MI_FLUSH_DW" , OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6, |
2119 | cmd_handler_mi_flush_dw}, |
2120 | |
2121 | {"MI_CLFLUSH" , OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1), |
2122 | 10, cmd_handler_mi_clflush}, |
2123 | |
2124 | {"MI_REPORT_PERF_COUNT" , OP_MI_REPORT_PERF_COUNT, |
2125 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(1), 6, |
2126 | cmd_handler_mi_report_perf_count, CMD_LEN(2)}, |
2127 | |
2128 | {"MI_LOAD_REGISTER_MEM" , OP_MI_LOAD_REGISTER_MEM, |
2129 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8, |
2130 | cmd_handler_lrm, CMD_LEN(2)}, |
2131 | |
2132 | {"MI_LOAD_REGISTER_REG" , OP_MI_LOAD_REGISTER_REG, |
2133 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, 0, 8, |
2134 | cmd_handler_lrr, CMD_LEN(1)}, |
2135 | |
2136 | {"MI_RS_STORE_DATA_IMM" , OP_MI_RS_STORE_DATA_IMM, |
2137 | F_LEN_VAR | F_LEN_VAR_FIXED, R_RCS, D_ALL, 0, |
2138 | 8, NULL, CMD_LEN(2)}, |
2139 | |
2140 | {"MI_LOAD_URB_MEM" , OP_MI_LOAD_URB_MEM, F_LEN_VAR | F_LEN_VAR_FIXED, |
2141 | R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL, CMD_LEN(2)}, |
2142 | |
2143 | {"MI_STORE_URM_MEM" , OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL, |
2144 | ADDR_FIX_1(2), 8, NULL}, |
2145 | |
2146 | {"MI_OP_2E" , OP_MI_2E, F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, |
2147 | ADDR_FIX_2(1, 2), 8, cmd_handler_mi_op_2e, CMD_LEN(3)}, |
2148 | |
2149 | {"MI_OP_2F" , OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1), |
2150 | 8, cmd_handler_mi_op_2f}, |
2151 | |
2152 | {"MI_BATCH_BUFFER_START" , OP_MI_BATCH_BUFFER_START, |
2153 | F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8, |
2154 | cmd_handler_mi_batch_buffer_start}, |
2155 | |
2156 | {"MI_CONDITIONAL_BATCH_BUFFER_END" , OP_MI_CONDITIONAL_BATCH_BUFFER_END, |
2157 | F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8, |
2158 | cmd_handler_mi_conditional_batch_buffer_end, CMD_LEN(2)}, |
2159 | |
2160 | {"MI_LOAD_SCAN_LINES_INCL" , OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST, |
2161 | R_RCS | R_BCS, D_ALL, 0, 2, NULL}, |
2162 | |
2163 | {"XY_SETUP_BLT" , OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2164 | ADDR_FIX_2(4, 7), 8, NULL}, |
2165 | |
2166 | {"XY_SETUP_CLIP_BLT" , OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2167 | 0, 8, NULL}, |
2168 | |
2169 | {"XY_SETUP_MONO_PATTERN_SL_BLT" , OP_XY_SETUP_MONO_PATTERN_SL_BLT, |
2170 | F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, |
2171 | |
2172 | {"XY_PIXEL_BLT" , OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL}, |
2173 | |
2174 | {"XY_SCANLINES_BLT" , OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2175 | 0, 8, NULL}, |
2176 | |
2177 | {"XY_TEXT_BLT" , OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2178 | ADDR_FIX_1(3), 8, NULL}, |
2179 | |
2180 | {"XY_TEXT_IMMEDIATE_BLT" , OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS, |
2181 | D_ALL, 0, 8, NULL}, |
2182 | |
2183 | {"XY_COLOR_BLT" , OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2184 | ADDR_FIX_1(4), 8, NULL}, |
2185 | |
2186 | {"XY_PAT_BLT" , OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2187 | ADDR_FIX_2(4, 5), 8, NULL}, |
2188 | |
2189 | {"XY_MONO_PAT_BLT" , OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2190 | ADDR_FIX_1(4), 8, NULL}, |
2191 | |
2192 | {"XY_SRC_COPY_BLT" , OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2193 | ADDR_FIX_2(4, 7), 8, NULL}, |
2194 | |
2195 | {"XY_MONO_SRC_COPY_BLT" , OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS, |
2196 | D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, |
2197 | |
2198 | {"XY_FULL_BLT" , OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL}, |
2199 | |
2200 | {"XY_FULL_MONO_SRC_BLT" , OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS, |
2201 | D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL}, |
2202 | |
2203 | {"XY_FULL_MONO_PATTERN_BLT" , OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR, |
2204 | R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, |
2205 | |
2206 | {"XY_FULL_MONO_PATTERN_MONO_SRC_BLT" , |
2207 | OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT, |
2208 | F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, |
2209 | |
2210 | {"XY_MONO_PAT_FIXED_BLT" , OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS, |
2211 | D_ALL, ADDR_FIX_1(4), 8, NULL}, |
2212 | |
2213 | {"XY_MONO_SRC_COPY_IMMEDIATE_BLT" , OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT, |
2214 | F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, |
2215 | |
2216 | {"XY_PAT_BLT_IMMEDIATE" , OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS, |
2217 | D_ALL, ADDR_FIX_1(4), 8, NULL}, |
2218 | |
2219 | {"XY_SRC_COPY_CHROMA_BLT" , OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS, |
2220 | D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, |
2221 | |
2222 | {"XY_FULL_IMMEDIATE_PATTERN_BLT" , OP_XY_FULL_IMMEDIATE_PATTERN_BLT, |
2223 | F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, |
2224 | |
2225 | {"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT" , |
2226 | OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT, |
2227 | F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, |
2228 | |
2229 | {"XY_PAT_CHROMA_BLT" , OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL, |
2230 | ADDR_FIX_2(4, 5), 8, NULL}, |
2231 | |
2232 | {"XY_PAT_CHROMA_BLT_IMMEDIATE" , OP_XY_PAT_CHROMA_BLT_IMMEDIATE, |
2233 | F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, |
2234 | |
2235 | {"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP" , |
2236 | OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, |
2237 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2238 | |
2239 | {"3DSTATE_VIEWPORT_STATE_POINTERS_CC" , |
2240 | OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC, |
2241 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2242 | |
2243 | {"3DSTATE_BLEND_STATE_POINTERS" , |
2244 | OP_3DSTATE_BLEND_STATE_POINTERS, |
2245 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2246 | |
2247 | {"3DSTATE_DEPTH_STENCIL_STATE_POINTERS" , |
2248 | OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS, |
2249 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2250 | |
2251 | {"3DSTATE_BINDING_TABLE_POINTERS_VS" , |
2252 | OP_3DSTATE_BINDING_TABLE_POINTERS_VS, |
2253 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2254 | |
2255 | {"3DSTATE_BINDING_TABLE_POINTERS_HS" , |
2256 | OP_3DSTATE_BINDING_TABLE_POINTERS_HS, |
2257 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2258 | |
2259 | {"3DSTATE_BINDING_TABLE_POINTERS_DS" , |
2260 | OP_3DSTATE_BINDING_TABLE_POINTERS_DS, |
2261 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2262 | |
2263 | {"3DSTATE_BINDING_TABLE_POINTERS_GS" , |
2264 | OP_3DSTATE_BINDING_TABLE_POINTERS_GS, |
2265 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2266 | |
2267 | {"3DSTATE_BINDING_TABLE_POINTERS_PS" , |
2268 | OP_3DSTATE_BINDING_TABLE_POINTERS_PS, |
2269 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2270 | |
2271 | {"3DSTATE_SAMPLER_STATE_POINTERS_VS" , |
2272 | OP_3DSTATE_SAMPLER_STATE_POINTERS_VS, |
2273 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2274 | |
2275 | {"3DSTATE_SAMPLER_STATE_POINTERS_HS" , |
2276 | OP_3DSTATE_SAMPLER_STATE_POINTERS_HS, |
2277 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2278 | |
2279 | {"3DSTATE_SAMPLER_STATE_POINTERS_DS" , |
2280 | OP_3DSTATE_SAMPLER_STATE_POINTERS_DS, |
2281 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2282 | |
2283 | {"3DSTATE_SAMPLER_STATE_POINTERS_GS" , |
2284 | OP_3DSTATE_SAMPLER_STATE_POINTERS_GS, |
2285 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2286 | |
2287 | {"3DSTATE_SAMPLER_STATE_POINTERS_PS" , |
2288 | OP_3DSTATE_SAMPLER_STATE_POINTERS_PS, |
2289 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2290 | |
2291 | {"3DSTATE_URB_VS" , OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL, |
2292 | 0, 8, NULL}, |
2293 | |
2294 | {"3DSTATE_URB_HS" , OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL, |
2295 | 0, 8, NULL}, |
2296 | |
2297 | {"3DSTATE_URB_DS" , OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL, |
2298 | 0, 8, NULL}, |
2299 | |
2300 | {"3DSTATE_URB_GS" , OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL, |
2301 | 0, 8, NULL}, |
2302 | |
2303 | {"3DSTATE_GATHER_CONSTANT_VS" , OP_3DSTATE_GATHER_CONSTANT_VS, |
2304 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2305 | |
2306 | {"3DSTATE_GATHER_CONSTANT_GS" , OP_3DSTATE_GATHER_CONSTANT_GS, |
2307 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2308 | |
2309 | {"3DSTATE_GATHER_CONSTANT_HS" , OP_3DSTATE_GATHER_CONSTANT_HS, |
2310 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2311 | |
2312 | {"3DSTATE_GATHER_CONSTANT_DS" , OP_3DSTATE_GATHER_CONSTANT_DS, |
2313 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2314 | |
2315 | {"3DSTATE_GATHER_CONSTANT_PS" , OP_3DSTATE_GATHER_CONSTANT_PS, |
2316 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2317 | |
2318 | {"3DSTATE_DX9_CONSTANTF_VS" , OP_3DSTATE_DX9_CONSTANTF_VS, |
2319 | F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL}, |
2320 | |
2321 | {"3DSTATE_DX9_CONSTANTF_PS" , OP_3DSTATE_DX9_CONSTANTF_PS, |
2322 | F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL}, |
2323 | |
2324 | {"3DSTATE_DX9_CONSTANTI_VS" , OP_3DSTATE_DX9_CONSTANTI_VS, |
2325 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2326 | |
2327 | {"3DSTATE_DX9_CONSTANTI_PS" , OP_3DSTATE_DX9_CONSTANTI_PS, |
2328 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2329 | |
2330 | {"3DSTATE_DX9_CONSTANTB_VS" , OP_3DSTATE_DX9_CONSTANTB_VS, |
2331 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2332 | |
2333 | {"3DSTATE_DX9_CONSTANTB_PS" , OP_3DSTATE_DX9_CONSTANTB_PS, |
2334 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2335 | |
2336 | {"3DSTATE_DX9_LOCAL_VALID_VS" , OP_3DSTATE_DX9_LOCAL_VALID_VS, |
2337 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2338 | |
2339 | {"3DSTATE_DX9_LOCAL_VALID_PS" , OP_3DSTATE_DX9_LOCAL_VALID_PS, |
2340 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2341 | |
2342 | {"3DSTATE_DX9_GENERATE_ACTIVE_VS" , OP_3DSTATE_DX9_GENERATE_ACTIVE_VS, |
2343 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2344 | |
2345 | {"3DSTATE_DX9_GENERATE_ACTIVE_PS" , OP_3DSTATE_DX9_GENERATE_ACTIVE_PS, |
2346 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2347 | |
2348 | {"3DSTATE_BINDING_TABLE_EDIT_VS" , OP_3DSTATE_BINDING_TABLE_EDIT_VS, |
2349 | F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, |
2350 | |
2351 | {"3DSTATE_BINDING_TABLE_EDIT_GS" , OP_3DSTATE_BINDING_TABLE_EDIT_GS, |
2352 | F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, |
2353 | |
2354 | {"3DSTATE_BINDING_TABLE_EDIT_HS" , OP_3DSTATE_BINDING_TABLE_EDIT_HS, |
2355 | F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, |
2356 | |
2357 | {"3DSTATE_BINDING_TABLE_EDIT_DS" , OP_3DSTATE_BINDING_TABLE_EDIT_DS, |
2358 | F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, |
2359 | |
2360 | {"3DSTATE_BINDING_TABLE_EDIT_PS" , OP_3DSTATE_BINDING_TABLE_EDIT_PS, |
2361 | F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, |
2362 | |
2363 | {"3DSTATE_VF_INSTANCING" , OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS, |
2364 | D_BDW_PLUS, 0, 8, NULL}, |
2365 | |
2366 | {"3DSTATE_VF_SGVS" , OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, |
2367 | NULL}, |
2368 | |
2369 | {"3DSTATE_VF_TOPOLOGY" , OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS, |
2370 | D_BDW_PLUS, 0, 8, NULL}, |
2371 | |
2372 | {"3DSTATE_WM_CHROMAKEY" , OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS, |
2373 | D_BDW_PLUS, 0, 8, NULL}, |
2374 | |
2375 | {"3DSTATE_PS_BLEND" , OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, |
2376 | 8, NULL}, |
2377 | |
2378 | {"3DSTATE_WM_DEPTH_STENCIL" , OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR, |
2379 | R_RCS, D_BDW_PLUS, 0, 8, NULL}, |
2380 | |
2381 | {"3DSTATE_PS_EXTRA" , OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, |
2382 | 8, NULL}, |
2383 | |
2384 | {"3DSTATE_RASTER" , OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, |
2385 | NULL}, |
2386 | |
2387 | {"3DSTATE_SBE_SWIZ" , OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, |
2388 | NULL}, |
2389 | |
2390 | {"3DSTATE_WM_HZ_OP" , OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, |
2391 | NULL}, |
2392 | |
2393 | {"3DSTATE_VERTEX_BUFFERS" , OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS, |
2394 | D_BDW_PLUS, 0, 8, NULL}, |
2395 | |
2396 | {"3DSTATE_VERTEX_ELEMENTS" , OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR, |
2397 | R_RCS, D_ALL, 0, 8, NULL}, |
2398 | |
2399 | {"3DSTATE_INDEX_BUFFER" , OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS, |
2400 | D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL}, |
2401 | |
2402 | {"3DSTATE_VF_STATISTICS" , OP_3DSTATE_VF_STATISTICS, F_LEN_CONST, |
2403 | R_RCS, D_ALL, 0, 1, NULL}, |
2404 | |
2405 | {"3DSTATE_VF" , OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2406 | |
2407 | {"3DSTATE_CC_STATE_POINTERS" , OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR, |
2408 | R_RCS, D_ALL, 0, 8, NULL}, |
2409 | |
2410 | {"3DSTATE_SCISSOR_STATE_POINTERS" , OP_3DSTATE_SCISSOR_STATE_POINTERS, |
2411 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2412 | |
2413 | {"3DSTATE_GS" , OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2414 | |
2415 | {"3DSTATE_CLIP" , OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2416 | |
2417 | {"3DSTATE_WM" , OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2418 | |
2419 | {"3DSTATE_CONSTANT_GS" , OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS, |
2420 | D_BDW_PLUS, 0, 8, NULL}, |
2421 | |
2422 | {"3DSTATE_CONSTANT_PS" , OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS, |
2423 | D_BDW_PLUS, 0, 8, NULL}, |
2424 | |
2425 | {"3DSTATE_SAMPLE_MASK" , OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS, |
2426 | D_ALL, 0, 8, NULL}, |
2427 | |
2428 | {"3DSTATE_CONSTANT_HS" , OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS, |
2429 | D_BDW_PLUS, 0, 8, NULL}, |
2430 | |
2431 | {"3DSTATE_CONSTANT_DS" , OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS, |
2432 | D_BDW_PLUS, 0, 8, NULL}, |
2433 | |
2434 | {"3DSTATE_HS" , OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2435 | |
2436 | {"3DSTATE_TE" , OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2437 | |
2438 | {"3DSTATE_DS" , OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2439 | |
2440 | {"3DSTATE_STREAMOUT" , OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS, |
2441 | D_ALL, 0, 8, NULL}, |
2442 | |
2443 | {"3DSTATE_SBE" , OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2444 | |
2445 | {"3DSTATE_PS" , OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2446 | |
2447 | {"3DSTATE_DRAWING_RECTANGLE" , OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR, |
2448 | R_RCS, D_ALL, 0, 8, NULL}, |
2449 | |
2450 | {"3DSTATE_SAMPLER_PALETTE_LOAD0" , OP_3DSTATE_SAMPLER_PALETTE_LOAD0, |
2451 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2452 | |
2453 | {"3DSTATE_CHROMA_KEY" , OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL, |
2454 | 0, 8, NULL}, |
2455 | |
2456 | {"3DSTATE_DEPTH_BUFFER" , OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS, |
2457 | D_ALL, ADDR_FIX_1(2), 8, NULL}, |
2458 | |
2459 | {"3DSTATE_POLY_STIPPLE_OFFSET" , OP_3DSTATE_POLY_STIPPLE_OFFSET, |
2460 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2461 | |
2462 | {"3DSTATE_POLY_STIPPLE_PATTERN" , OP_3DSTATE_POLY_STIPPLE_PATTERN, |
2463 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2464 | |
2465 | {"3DSTATE_LINE_STIPPLE" , OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS, |
2466 | D_ALL, 0, 8, NULL}, |
2467 | |
2468 | {"3DSTATE_AA_LINE_PARAMS" , OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS, |
2469 | D_ALL, 0, 8, NULL}, |
2470 | |
2471 | {"3DSTATE_GS_SVB_INDEX" , OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS, |
2472 | D_ALL, 0, 8, NULL}, |
2473 | |
2474 | {"3DSTATE_SAMPLER_PALETTE_LOAD1" , OP_3DSTATE_SAMPLER_PALETTE_LOAD1, |
2475 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2476 | |
2477 | {"3DSTATE_MULTISAMPLE" , OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS, |
2478 | D_BDW_PLUS, 0, 8, NULL}, |
2479 | |
2480 | {"3DSTATE_STENCIL_BUFFER" , OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS, |
2481 | D_ALL, ADDR_FIX_1(2), 8, NULL}, |
2482 | |
2483 | {"3DSTATE_HIER_DEPTH_BUFFER" , OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR, |
2484 | R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL}, |
2485 | |
2486 | {"3DSTATE_CLEAR_PARAMS" , OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR, |
2487 | R_RCS, D_ALL, 0, 8, NULL}, |
2488 | |
2489 | {"3DSTATE_PUSH_CONSTANT_ALLOC_VS" , OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS, |
2490 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2491 | |
2492 | {"3DSTATE_PUSH_CONSTANT_ALLOC_HS" , OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS, |
2493 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2494 | |
2495 | {"3DSTATE_PUSH_CONSTANT_ALLOC_DS" , OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS, |
2496 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2497 | |
2498 | {"3DSTATE_PUSH_CONSTANT_ALLOC_GS" , OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS, |
2499 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2500 | |
2501 | {"3DSTATE_PUSH_CONSTANT_ALLOC_PS" , OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS, |
2502 | F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2503 | |
2504 | {"3DSTATE_MONOFILTER_SIZE" , OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR, |
2505 | R_RCS, D_ALL, 0, 8, NULL}, |
2506 | |
2507 | {"3DSTATE_SO_DECL_LIST" , OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS, |
2508 | D_ALL, 0, 9, NULL}, |
2509 | |
2510 | {"3DSTATE_SO_BUFFER" , OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS, |
2511 | ADDR_FIX_2(2, 4), 8, NULL}, |
2512 | |
2513 | {"3DSTATE_BINDING_TABLE_POOL_ALLOC" , |
2514 | OP_3DSTATE_BINDING_TABLE_POOL_ALLOC, |
2515 | F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, |
2516 | |
2517 | {"3DSTATE_GATHER_POOL_ALLOC" , OP_3DSTATE_GATHER_POOL_ALLOC, |
2518 | F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, |
2519 | |
2520 | {"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC" , |
2521 | OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC, |
2522 | F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, |
2523 | |
2524 | {"3DSTATE_SAMPLE_PATTERN" , OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS, |
2525 | D_BDW_PLUS, 0, 8, NULL}, |
2526 | |
2527 | {"PIPE_CONTROL" , OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL, |
2528 | ADDR_FIX_1(2), 8, cmd_handler_pipe_control}, |
2529 | |
2530 | {"3DPRIMITIVE" , OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2531 | |
2532 | {"PIPELINE_SELECT" , OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0, |
2533 | 1, NULL}, |
2534 | |
2535 | {"STATE_PREFETCH" , OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL, |
2536 | ADDR_FIX_1(1), 8, NULL}, |
2537 | |
2538 | {"STATE_SIP" , OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2539 | |
2540 | {"STATE_BASE_ADDRESS" , OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS, |
2541 | ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL}, |
2542 | |
2543 | {"OP_3D_MEDIA_0_1_4" , OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL, |
2544 | ADDR_FIX_1(1), 8, NULL}, |
2545 | |
2546 | {"OP_SWTESS_BASE_ADDRESS" , OP_SWTESS_BASE_ADDRESS, |
2547 | F_LEN_VAR, R_RCS, D_ALL, ADDR_FIX_2(1, 2), 3, NULL}, |
2548 | |
2549 | {"3DSTATE_VS" , OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2550 | |
2551 | {"3DSTATE_SF" , OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, |
2552 | |
2553 | {"3DSTATE_CONSTANT_VS" , OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS, |
2554 | 0, 8, NULL}, |
2555 | |
2556 | {"3DSTATE_COMPONENT_PACKING" , OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS, |
2557 | D_SKL_PLUS, 0, 8, NULL}, |
2558 | |
2559 | {"MEDIA_INTERFACE_DESCRIPTOR_LOAD" , OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD, |
2560 | F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL}, |
2561 | |
2562 | {"MEDIA_GATEWAY_STATE" , OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL, |
2563 | 0, 16, NULL}, |
2564 | |
2565 | {"MEDIA_STATE_FLUSH" , OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL, |
2566 | 0, 16, NULL}, |
2567 | |
2568 | {"MEDIA_POOL_STATE" , OP_MEDIA_POOL_STATE, F_LEN_VAR, R_RCS, D_ALL, |
2569 | 0, 16, NULL}, |
2570 | |
2571 | {"MEDIA_OBJECT" , OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL}, |
2572 | |
2573 | {"MEDIA_CURBE_LOAD" , OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL, |
2574 | 0, 16, NULL}, |
2575 | |
2576 | {"MEDIA_OBJECT_PRT" , OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL, |
2577 | 0, 16, NULL}, |
2578 | |
2579 | {"MEDIA_OBJECT_WALKER" , OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL, |
2580 | 0, 16, NULL}, |
2581 | |
2582 | {"GPGPU_WALKER" , OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL, |
2583 | 0, 8, NULL}, |
2584 | |
2585 | {"MEDIA_VFE_STATE" , OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16, |
2586 | NULL}, |
2587 | |
2588 | {"3DSTATE_VF_STATISTICS_GM45" , OP_3DSTATE_VF_STATISTICS_GM45, |
2589 | F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, |
2590 | |
2591 | {"MFX_PIPE_MODE_SELECT" , OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR, |
2592 | R_VCS, D_ALL, 0, 12, NULL}, |
2593 | |
2594 | {"MFX_SURFACE_STATE" , OP_MFX_SURFACE_STATE, F_LEN_VAR, |
2595 | R_VCS, D_ALL, 0, 12, NULL}, |
2596 | |
2597 | {"MFX_PIPE_BUF_ADDR_STATE" , OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR, |
2598 | R_VCS, D_BDW_PLUS, 0, 12, NULL}, |
2599 | |
2600 | {"MFX_IND_OBJ_BASE_ADDR_STATE" , OP_MFX_IND_OBJ_BASE_ADDR_STATE, |
2601 | F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL}, |
2602 | |
2603 | {"MFX_BSP_BUF_BASE_ADDR_STATE" , OP_MFX_BSP_BUF_BASE_ADDR_STATE, |
2604 | F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL}, |
2605 | |
2606 | {"OP_2_0_0_5" , OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL}, |
2607 | |
2608 | {"MFX_STATE_POINTER" , OP_MFX_STATE_POINTER, F_LEN_VAR, |
2609 | R_VCS, D_ALL, 0, 12, NULL}, |
2610 | |
2611 | {"MFX_QM_STATE" , OP_MFX_QM_STATE, F_LEN_VAR, |
2612 | R_VCS, D_ALL, 0, 12, NULL}, |
2613 | |
2614 | {"MFX_FQM_STATE" , OP_MFX_FQM_STATE, F_LEN_VAR, |
2615 | R_VCS, D_ALL, 0, 12, NULL}, |
2616 | |
2617 | {"MFX_PAK_INSERT_OBJECT" , OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR, |
2618 | R_VCS, D_ALL, 0, 12, NULL}, |
2619 | |
2620 | {"MFX_STITCH_OBJECT" , OP_MFX_STITCH_OBJECT, F_LEN_VAR, |
2621 | R_VCS, D_ALL, 0, 12, NULL}, |
2622 | |
2623 | {"MFD_IT_OBJECT" , OP_MFD_IT_OBJECT, F_LEN_VAR, |
2624 | R_VCS, D_ALL, 0, 12, NULL}, |
2625 | |
2626 | {"MFX_WAIT" , OP_MFX_WAIT, F_LEN_VAR, |
2627 | R_VCS, D_ALL, 0, 6, NULL}, |
2628 | |
2629 | {"MFX_AVC_IMG_STATE" , OP_MFX_AVC_IMG_STATE, F_LEN_VAR, |
2630 | R_VCS, D_ALL, 0, 12, NULL}, |
2631 | |
2632 | {"MFX_AVC_QM_STATE" , OP_MFX_AVC_QM_STATE, F_LEN_VAR, |
2633 | R_VCS, D_ALL, 0, 12, NULL}, |
2634 | |
2635 | {"MFX_AVC_DIRECTMODE_STATE" , OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR, |
2636 | R_VCS, D_ALL, 0, 12, NULL}, |
2637 | |
2638 | {"MFX_AVC_SLICE_STATE" , OP_MFX_AVC_SLICE_STATE, F_LEN_VAR, |
2639 | R_VCS, D_ALL, 0, 12, NULL}, |
2640 | |
2641 | {"MFX_AVC_REF_IDX_STATE" , OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR, |
2642 | R_VCS, D_ALL, 0, 12, NULL}, |
2643 | |
2644 | {"MFX_AVC_WEIGHTOFFSET_STATE" , OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR, |
2645 | R_VCS, D_ALL, 0, 12, NULL}, |
2646 | |
2647 | {"MFD_AVC_PICID_STATE" , OP_MFD_AVC_PICID_STATE, F_LEN_VAR, |
2648 | R_VCS, D_ALL, 0, 12, NULL}, |
2649 | {"MFD_AVC_DPB_STATE" , OP_MFD_AVC_DPB_STATE, F_LEN_VAR, |
2650 | R_VCS, D_ALL, 0, 12, NULL}, |
2651 | |
2652 | {"MFD_AVC_BSD_OBJECT" , OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR, |
2653 | R_VCS, D_ALL, 0, 12, NULL}, |
2654 | |
2655 | {"MFD_AVC_SLICEADDR" , OP_MFD_AVC_SLICEADDR, F_LEN_VAR, |
2656 | R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL}, |
2657 | |
2658 | {"MFC_AVC_PAK_OBJECT" , OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR, |
2659 | R_VCS, D_ALL, 0, 12, NULL}, |
2660 | |
2661 | {"MFX_VC1_PRED_PIPE_STATE" , OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR, |
2662 | R_VCS, D_ALL, 0, 12, NULL}, |
2663 | |
2664 | {"MFX_VC1_DIRECTMODE_STATE" , OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR, |
2665 | R_VCS, D_ALL, 0, 12, NULL}, |
2666 | |
2667 | {"MFD_VC1_SHORT_PIC_STATE" , OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR, |
2668 | R_VCS, D_ALL, 0, 12, NULL}, |
2669 | |
2670 | {"MFD_VC1_LONG_PIC_STATE" , OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR, |
2671 | R_VCS, D_ALL, 0, 12, NULL}, |
2672 | |
2673 | {"MFD_VC1_BSD_OBJECT" , OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR, |
2674 | R_VCS, D_ALL, 0, 12, NULL}, |
2675 | |
2676 | {"MFC_MPEG2_SLICEGROUP_STATE" , OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR, |
2677 | R_VCS, D_ALL, 0, 12, NULL}, |
2678 | |
2679 | {"MFC_MPEG2_PAK_OBJECT" , OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR, |
2680 | R_VCS, D_ALL, 0, 12, NULL}, |
2681 | |
2682 | {"MFX_MPEG2_PIC_STATE" , OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR, |
2683 | R_VCS, D_ALL, 0, 12, NULL}, |
2684 | |
2685 | {"MFX_MPEG2_QM_STATE" , OP_MFX_MPEG2_QM_STATE, F_LEN_VAR, |
2686 | R_VCS, D_ALL, 0, 12, NULL}, |
2687 | |
2688 | {"MFD_MPEG2_BSD_OBJECT" , OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR, |
2689 | R_VCS, D_ALL, 0, 12, NULL}, |
2690 | |
2691 | {"MFX_2_6_0_0" , OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL, |
2692 | 0, 16, NULL}, |
2693 | |
2694 | {"MFX_2_6_0_9" , OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL}, |
2695 | |
2696 | {"MFX_2_6_0_8" , OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL}, |
2697 | |
2698 | {"MFX_JPEG_PIC_STATE" , OP_MFX_JPEG_PIC_STATE, F_LEN_VAR, |
2699 | R_VCS, D_ALL, 0, 12, NULL}, |
2700 | |
2701 | {"MFX_JPEG_HUFF_TABLE_STATE" , OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR, |
2702 | R_VCS, D_ALL, 0, 12, NULL}, |
2703 | |
2704 | {"MFD_JPEG_BSD_OBJECT" , OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR, |
2705 | R_VCS, D_ALL, 0, 12, NULL}, |
2706 | |
2707 | {"VEBOX_STATE" , OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL}, |
2708 | |
2709 | {"VEBOX_SURFACE_STATE" , OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL, |
2710 | 0, 12, NULL}, |
2711 | |
2712 | {"VEB_DI_IECP" , OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS, |
2713 | 0, 12, NULL}, |
2714 | }; |
2715 | |
2716 | static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e) |
2717 | { |
2718 | hash_add(gvt->cmd_table, &e->hlist, e->info->opcode); |
2719 | } |
2720 | |
2721 | /* call the cmd handler, and advance ip */ |
2722 | static int cmd_parser_exec(struct parser_exec_state *s) |
2723 | { |
2724 | struct intel_vgpu *vgpu = s->vgpu; |
2725 | const struct cmd_info *info; |
2726 | u32 cmd; |
2727 | int ret = 0; |
2728 | |
2729 | cmd = cmd_val(s, index: 0); |
2730 | |
2731 | /* fastpath for MI_NOOP */ |
2732 | if (cmd == MI_NOOP) |
2733 | info = &cmd_info[mi_noop_index]; |
2734 | else |
2735 | info = get_cmd_info(gvt: s->vgpu->gvt, cmd, engine: s->engine); |
2736 | |
2737 | if (info == NULL) { |
2738 | gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %s, workload=%p\n" , |
2739 | cmd, get_opcode(cmd, s->engine), |
2740 | repr_addr_type(s->buf_addr_type), |
2741 | s->engine->name, s->workload); |
2742 | return -EBADRQC; |
2743 | } |
2744 | |
2745 | s->info = info; |
2746 | |
2747 | trace_gvt_command(vgpu_id: vgpu->id, ring_id: s->engine->id, ip_gma: s->ip_gma, cmd_va: s->ip_va, |
2748 | cmd_len: cmd_length(s), buf_type: s->buf_type, buf_addr_type: s->buf_addr_type, |
2749 | workload: s->workload, cmd_name: info->name); |
2750 | |
2751 | if ((info->flag & F_LEN_MASK) == F_LEN_VAR_FIXED) { |
2752 | ret = gvt_check_valid_cmd_length(len: cmd_length(s), |
2753 | valid_len: info->valid_len); |
2754 | if (ret) |
2755 | return ret; |
2756 | } |
2757 | |
2758 | if (info->handler) { |
2759 | ret = info->handler(s); |
2760 | if (ret < 0) { |
2761 | gvt_vgpu_err("%s handler error\n" , info->name); |
2762 | return ret; |
2763 | } |
2764 | } |
2765 | |
2766 | if (!(info->flag & F_IP_ADVANCE_CUSTOM)) { |
2767 | ret = cmd_advance_default(s); |
2768 | if (ret) { |
2769 | gvt_vgpu_err("%s IP advance error\n" , info->name); |
2770 | return ret; |
2771 | } |
2772 | } |
2773 | return 0; |
2774 | } |
2775 | |
2776 | static inline bool gma_out_of_range(unsigned long gma, |
2777 | unsigned long gma_head, unsigned int gma_tail) |
2778 | { |
2779 | if (gma_tail >= gma_head) |
2780 | return (gma < gma_head) || (gma > gma_tail); |
2781 | else |
2782 | return (gma > gma_tail) && (gma < gma_head); |
2783 | } |
2784 | |
2785 | /* Keep the consistent return type, e.g EBADRQC for unknown |
2786 | * cmd, EFAULT for invalid address, EPERM for nonpriv. later |
2787 | * works as the input of VM healthy status. |
2788 | */ |
2789 | static int command_scan(struct parser_exec_state *s, |
2790 | unsigned long rb_head, unsigned long rb_tail, |
2791 | unsigned long rb_start, unsigned long rb_len) |
2792 | { |
2793 | |
2794 | unsigned long gma_head, gma_tail, gma_bottom; |
2795 | int ret = 0; |
2796 | struct intel_vgpu *vgpu = s->vgpu; |
2797 | |
2798 | gma_head = rb_start + rb_head; |
2799 | gma_tail = rb_start + rb_tail; |
2800 | gma_bottom = rb_start + rb_len; |
2801 | |
2802 | while (s->ip_gma != gma_tail) { |
2803 | if (s->buf_type == RING_BUFFER_INSTRUCTION || |
2804 | s->buf_type == RING_BUFFER_CTX) { |
2805 | if (!(s->ip_gma >= rb_start) || |
2806 | !(s->ip_gma < gma_bottom)) { |
2807 | gvt_vgpu_err("ip_gma %lx out of ring scope." |
2808 | "(base:0x%lx, bottom: 0x%lx)\n" , |
2809 | s->ip_gma, rb_start, |
2810 | gma_bottom); |
2811 | parser_exec_state_dump(s); |
2812 | return -EFAULT; |
2813 | } |
2814 | if (gma_out_of_range(gma: s->ip_gma, gma_head, gma_tail)) { |
2815 | gvt_vgpu_err("ip_gma %lx out of range." |
2816 | "base 0x%lx head 0x%lx tail 0x%lx\n" , |
2817 | s->ip_gma, rb_start, |
2818 | rb_head, rb_tail); |
2819 | parser_exec_state_dump(s); |
2820 | break; |
2821 | } |
2822 | } |
2823 | ret = cmd_parser_exec(s); |
2824 | if (ret) { |
2825 | gvt_vgpu_err("cmd parser error\n" ); |
2826 | parser_exec_state_dump(s); |
2827 | break; |
2828 | } |
2829 | } |
2830 | |
2831 | return ret; |
2832 | } |
2833 | |
2834 | static int scan_workload(struct intel_vgpu_workload *workload) |
2835 | { |
2836 | unsigned long gma_head, gma_tail; |
2837 | struct parser_exec_state s; |
2838 | int ret = 0; |
2839 | |
2840 | /* ring base is page aligned */ |
2841 | if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE))) |
2842 | return -EINVAL; |
2843 | |
2844 | gma_head = workload->rb_start + workload->rb_head; |
2845 | gma_tail = workload->rb_start + workload->rb_tail; |
2846 | |
2847 | s.buf_type = RING_BUFFER_INSTRUCTION; |
2848 | s.buf_addr_type = GTT_BUFFER; |
2849 | s.vgpu = workload->vgpu; |
2850 | s.engine = workload->engine; |
2851 | s.ring_start = workload->rb_start; |
2852 | s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl); |
2853 | s.ring_head = gma_head; |
2854 | s.ring_tail = gma_tail; |
2855 | s.rb_va = workload->shadow_ring_buffer_va; |
2856 | s.workload = workload; |
2857 | s.is_ctx_wa = false; |
2858 | |
2859 | if (bypass_scan_mask & workload->engine->mask || gma_head == gma_tail) |
2860 | return 0; |
2861 | |
2862 | ret = ip_gma_set(s: &s, ip_gma: gma_head); |
2863 | if (ret) |
2864 | goto out; |
2865 | |
2866 | ret = command_scan(s: &s, rb_head: workload->rb_head, rb_tail: workload->rb_tail, |
2867 | rb_start: workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl)); |
2868 | |
2869 | out: |
2870 | return ret; |
2871 | } |
2872 | |
2873 | static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
2874 | { |
2875 | |
2876 | unsigned long gma_head, gma_tail, ring_size, ring_tail; |
2877 | struct parser_exec_state s; |
2878 | int ret = 0; |
2879 | struct intel_vgpu_workload *workload = container_of(wa_ctx, |
2880 | struct intel_vgpu_workload, |
2881 | wa_ctx); |
2882 | |
2883 | /* ring base is page aligned */ |
2884 | if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, |
2885 | I915_GTT_PAGE_SIZE))) |
2886 | return -EINVAL; |
2887 | |
2888 | ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(u32); |
2889 | ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES, |
2890 | PAGE_SIZE); |
2891 | gma_head = wa_ctx->indirect_ctx.guest_gma; |
2892 | gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail; |
2893 | |
2894 | s.buf_type = RING_BUFFER_INSTRUCTION; |
2895 | s.buf_addr_type = GTT_BUFFER; |
2896 | s.vgpu = workload->vgpu; |
2897 | s.engine = workload->engine; |
2898 | s.ring_start = wa_ctx->indirect_ctx.guest_gma; |
2899 | s.ring_size = ring_size; |
2900 | s.ring_head = gma_head; |
2901 | s.ring_tail = gma_tail; |
2902 | s.rb_va = wa_ctx->indirect_ctx.shadow_va; |
2903 | s.workload = workload; |
2904 | s.is_ctx_wa = true; |
2905 | |
2906 | ret = ip_gma_set(s: &s, ip_gma: gma_head); |
2907 | if (ret) |
2908 | goto out; |
2909 | |
2910 | ret = command_scan(s: &s, rb_head: 0, rb_tail: ring_tail, |
2911 | rb_start: wa_ctx->indirect_ctx.guest_gma, rb_len: ring_size); |
2912 | out: |
2913 | return ret; |
2914 | } |
2915 | |
2916 | static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload) |
2917 | { |
2918 | struct intel_vgpu *vgpu = workload->vgpu; |
2919 | struct intel_vgpu_submission *s = &vgpu->submission; |
2920 | unsigned long gma_head, gma_tail, gma_top, guest_rb_size; |
2921 | void *shadow_ring_buffer_va; |
2922 | int ret; |
2923 | |
2924 | guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl); |
2925 | |
2926 | /* calculate workload ring buffer size */ |
2927 | workload->rb_len = (workload->rb_tail + guest_rb_size - |
2928 | workload->rb_head) % guest_rb_size; |
2929 | |
2930 | gma_head = workload->rb_start + workload->rb_head; |
2931 | gma_tail = workload->rb_start + workload->rb_tail; |
2932 | gma_top = workload->rb_start + guest_rb_size; |
2933 | |
2934 | if (workload->rb_len > s->ring_scan_buffer_size[workload->engine->id]) { |
2935 | void *p; |
2936 | |
2937 | /* realloc the new ring buffer if needed */ |
2938 | p = krealloc(objp: s->ring_scan_buffer[workload->engine->id], |
2939 | new_size: workload->rb_len, GFP_KERNEL); |
2940 | if (!p) { |
2941 | gvt_vgpu_err("fail to re-alloc ring scan buffer\n" ); |
2942 | return -ENOMEM; |
2943 | } |
2944 | s->ring_scan_buffer[workload->engine->id] = p; |
2945 | s->ring_scan_buffer_size[workload->engine->id] = workload->rb_len; |
2946 | } |
2947 | |
2948 | shadow_ring_buffer_va = s->ring_scan_buffer[workload->engine->id]; |
2949 | |
2950 | /* get shadow ring buffer va */ |
2951 | workload->shadow_ring_buffer_va = shadow_ring_buffer_va; |
2952 | |
2953 | /* head > tail --> copy head <-> top */ |
2954 | if (gma_head > gma_tail) { |
2955 | ret = copy_gma_to_hva(vgpu, mm: vgpu->gtt.ggtt_mm, |
2956 | gma: gma_head, end_gma: gma_top, va: shadow_ring_buffer_va); |
2957 | if (ret < 0) { |
2958 | gvt_vgpu_err("fail to copy guest ring buffer\n" ); |
2959 | return ret; |
2960 | } |
2961 | shadow_ring_buffer_va += ret; |
2962 | gma_head = workload->rb_start; |
2963 | } |
2964 | |
2965 | /* copy head or start <-> tail */ |
2966 | ret = copy_gma_to_hva(vgpu, mm: vgpu->gtt.ggtt_mm, gma: gma_head, end_gma: gma_tail, |
2967 | va: shadow_ring_buffer_va); |
2968 | if (ret < 0) { |
2969 | gvt_vgpu_err("fail to copy guest ring buffer\n" ); |
2970 | return ret; |
2971 | } |
2972 | return 0; |
2973 | } |
2974 | |
2975 | int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload) |
2976 | { |
2977 | int ret; |
2978 | struct intel_vgpu *vgpu = workload->vgpu; |
2979 | |
2980 | ret = shadow_workload_ring_buffer(workload); |
2981 | if (ret) { |
2982 | gvt_vgpu_err("fail to shadow workload ring_buffer\n" ); |
2983 | return ret; |
2984 | } |
2985 | |
2986 | ret = scan_workload(workload); |
2987 | if (ret) { |
2988 | gvt_vgpu_err("scan workload error\n" ); |
2989 | return ret; |
2990 | } |
2991 | return 0; |
2992 | } |
2993 | |
2994 | static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
2995 | { |
2996 | int ctx_size = wa_ctx->indirect_ctx.size; |
2997 | unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma; |
2998 | struct intel_vgpu_workload *workload = container_of(wa_ctx, |
2999 | struct intel_vgpu_workload, |
3000 | wa_ctx); |
3001 | struct intel_vgpu *vgpu = workload->vgpu; |
3002 | struct drm_i915_gem_object *obj; |
3003 | int ret = 0; |
3004 | void *map; |
3005 | |
3006 | obj = i915_gem_object_create_shmem(i915: workload->engine->i915, |
3007 | roundup(ctx_size + CACHELINE_BYTES, |
3008 | PAGE_SIZE)); |
3009 | if (IS_ERR(ptr: obj)) |
3010 | return PTR_ERR(ptr: obj); |
3011 | |
3012 | /* get the va of the shadow batch buffer */ |
3013 | map = i915_gem_object_pin_map(obj, type: I915_MAP_WB); |
3014 | if (IS_ERR(ptr: map)) { |
3015 | gvt_vgpu_err("failed to vmap shadow indirect ctx\n" ); |
3016 | ret = PTR_ERR(ptr: map); |
3017 | goto put_obj; |
3018 | } |
3019 | |
3020 | i915_gem_object_lock(obj, NULL); |
3021 | ret = i915_gem_object_set_to_cpu_domain(obj, write: false); |
3022 | i915_gem_object_unlock(obj); |
3023 | if (ret) { |
3024 | gvt_vgpu_err("failed to set shadow indirect ctx to CPU\n" ); |
3025 | goto unmap_src; |
3026 | } |
3027 | |
3028 | ret = copy_gma_to_hva(vgpu: workload->vgpu, |
3029 | mm: workload->vgpu->gtt.ggtt_mm, |
3030 | gma: guest_gma, end_gma: guest_gma + ctx_size, |
3031 | va: map); |
3032 | if (ret < 0) { |
3033 | gvt_vgpu_err("fail to copy guest indirect ctx\n" ); |
3034 | goto unmap_src; |
3035 | } |
3036 | |
3037 | wa_ctx->indirect_ctx.obj = obj; |
3038 | wa_ctx->indirect_ctx.shadow_va = map; |
3039 | return 0; |
3040 | |
3041 | unmap_src: |
3042 | i915_gem_object_unpin_map(obj); |
3043 | put_obj: |
3044 | i915_gem_object_put(obj); |
3045 | return ret; |
3046 | } |
3047 | |
3048 | static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
3049 | { |
3050 | u32 per_ctx_start[CACHELINE_DWORDS] = {}; |
3051 | unsigned char *bb_start_sva; |
3052 | |
3053 | if (!wa_ctx->per_ctx.valid) |
3054 | return 0; |
3055 | |
3056 | per_ctx_start[0] = 0x18800001; |
3057 | per_ctx_start[1] = wa_ctx->per_ctx.guest_gma; |
3058 | |
3059 | bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va + |
3060 | wa_ctx->indirect_ctx.size; |
3061 | |
3062 | memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES); |
3063 | |
3064 | return 0; |
3065 | } |
3066 | |
3067 | int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
3068 | { |
3069 | int ret; |
3070 | struct intel_vgpu_workload *workload = container_of(wa_ctx, |
3071 | struct intel_vgpu_workload, |
3072 | wa_ctx); |
3073 | struct intel_vgpu *vgpu = workload->vgpu; |
3074 | |
3075 | if (wa_ctx->indirect_ctx.size == 0) |
3076 | return 0; |
3077 | |
3078 | ret = shadow_indirect_ctx(wa_ctx); |
3079 | if (ret) { |
3080 | gvt_vgpu_err("fail to shadow indirect ctx\n" ); |
3081 | return ret; |
3082 | } |
3083 | |
3084 | combine_wa_ctx(wa_ctx); |
3085 | |
3086 | ret = scan_wa_ctx(wa_ctx); |
3087 | if (ret) { |
3088 | gvt_vgpu_err("scan wa ctx error\n" ); |
3089 | return ret; |
3090 | } |
3091 | |
3092 | return 0; |
3093 | } |
3094 | |
3095 | /* generate dummy contexts by sending empty requests to HW, and let |
3096 | * the HW to fill Engine Contexts. This dummy contexts are used for |
3097 | * initialization purpose (update reg whitelist), so referred to as |
3098 | * init context here |
3099 | */ |
3100 | void intel_gvt_update_reg_whitelist(struct intel_vgpu *vgpu) |
3101 | { |
3102 | const unsigned long start = LRC_STATE_PN * PAGE_SIZE; |
3103 | struct intel_gvt *gvt = vgpu->gvt; |
3104 | struct intel_engine_cs *engine; |
3105 | enum intel_engine_id id; |
3106 | |
3107 | if (gvt->is_reg_whitelist_updated) |
3108 | return; |
3109 | |
3110 | /* scan init ctx to update cmd accessible list */ |
3111 | for_each_engine(engine, gvt->gt, id) { |
3112 | struct parser_exec_state s; |
3113 | void *vaddr; |
3114 | int ret; |
3115 | |
3116 | if (!engine->default_state) |
3117 | continue; |
3118 | |
3119 | vaddr = shmem_pin_map(file: engine->default_state); |
3120 | if (!vaddr) { |
3121 | gvt_err("failed to map %s->default state\n" , |
3122 | engine->name); |
3123 | return; |
3124 | } |
3125 | |
3126 | s.buf_type = RING_BUFFER_CTX; |
3127 | s.buf_addr_type = GTT_BUFFER; |
3128 | s.vgpu = vgpu; |
3129 | s.engine = engine; |
3130 | s.ring_start = 0; |
3131 | s.ring_size = engine->context_size - start; |
3132 | s.ring_head = 0; |
3133 | s.ring_tail = s.ring_size; |
3134 | s.rb_va = vaddr + start; |
3135 | s.workload = NULL; |
3136 | s.is_ctx_wa = false; |
3137 | s.is_init_ctx = true; |
3138 | |
3139 | /* skipping the first RING_CTX_SIZE(0x50) dwords */ |
3140 | ret = ip_gma_set(s: &s, RING_CTX_SIZE); |
3141 | if (ret == 0) { |
3142 | ret = command_scan(s: &s, rb_head: 0, rb_tail: s.ring_size, rb_start: 0, rb_len: s.ring_size); |
3143 | if (ret) |
3144 | gvt_err("Scan init ctx error\n" ); |
3145 | } |
3146 | |
3147 | shmem_unpin_map(file: engine->default_state, ptr: vaddr); |
3148 | if (ret) |
3149 | return; |
3150 | } |
3151 | |
3152 | gvt->is_reg_whitelist_updated = true; |
3153 | } |
3154 | |
3155 | int intel_gvt_scan_engine_context(struct intel_vgpu_workload *workload) |
3156 | { |
3157 | struct intel_vgpu *vgpu = workload->vgpu; |
3158 | unsigned long gma_head, gma_tail, gma_start, ctx_size; |
3159 | struct parser_exec_state s; |
3160 | int ring_id = workload->engine->id; |
3161 | struct intel_context *ce = vgpu->submission.shadow[ring_id]; |
3162 | int ret; |
3163 | |
3164 | GEM_BUG_ON(atomic_read(&ce->pin_count) < 0); |
3165 | |
3166 | ctx_size = workload->engine->context_size - PAGE_SIZE; |
3167 | |
3168 | /* Only ring contxt is loaded to HW for inhibit context, no need to |
3169 | * scan engine context |
3170 | */ |
3171 | if (is_inhibit_context(ce)) |
3172 | return 0; |
3173 | |
3174 | gma_start = i915_ggtt_offset(vma: ce->state) + LRC_STATE_PN*PAGE_SIZE; |
3175 | gma_head = 0; |
3176 | gma_tail = ctx_size; |
3177 | |
3178 | s.buf_type = RING_BUFFER_CTX; |
3179 | s.buf_addr_type = GTT_BUFFER; |
3180 | s.vgpu = workload->vgpu; |
3181 | s.engine = workload->engine; |
3182 | s.ring_start = gma_start; |
3183 | s.ring_size = ctx_size; |
3184 | s.ring_head = gma_start + gma_head; |
3185 | s.ring_tail = gma_start + gma_tail; |
3186 | s.rb_va = ce->lrc_reg_state; |
3187 | s.workload = workload; |
3188 | s.is_ctx_wa = false; |
3189 | s.is_init_ctx = false; |
3190 | |
3191 | /* don't scan the first RING_CTX_SIZE(0x50) dwords, as it's ring |
3192 | * context |
3193 | */ |
3194 | ret = ip_gma_set(s: &s, ip_gma: gma_start + gma_head + RING_CTX_SIZE); |
3195 | if (ret) |
3196 | goto out; |
3197 | |
3198 | ret = command_scan(s: &s, rb_head: gma_head, rb_tail: gma_tail, |
3199 | rb_start: gma_start, rb_len: ctx_size); |
3200 | out: |
3201 | if (ret) |
3202 | gvt_vgpu_err("scan shadow ctx error\n" ); |
3203 | |
3204 | return ret; |
3205 | } |
3206 | |
3207 | static int init_cmd_table(struct intel_gvt *gvt) |
3208 | { |
3209 | unsigned int gen_type = intel_gvt_get_device_type(gvt); |
3210 | int i; |
3211 | |
3212 | for (i = 0; i < ARRAY_SIZE(cmd_info); i++) { |
3213 | struct cmd_entry *e; |
3214 | |
3215 | if (!(cmd_info[i].devices & gen_type)) |
3216 | continue; |
3217 | |
3218 | e = kzalloc(size: sizeof(*e), GFP_KERNEL); |
3219 | if (!e) |
3220 | return -ENOMEM; |
3221 | |
3222 | e->info = &cmd_info[i]; |
3223 | if (cmd_info[i].opcode == OP_MI_NOOP) |
3224 | mi_noop_index = i; |
3225 | |
3226 | INIT_HLIST_NODE(h: &e->hlist); |
3227 | add_cmd_entry(gvt, e); |
3228 | gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n" , |
3229 | e->info->name, e->info->opcode, e->info->flag, |
3230 | e->info->devices, e->info->rings); |
3231 | } |
3232 | |
3233 | return 0; |
3234 | } |
3235 | |
3236 | static void clean_cmd_table(struct intel_gvt *gvt) |
3237 | { |
3238 | struct hlist_node *tmp; |
3239 | struct cmd_entry *e; |
3240 | int i; |
3241 | |
3242 | hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist) |
3243 | kfree(objp: e); |
3244 | |
3245 | hash_init(gvt->cmd_table); |
3246 | } |
3247 | |
3248 | void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt) |
3249 | { |
3250 | clean_cmd_table(gvt); |
3251 | } |
3252 | |
3253 | int intel_gvt_init_cmd_parser(struct intel_gvt *gvt) |
3254 | { |
3255 | int ret; |
3256 | |
3257 | ret = init_cmd_table(gvt); |
3258 | if (ret) { |
3259 | intel_gvt_clean_cmd_parser(gvt); |
3260 | return ret; |
3261 | } |
3262 | return 0; |
3263 | } |
3264 | |