1 | /* |
2 | * Copyright © 2013 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Brad Volkin <bradley.d.volkin@intel.com> |
25 | * |
26 | */ |
27 | |
28 | #include <linux/highmem.h> |
29 | |
30 | #include <drm/drm_cache.h> |
31 | |
32 | #include "gt/intel_engine.h" |
33 | #include "gt/intel_engine_regs.h" |
34 | #include "gt/intel_gpu_commands.h" |
35 | #include "gt/intel_gt_regs.h" |
36 | |
37 | #include "i915_cmd_parser.h" |
38 | #include "i915_drv.h" |
39 | #include "i915_memcpy.h" |
40 | #include "i915_reg.h" |
41 | |
42 | /** |
43 | * DOC: batch buffer command parser |
44 | * |
45 | * Motivation: |
46 | * Certain OpenGL features (e.g. transform feedback, performance monitoring) |
47 | * require userspace code to submit batches containing commands such as |
48 | * MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some |
49 | * generations of the hardware will noop these commands in "unsecure" batches |
50 | * (which includes all userspace batches submitted via i915) even though the |
51 | * commands may be safe and represent the intended programming model of the |
52 | * device. |
53 | * |
54 | * The software command parser is similar in operation to the command parsing |
55 | * done in hardware for unsecure batches. However, the software parser allows |
56 | * some operations that would be noop'd by hardware, if the parser determines |
57 | * the operation is safe, and submits the batch as "secure" to prevent hardware |
58 | * parsing. |
59 | * |
60 | * Threats: |
61 | * At a high level, the hardware (and software) checks attempt to prevent |
62 | * granting userspace undue privileges. There are three categories of privilege. |
63 | * |
64 | * First, commands which are explicitly defined as privileged or which should |
65 | * only be used by the kernel driver. The parser rejects such commands |
66 | * |
67 | * Second, commands which access registers. To support correct/enhanced |
68 | * userspace functionality, particularly certain OpenGL extensions, the parser |
69 | * provides a whitelist of registers which userspace may safely access |
70 | * |
71 | * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc). |
72 | * The parser always rejects such commands. |
73 | * |
74 | * The majority of the problematic commands fall in the MI_* range, with only a |
75 | * few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW). |
76 | * |
77 | * Implementation: |
78 | * Each engine maintains tables of commands and registers which the parser |
79 | * uses in scanning batch buffers submitted to that engine. |
80 | * |
81 | * Since the set of commands that the parser must check for is significantly |
82 | * smaller than the number of commands supported, the parser tables contain only |
83 | * those commands required by the parser. This generally works because command |
84 | * opcode ranges have standard command length encodings. So for commands that |
85 | * the parser does not need to check, it can easily skip them. This is |
86 | * implemented via a per-engine length decoding vfunc. |
87 | * |
88 | * Unfortunately, there are a number of commands that do not follow the standard |
89 | * length encoding for their opcode range, primarily amongst the MI_* commands. |
90 | * To handle this, the parser provides a way to define explicit "skip" entries |
91 | * in the per-engine command tables. |
92 | * |
93 | * Other command table entries map fairly directly to high level categories |
94 | * mentioned above: rejected, register whitelist. The parser implements a number |
95 | * of checks, including the privileged memory checks, via a general bitmasking |
96 | * mechanism. |
97 | */ |
98 | |
99 | /* |
100 | * A command that requires special handling by the command parser. |
101 | */ |
102 | struct drm_i915_cmd_descriptor { |
103 | /* |
104 | * Flags describing how the command parser processes the command. |
105 | * |
106 | * CMD_DESC_FIXED: The command has a fixed length if this is set, |
107 | * a length mask if not set |
108 | * CMD_DESC_SKIP: The command is allowed but does not follow the |
109 | * standard length encoding for the opcode range in |
110 | * which it falls |
111 | * CMD_DESC_REJECT: The command is never allowed |
112 | * CMD_DESC_REGISTER: The command should be checked against the |
113 | * register whitelist for the appropriate ring |
114 | */ |
115 | u32 flags; |
116 | #define CMD_DESC_FIXED (1<<0) |
117 | #define CMD_DESC_SKIP (1<<1) |
118 | #define CMD_DESC_REJECT (1<<2) |
119 | #define CMD_DESC_REGISTER (1<<3) |
120 | #define CMD_DESC_BITMASK (1<<4) |
121 | |
122 | /* |
123 | * The command's unique identification bits and the bitmask to get them. |
124 | * This isn't strictly the opcode field as defined in the spec and may |
125 | * also include type, subtype, and/or subop fields. |
126 | */ |
127 | struct { |
128 | u32 value; |
129 | u32 mask; |
130 | } cmd; |
131 | |
132 | /* |
133 | * The command's length. The command is either fixed length (i.e. does |
134 | * not include a length field) or has a length field mask. The flag |
135 | * CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has |
136 | * a length mask. All command entries in a command table must include |
137 | * length information. |
138 | */ |
139 | union { |
140 | u32 fixed; |
141 | u32 mask; |
142 | } length; |
143 | |
144 | /* |
145 | * Describes where to find a register address in the command to check |
146 | * against the ring's register whitelist. Only valid if flags has the |
147 | * CMD_DESC_REGISTER bit set. |
148 | * |
149 | * A non-zero step value implies that the command may access multiple |
150 | * registers in sequence (e.g. LRI), in that case step gives the |
151 | * distance in dwords between individual offset fields. |
152 | */ |
153 | struct { |
154 | u32 offset; |
155 | u32 mask; |
156 | u32 step; |
157 | } reg; |
158 | |
159 | #define MAX_CMD_DESC_BITMASKS 3 |
160 | /* |
161 | * Describes command checks where a particular dword is masked and |
162 | * compared against an expected value. If the command does not match |
163 | * the expected value, the parser rejects it. Only valid if flags has |
164 | * the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero |
165 | * are valid. |
166 | * |
167 | * If the check specifies a non-zero condition_mask then the parser |
168 | * only performs the check when the bits specified by condition_mask |
169 | * are non-zero. |
170 | */ |
171 | struct { |
172 | u32 offset; |
173 | u32 mask; |
174 | u32 expected; |
175 | u32 condition_offset; |
176 | u32 condition_mask; |
177 | } bits[MAX_CMD_DESC_BITMASKS]; |
178 | }; |
179 | |
180 | /* |
181 | * A table of commands requiring special handling by the command parser. |
182 | * |
183 | * Each engine has an array of tables. Each table consists of an array of |
184 | * command descriptors, which must be sorted with command opcodes in |
185 | * ascending order. |
186 | */ |
187 | struct drm_i915_cmd_table { |
188 | const struct drm_i915_cmd_descriptor *table; |
189 | int count; |
190 | }; |
191 | |
192 | #define STD_MI_OPCODE_SHIFT (32 - 9) |
193 | #define STD_3D_OPCODE_SHIFT (32 - 16) |
194 | #define STD_2D_OPCODE_SHIFT (32 - 10) |
195 | #define STD_MFX_OPCODE_SHIFT (32 - 16) |
196 | #define MIN_OPCODE_SHIFT 16 |
197 | |
198 | #define CMD(op, opm, f, lm, fl, ...) \ |
199 | { \ |
200 | .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \ |
201 | .cmd = { (op & ~0u << (opm)), ~0u << (opm) }, \ |
202 | .length = { (lm) }, \ |
203 | __VA_ARGS__ \ |
204 | } |
205 | |
206 | /* Convenience macros to compress the tables */ |
207 | #define SMI STD_MI_OPCODE_SHIFT |
208 | #define S3D STD_3D_OPCODE_SHIFT |
209 | #define S2D STD_2D_OPCODE_SHIFT |
210 | #define SMFX STD_MFX_OPCODE_SHIFT |
211 | #define F true |
212 | #define S CMD_DESC_SKIP |
213 | #define R CMD_DESC_REJECT |
214 | #define W CMD_DESC_REGISTER |
215 | #define B CMD_DESC_BITMASK |
216 | |
217 | /* Command Mask Fixed Len Action |
218 | ---------------------------------------------------------- */ |
219 | static const struct drm_i915_cmd_descriptor gen7_common_cmds[] = { |
220 | CMD( MI_NOOP, SMI, F, 1, S ), |
221 | CMD( MI_USER_INTERRUPT, SMI, F, 1, R ), |
222 | CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, R ), |
223 | CMD( MI_ARB_CHECK, SMI, F, 1, S ), |
224 | CMD( MI_REPORT_HEAD, SMI, F, 1, S ), |
225 | CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ), |
226 | CMD( MI_SEMAPHORE_MBOX, SMI, !F, 0xFF, R ), |
227 | CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ), |
228 | CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W, |
229 | .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ), |
230 | CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B, |
231 | .reg = { .offset = 1, .mask = 0x007FFFFC }, |
232 | .bits = {{ |
233 | .offset = 0, |
234 | .mask = MI_GLOBAL_GTT, |
235 | .expected = 0, |
236 | }}, ), |
237 | CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B, |
238 | .reg = { .offset = 1, .mask = 0x007FFFFC }, |
239 | .bits = {{ |
240 | .offset = 0, |
241 | .mask = MI_GLOBAL_GTT, |
242 | .expected = 0, |
243 | }}, ), |
244 | /* |
245 | * MI_BATCH_BUFFER_START requires some special handling. It's not |
246 | * really a 'skip' action but it doesn't seem like it's worth adding |
247 | * a new action. See intel_engine_cmd_parser(). |
248 | */ |
249 | CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ), |
250 | }; |
251 | |
252 | static const struct drm_i915_cmd_descriptor gen7_render_cmds[] = { |
253 | CMD( MI_FLUSH, SMI, F, 1, S ), |
254 | CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), |
255 | CMD( MI_PREDICATE, SMI, F, 1, S ), |
256 | CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ), |
257 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
258 | CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), |
259 | CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ), |
260 | CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ), |
261 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B, |
262 | .bits = {{ |
263 | .offset = 0, |
264 | .mask = MI_GLOBAL_GTT, |
265 | .expected = 0, |
266 | }}, ), |
267 | CMD( MI_UPDATE_GTT, SMI, !F, 0xFF, R ), |
268 | CMD( MI_CLFLUSH, SMI, !F, 0x3FF, B, |
269 | .bits = {{ |
270 | .offset = 0, |
271 | .mask = MI_GLOBAL_GTT, |
272 | .expected = 0, |
273 | }}, ), |
274 | CMD( MI_REPORT_PERF_COUNT, SMI, !F, 0x3F, B, |
275 | .bits = {{ |
276 | .offset = 1, |
277 | .mask = MI_REPORT_PERF_COUNT_GGTT, |
278 | .expected = 0, |
279 | }}, ), |
280 | CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, |
281 | .bits = {{ |
282 | .offset = 0, |
283 | .mask = MI_GLOBAL_GTT, |
284 | .expected = 0, |
285 | }}, ), |
286 | CMD( GFX_OP_3DSTATE_VF_STATISTICS, S3D, F, 1, S ), |
287 | CMD( PIPELINE_SELECT, S3D, F, 1, S ), |
288 | CMD( MEDIA_VFE_STATE, S3D, !F, 0xFFFF, B, |
289 | .bits = {{ |
290 | .offset = 2, |
291 | .mask = MEDIA_VFE_STATE_MMIO_ACCESS_MASK, |
292 | .expected = 0, |
293 | }}, ), |
294 | CMD( GPGPU_OBJECT, S3D, !F, 0xFF, S ), |
295 | CMD( GPGPU_WALKER, S3D, !F, 0xFF, S ), |
296 | CMD( GFX_OP_3DSTATE_SO_DECL_LIST, S3D, !F, 0x1FF, S ), |
297 | CMD( GFX_OP_PIPE_CONTROL(5), S3D, !F, 0xFF, B, |
298 | .bits = {{ |
299 | .offset = 1, |
300 | .mask = (PIPE_CONTROL_MMIO_WRITE | PIPE_CONTROL_NOTIFY), |
301 | .expected = 0, |
302 | }, |
303 | { |
304 | .offset = 1, |
305 | .mask = (PIPE_CONTROL_GLOBAL_GTT_IVB | |
306 | PIPE_CONTROL_STORE_DATA_INDEX), |
307 | .expected = 0, |
308 | .condition_offset = 1, |
309 | .condition_mask = PIPE_CONTROL_POST_SYNC_OP_MASK, |
310 | }}, ), |
311 | }; |
312 | |
313 | static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = { |
314 | CMD( MI_SET_PREDICATE, SMI, F, 1, S ), |
315 | CMD( MI_RS_CONTROL, SMI, F, 1, S ), |
316 | CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ), |
317 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
318 | CMD( MI_RS_CONTEXT, SMI, F, 1, S ), |
319 | CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ), |
320 | CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), |
321 | CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W, |
322 | .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ), |
323 | CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ), |
324 | CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ), |
325 | CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ), |
326 | CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_VS, S3D, !F, 0x7FF, S ), |
327 | CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_PS, S3D, !F, 0x7FF, S ), |
328 | |
329 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS, S3D, !F, 0x1FF, S ), |
330 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS, S3D, !F, 0x1FF, S ), |
331 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS, S3D, !F, 0x1FF, S ), |
332 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS, S3D, !F, 0x1FF, S ), |
333 | CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ), |
334 | }; |
335 | |
336 | static const struct drm_i915_cmd_descriptor gen7_video_cmds[] = { |
337 | CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), |
338 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
339 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, |
340 | .bits = {{ |
341 | .offset = 0, |
342 | .mask = MI_GLOBAL_GTT, |
343 | .expected = 0, |
344 | }}, ), |
345 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), |
346 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, |
347 | .bits = {{ |
348 | .offset = 0, |
349 | .mask = MI_FLUSH_DW_NOTIFY, |
350 | .expected = 0, |
351 | }, |
352 | { |
353 | .offset = 1, |
354 | .mask = MI_FLUSH_DW_USE_GTT, |
355 | .expected = 0, |
356 | .condition_offset = 0, |
357 | .condition_mask = MI_FLUSH_DW_OP_MASK, |
358 | }, |
359 | { |
360 | .offset = 0, |
361 | .mask = MI_FLUSH_DW_STORE_INDEX, |
362 | .expected = 0, |
363 | .condition_offset = 0, |
364 | .condition_mask = MI_FLUSH_DW_OP_MASK, |
365 | }}, ), |
366 | CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, |
367 | .bits = {{ |
368 | .offset = 0, |
369 | .mask = MI_GLOBAL_GTT, |
370 | .expected = 0, |
371 | }}, ), |
372 | /* |
373 | * MFX_WAIT doesn't fit the way we handle length for most commands. |
374 | * It has a length field but it uses a non-standard length bias. |
375 | * It is always 1 dword though, so just treat it as fixed length. |
376 | */ |
377 | CMD( MFX_WAIT, SMFX, F, 1, S ), |
378 | }; |
379 | |
380 | static const struct drm_i915_cmd_descriptor gen7_vecs_cmds[] = { |
381 | CMD( MI_ARB_ON_OFF, SMI, F, 1, R ), |
382 | CMD( MI_SET_APPID, SMI, F, 1, S ), |
383 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B, |
384 | .bits = {{ |
385 | .offset = 0, |
386 | .mask = MI_GLOBAL_GTT, |
387 | .expected = 0, |
388 | }}, ), |
389 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), |
390 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, |
391 | .bits = {{ |
392 | .offset = 0, |
393 | .mask = MI_FLUSH_DW_NOTIFY, |
394 | .expected = 0, |
395 | }, |
396 | { |
397 | .offset = 1, |
398 | .mask = MI_FLUSH_DW_USE_GTT, |
399 | .expected = 0, |
400 | .condition_offset = 0, |
401 | .condition_mask = MI_FLUSH_DW_OP_MASK, |
402 | }, |
403 | { |
404 | .offset = 0, |
405 | .mask = MI_FLUSH_DW_STORE_INDEX, |
406 | .expected = 0, |
407 | .condition_offset = 0, |
408 | .condition_mask = MI_FLUSH_DW_OP_MASK, |
409 | }}, ), |
410 | CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B, |
411 | .bits = {{ |
412 | .offset = 0, |
413 | .mask = MI_GLOBAL_GTT, |
414 | .expected = 0, |
415 | }}, ), |
416 | }; |
417 | |
418 | static const struct drm_i915_cmd_descriptor gen7_blt_cmds[] = { |
419 | CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ), |
420 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B, |
421 | .bits = {{ |
422 | .offset = 0, |
423 | .mask = MI_GLOBAL_GTT, |
424 | .expected = 0, |
425 | }}, ), |
426 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ), |
427 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B, |
428 | .bits = {{ |
429 | .offset = 0, |
430 | .mask = MI_FLUSH_DW_NOTIFY, |
431 | .expected = 0, |
432 | }, |
433 | { |
434 | .offset = 1, |
435 | .mask = MI_FLUSH_DW_USE_GTT, |
436 | .expected = 0, |
437 | .condition_offset = 0, |
438 | .condition_mask = MI_FLUSH_DW_OP_MASK, |
439 | }, |
440 | { |
441 | .offset = 0, |
442 | .mask = MI_FLUSH_DW_STORE_INDEX, |
443 | .expected = 0, |
444 | .condition_offset = 0, |
445 | .condition_mask = MI_FLUSH_DW_OP_MASK, |
446 | }}, ), |
447 | CMD( COLOR_BLT, S2D, !F, 0x3F, S ), |
448 | CMD( SRC_COPY_BLT, S2D, !F, 0x3F, S ), |
449 | }; |
450 | |
451 | static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = { |
452 | CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ), |
453 | CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), |
454 | }; |
455 | |
456 | /* |
457 | * For Gen9 we can still rely on the h/w to enforce cmd security, and only |
458 | * need to re-enforce the register access checks. We therefore only need to |
459 | * teach the cmdparser how to find the end of each command, and identify |
460 | * register accesses. The table doesn't need to reject any commands, and so |
461 | * the only commands listed here are: |
462 | * 1) Those that touch registers |
463 | * 2) Those that do not have the default 8-bit length |
464 | * |
465 | * Note that the default MI length mask chosen for this table is 0xFF, not |
466 | * the 0x3F used on older devices. This is because the vast majority of MI |
467 | * cmds on Gen9 use a standard 8-bit Length field. |
468 | * All the Gen9 blitter instructions are standard 0xFF length mask, and |
469 | * none allow access to non-general registers, so in fact no BLT cmds are |
470 | * included in the table at all. |
471 | * |
472 | */ |
473 | static const struct drm_i915_cmd_descriptor gen9_blt_cmds[] = { |
474 | CMD( MI_NOOP, SMI, F, 1, S ), |
475 | CMD( MI_USER_INTERRUPT, SMI, F, 1, S ), |
476 | CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, S ), |
477 | CMD( MI_FLUSH, SMI, F, 1, S ), |
478 | CMD( MI_ARB_CHECK, SMI, F, 1, S ), |
479 | CMD( MI_REPORT_HEAD, SMI, F, 1, S ), |
480 | CMD( MI_ARB_ON_OFF, SMI, F, 1, S ), |
481 | CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ), |
482 | CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, S ), |
483 | CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, S ), |
484 | CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, S ), |
485 | CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W, |
486 | .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ), |
487 | CMD( MI_UPDATE_GTT, SMI, !F, 0x3FF, S ), |
488 | CMD( MI_STORE_REGISTER_MEM_GEN8, SMI, F, 4, W, |
489 | .reg = { .offset = 1, .mask = 0x007FFFFC } ), |
490 | CMD( MI_FLUSH_DW, SMI, !F, 0x3F, S ), |
491 | CMD( MI_LOAD_REGISTER_MEM_GEN8, SMI, F, 4, W, |
492 | .reg = { .offset = 1, .mask = 0x007FFFFC } ), |
493 | CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W, |
494 | .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ), |
495 | |
496 | /* |
497 | * We allow BB_START but apply further checks. We just sanitize the |
498 | * basic fields here. |
499 | */ |
500 | #define MI_BB_START_OPERAND_MASK GENMASK(SMI-1, 0) |
501 | #define MI_BB_START_OPERAND_EXPECT (MI_BATCH_PPGTT_HSW | 1) |
502 | CMD( MI_BATCH_BUFFER_START_GEN8, SMI, !F, 0xFF, B, |
503 | .bits = {{ |
504 | .offset = 0, |
505 | .mask = MI_BB_START_OPERAND_MASK, |
506 | .expected = MI_BB_START_OPERAND_EXPECT, |
507 | }}, ), |
508 | }; |
509 | |
510 | static const struct drm_i915_cmd_descriptor noop_desc = |
511 | CMD(MI_NOOP, SMI, F, 1, S); |
512 | |
513 | #undef CMD |
514 | #undef SMI |
515 | #undef S3D |
516 | #undef S2D |
517 | #undef SMFX |
518 | #undef F |
519 | #undef S |
520 | #undef R |
521 | #undef W |
522 | #undef B |
523 | |
524 | static const struct drm_i915_cmd_table gen7_render_cmd_table[] = { |
525 | { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) }, |
526 | { gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) }, |
527 | }; |
528 | |
529 | static const struct drm_i915_cmd_table hsw_render_ring_cmd_table[] = { |
530 | { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) }, |
531 | { gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) }, |
532 | { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) }, |
533 | }; |
534 | |
535 | static const struct drm_i915_cmd_table gen7_video_cmd_table[] = { |
536 | { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) }, |
537 | { gen7_video_cmds, ARRAY_SIZE(gen7_video_cmds) }, |
538 | }; |
539 | |
540 | static const struct drm_i915_cmd_table hsw_vebox_cmd_table[] = { |
541 | { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) }, |
542 | { gen7_vecs_cmds, ARRAY_SIZE(gen7_vecs_cmds) }, |
543 | }; |
544 | |
545 | static const struct drm_i915_cmd_table gen7_blt_cmd_table[] = { |
546 | { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) }, |
547 | { gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) }, |
548 | }; |
549 | |
550 | static const struct drm_i915_cmd_table hsw_blt_ring_cmd_table[] = { |
551 | { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) }, |
552 | { gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) }, |
553 | { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) }, |
554 | }; |
555 | |
556 | static const struct drm_i915_cmd_table gen9_blt_cmd_table[] = { |
557 | { gen9_blt_cmds, ARRAY_SIZE(gen9_blt_cmds) }, |
558 | }; |
559 | |
560 | |
561 | /* |
562 | * Register whitelists, sorted by increasing register offset. |
563 | */ |
564 | |
565 | /* |
566 | * An individual whitelist entry granting access to register addr. If |
567 | * mask is non-zero the argument of immediate register writes will be |
568 | * AND-ed with mask, and the command will be rejected if the result |
569 | * doesn't match value. |
570 | * |
571 | * Registers with non-zero mask are only allowed to be written using |
572 | * LRI. |
573 | */ |
574 | struct drm_i915_reg_descriptor { |
575 | i915_reg_t addr; |
576 | u32 mask; |
577 | u32 value; |
578 | }; |
579 | |
580 | /* Convenience macro for adding 32-bit registers. */ |
581 | #define REG32(_reg, ...) \ |
582 | { .addr = (_reg), __VA_ARGS__ } |
583 | |
584 | #define REG32_IDX(_reg, idx) \ |
585 | { .addr = _reg(idx) } |
586 | |
587 | /* |
588 | * Convenience macro for adding 64-bit registers. |
589 | * |
590 | * Some registers that userspace accesses are 64 bits. The register |
591 | * access commands only allow 32-bit accesses. Hence, we have to include |
592 | * entries for both halves of the 64-bit registers. |
593 | */ |
594 | #define REG64(_reg) \ |
595 | { .addr = _reg }, \ |
596 | { .addr = _reg ## _UDW } |
597 | |
598 | #define REG64_IDX(_reg, idx) \ |
599 | { .addr = _reg(idx) }, \ |
600 | { .addr = _reg ## _UDW(idx) } |
601 | |
602 | #define REG64_BASE_IDX(_reg, base, idx) \ |
603 | { .addr = _reg(base, idx) }, \ |
604 | { .addr = _reg ## _UDW(base, idx) } |
605 | |
606 | static const struct drm_i915_reg_descriptor gen7_render_regs[] = { |
607 | REG64(GPGPU_THREADS_DISPATCHED), |
608 | REG64(HS_INVOCATION_COUNT), |
609 | REG64(DS_INVOCATION_COUNT), |
610 | REG64(IA_VERTICES_COUNT), |
611 | REG64(IA_PRIMITIVES_COUNT), |
612 | REG64(VS_INVOCATION_COUNT), |
613 | REG64(GS_INVOCATION_COUNT), |
614 | REG64(GS_PRIMITIVES_COUNT), |
615 | REG64(CL_INVOCATION_COUNT), |
616 | REG64(CL_PRIMITIVES_COUNT), |
617 | REG64(PS_INVOCATION_COUNT), |
618 | REG64(PS_DEPTH_COUNT), |
619 | REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), |
620 | REG64_IDX(MI_PREDICATE_SRC0, RENDER_RING_BASE), |
621 | REG64_IDX(MI_PREDICATE_SRC1, RENDER_RING_BASE), |
622 | REG32(GEN7_3DPRIM_END_OFFSET), |
623 | REG32(GEN7_3DPRIM_START_VERTEX), |
624 | REG32(GEN7_3DPRIM_VERTEX_COUNT), |
625 | REG32(GEN7_3DPRIM_INSTANCE_COUNT), |
626 | REG32(GEN7_3DPRIM_START_INSTANCE), |
627 | REG32(GEN7_3DPRIM_BASE_VERTEX), |
628 | REG32(GEN7_GPGPU_DISPATCHDIMX), |
629 | REG32(GEN7_GPGPU_DISPATCHDIMY), |
630 | REG32(GEN7_GPGPU_DISPATCHDIMZ), |
631 | REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE), |
632 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0), |
633 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1), |
634 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2), |
635 | REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3), |
636 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0), |
637 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1), |
638 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2), |
639 | REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3), |
640 | REG32(GEN7_SO_WRITE_OFFSET(0)), |
641 | REG32(GEN7_SO_WRITE_OFFSET(1)), |
642 | REG32(GEN7_SO_WRITE_OFFSET(2)), |
643 | REG32(GEN7_SO_WRITE_OFFSET(3)), |
644 | REG32(GEN7_L3SQCREG1), |
645 | REG32(GEN7_L3CNTLREG2), |
646 | REG32(GEN7_L3CNTLREG3), |
647 | REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), |
648 | }; |
649 | |
650 | static const struct drm_i915_reg_descriptor hsw_render_regs[] = { |
651 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 0), |
652 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 1), |
653 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 2), |
654 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 3), |
655 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 4), |
656 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 5), |
657 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 6), |
658 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 7), |
659 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 8), |
660 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 9), |
661 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 10), |
662 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 11), |
663 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 12), |
664 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 13), |
665 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 14), |
666 | REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 15), |
667 | REG32(HSW_SCRATCH1, |
668 | .mask = ~HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE, |
669 | .value = 0), |
670 | REG32(HSW_ROW_CHICKEN3, |
671 | .mask = ~(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE << 16 | |
672 | HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), |
673 | .value = 0), |
674 | }; |
675 | |
676 | static const struct drm_i915_reg_descriptor gen7_blt_regs[] = { |
677 | REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), |
678 | REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE), |
679 | REG32(BCS_SWCTRL), |
680 | REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), |
681 | }; |
682 | |
683 | static const struct drm_i915_reg_descriptor gen9_blt_regs[] = { |
684 | REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE), |
685 | REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE), |
686 | REG32(BCS_SWCTRL), |
687 | REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE), |
688 | REG32_IDX(RING_CTX_TIMESTAMP, BLT_RING_BASE), |
689 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 0), |
690 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 1), |
691 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 2), |
692 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 3), |
693 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 4), |
694 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 5), |
695 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 6), |
696 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 7), |
697 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 8), |
698 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 9), |
699 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 10), |
700 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 11), |
701 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 12), |
702 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 13), |
703 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 14), |
704 | REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 15), |
705 | }; |
706 | |
707 | #undef REG64 |
708 | #undef REG32 |
709 | |
710 | struct drm_i915_reg_table { |
711 | const struct drm_i915_reg_descriptor *regs; |
712 | int num_regs; |
713 | }; |
714 | |
715 | static const struct drm_i915_reg_table ivb_render_reg_tables[] = { |
716 | { gen7_render_regs, ARRAY_SIZE(gen7_render_regs) }, |
717 | }; |
718 | |
719 | static const struct drm_i915_reg_table ivb_blt_reg_tables[] = { |
720 | { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) }, |
721 | }; |
722 | |
723 | static const struct drm_i915_reg_table hsw_render_reg_tables[] = { |
724 | { gen7_render_regs, ARRAY_SIZE(gen7_render_regs) }, |
725 | { hsw_render_regs, ARRAY_SIZE(hsw_render_regs) }, |
726 | }; |
727 | |
728 | static const struct drm_i915_reg_table hsw_blt_reg_tables[] = { |
729 | { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) }, |
730 | }; |
731 | |
732 | static const struct drm_i915_reg_table gen9_blt_reg_tables[] = { |
733 | { gen9_blt_regs, ARRAY_SIZE(gen9_blt_regs) }, |
734 | }; |
735 | |
736 | static u32 gen7_render_get_cmd_length_mask(u32 ) |
737 | { |
738 | u32 client = cmd_header >> INSTR_CLIENT_SHIFT; |
739 | u32 subclient = |
740 | (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; |
741 | |
742 | if (client == INSTR_MI_CLIENT) |
743 | return 0x3F; |
744 | else if (client == INSTR_RC_CLIENT) { |
745 | if (subclient == INSTR_MEDIA_SUBCLIENT) |
746 | return 0xFFFF; |
747 | else |
748 | return 0xFF; |
749 | } |
750 | |
751 | DRM_DEBUG("CMD: Abnormal rcs cmd length! 0x%08X\n" , cmd_header); |
752 | return 0; |
753 | } |
754 | |
755 | static u32 gen7_bsd_get_cmd_length_mask(u32 ) |
756 | { |
757 | u32 client = cmd_header >> INSTR_CLIENT_SHIFT; |
758 | u32 subclient = |
759 | (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT; |
760 | u32 op = (cmd_header & INSTR_26_TO_24_MASK) >> INSTR_26_TO_24_SHIFT; |
761 | |
762 | if (client == INSTR_MI_CLIENT) |
763 | return 0x3F; |
764 | else if (client == INSTR_RC_CLIENT) { |
765 | if (subclient == INSTR_MEDIA_SUBCLIENT) { |
766 | if (op == 6) |
767 | return 0xFFFF; |
768 | else |
769 | return 0xFFF; |
770 | } else |
771 | return 0xFF; |
772 | } |
773 | |
774 | DRM_DEBUG("CMD: Abnormal bsd cmd length! 0x%08X\n" , cmd_header); |
775 | return 0; |
776 | } |
777 | |
778 | static u32 gen7_blt_get_cmd_length_mask(u32 ) |
779 | { |
780 | u32 client = cmd_header >> INSTR_CLIENT_SHIFT; |
781 | |
782 | if (client == INSTR_MI_CLIENT) |
783 | return 0x3F; |
784 | else if (client == INSTR_BC_CLIENT) |
785 | return 0xFF; |
786 | |
787 | DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n" , cmd_header); |
788 | return 0; |
789 | } |
790 | |
791 | static u32 gen9_blt_get_cmd_length_mask(u32 ) |
792 | { |
793 | u32 client = cmd_header >> INSTR_CLIENT_SHIFT; |
794 | |
795 | if (client == INSTR_MI_CLIENT || client == INSTR_BC_CLIENT) |
796 | return 0xFF; |
797 | |
798 | DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n" , cmd_header); |
799 | return 0; |
800 | } |
801 | |
802 | static bool validate_cmds_sorted(const struct intel_engine_cs *engine, |
803 | const struct drm_i915_cmd_table *cmd_tables, |
804 | int cmd_table_count) |
805 | { |
806 | int i; |
807 | bool ret = true; |
808 | |
809 | if (!cmd_tables || cmd_table_count == 0) |
810 | return true; |
811 | |
812 | for (i = 0; i < cmd_table_count; i++) { |
813 | const struct drm_i915_cmd_table *table = &cmd_tables[i]; |
814 | u32 previous = 0; |
815 | int j; |
816 | |
817 | for (j = 0; j < table->count; j++) { |
818 | const struct drm_i915_cmd_descriptor *desc = |
819 | &table->table[j]; |
820 | u32 curr = desc->cmd.value & desc->cmd.mask; |
821 | |
822 | if (curr < previous) { |
823 | drm_err(&engine->i915->drm, |
824 | "CMD: %s [%d] command table not sorted: " |
825 | "table=%d entry=%d cmd=0x%08X prev=0x%08X\n" , |
826 | engine->name, engine->id, |
827 | i, j, curr, previous); |
828 | ret = false; |
829 | } |
830 | |
831 | previous = curr; |
832 | } |
833 | } |
834 | |
835 | return ret; |
836 | } |
837 | |
838 | static bool check_sorted(const struct intel_engine_cs *engine, |
839 | const struct drm_i915_reg_descriptor *reg_table, |
840 | int reg_count) |
841 | { |
842 | int i; |
843 | u32 previous = 0; |
844 | bool ret = true; |
845 | |
846 | for (i = 0; i < reg_count; i++) { |
847 | u32 curr = i915_mmio_reg_offset(reg_table[i].addr); |
848 | |
849 | if (curr < previous) { |
850 | drm_err(&engine->i915->drm, |
851 | "CMD: %s [%d] register table not sorted: " |
852 | "entry=%d reg=0x%08X prev=0x%08X\n" , |
853 | engine->name, engine->id, |
854 | i, curr, previous); |
855 | ret = false; |
856 | } |
857 | |
858 | previous = curr; |
859 | } |
860 | |
861 | return ret; |
862 | } |
863 | |
864 | static bool validate_regs_sorted(struct intel_engine_cs *engine) |
865 | { |
866 | int i; |
867 | const struct drm_i915_reg_table *table; |
868 | |
869 | for (i = 0; i < engine->reg_table_count; i++) { |
870 | table = &engine->reg_tables[i]; |
871 | if (!check_sorted(engine, reg_table: table->regs, reg_count: table->num_regs)) |
872 | return false; |
873 | } |
874 | |
875 | return true; |
876 | } |
877 | |
878 | struct cmd_node { |
879 | const struct drm_i915_cmd_descriptor *desc; |
880 | struct hlist_node node; |
881 | }; |
882 | |
883 | /* |
884 | * Different command ranges have different numbers of bits for the opcode. For |
885 | * example, MI commands use bits 31:23 while 3D commands use bits 31:16. The |
886 | * problem is that, for example, MI commands use bits 22:16 for other fields |
887 | * such as GGTT vs PPGTT bits. If we include those bits in the mask then when |
888 | * we mask a command from a batch it could hash to the wrong bucket due to |
889 | * non-opcode bits being set. But if we don't include those bits, some 3D |
890 | * commands may hash to the same bucket due to not including opcode bits that |
891 | * make the command unique. For now, we will risk hashing to the same bucket. |
892 | */ |
893 | static inline u32 (u32 x) |
894 | { |
895 | switch (x >> INSTR_CLIENT_SHIFT) { |
896 | default: |
897 | case INSTR_MI_CLIENT: |
898 | return x >> STD_MI_OPCODE_SHIFT; |
899 | case INSTR_RC_CLIENT: |
900 | return x >> STD_3D_OPCODE_SHIFT; |
901 | case INSTR_BC_CLIENT: |
902 | return x >> STD_2D_OPCODE_SHIFT; |
903 | } |
904 | } |
905 | |
906 | static int init_hash_table(struct intel_engine_cs *engine, |
907 | const struct drm_i915_cmd_table *cmd_tables, |
908 | int cmd_table_count) |
909 | { |
910 | int i, j; |
911 | |
912 | hash_init(engine->cmd_hash); |
913 | |
914 | for (i = 0; i < cmd_table_count; i++) { |
915 | const struct drm_i915_cmd_table *table = &cmd_tables[i]; |
916 | |
917 | for (j = 0; j < table->count; j++) { |
918 | const struct drm_i915_cmd_descriptor *desc = |
919 | &table->table[j]; |
920 | struct cmd_node *desc_node = |
921 | kmalloc(size: sizeof(*desc_node), GFP_KERNEL); |
922 | |
923 | if (!desc_node) |
924 | return -ENOMEM; |
925 | |
926 | desc_node->desc = desc; |
927 | hash_add(engine->cmd_hash, &desc_node->node, |
928 | cmd_header_key(desc->cmd.value)); |
929 | } |
930 | } |
931 | |
932 | return 0; |
933 | } |
934 | |
935 | static void fini_hash_table(struct intel_engine_cs *engine) |
936 | { |
937 | struct hlist_node *tmp; |
938 | struct cmd_node *desc_node; |
939 | int i; |
940 | |
941 | hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) { |
942 | hash_del(node: &desc_node->node); |
943 | kfree(objp: desc_node); |
944 | } |
945 | } |
946 | |
947 | /** |
948 | * intel_engine_init_cmd_parser() - set cmd parser related fields for an engine |
949 | * @engine: the engine to initialize |
950 | * |
951 | * Optionally initializes fields related to batch buffer command parsing in the |
952 | * struct intel_engine_cs based on whether the platform requires software |
953 | * command parsing. |
954 | */ |
955 | int intel_engine_init_cmd_parser(struct intel_engine_cs *engine) |
956 | { |
957 | const struct drm_i915_cmd_table *cmd_tables; |
958 | int cmd_table_count; |
959 | int ret; |
960 | |
961 | if (GRAPHICS_VER(engine->i915) != 7 && !(GRAPHICS_VER(engine->i915) == 9 && |
962 | engine->class == COPY_ENGINE_CLASS)) |
963 | return 0; |
964 | |
965 | switch (engine->class) { |
966 | case RENDER_CLASS: |
967 | if (IS_HASWELL(engine->i915)) { |
968 | cmd_tables = hsw_render_ring_cmd_table; |
969 | cmd_table_count = |
970 | ARRAY_SIZE(hsw_render_ring_cmd_table); |
971 | } else { |
972 | cmd_tables = gen7_render_cmd_table; |
973 | cmd_table_count = ARRAY_SIZE(gen7_render_cmd_table); |
974 | } |
975 | |
976 | if (IS_HASWELL(engine->i915)) { |
977 | engine->reg_tables = hsw_render_reg_tables; |
978 | engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables); |
979 | } else { |
980 | engine->reg_tables = ivb_render_reg_tables; |
981 | engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables); |
982 | } |
983 | engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask; |
984 | break; |
985 | case VIDEO_DECODE_CLASS: |
986 | cmd_tables = gen7_video_cmd_table; |
987 | cmd_table_count = ARRAY_SIZE(gen7_video_cmd_table); |
988 | engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; |
989 | break; |
990 | case COPY_ENGINE_CLASS: |
991 | engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask; |
992 | if (GRAPHICS_VER(engine->i915) == 9) { |
993 | cmd_tables = gen9_blt_cmd_table; |
994 | cmd_table_count = ARRAY_SIZE(gen9_blt_cmd_table); |
995 | engine->get_cmd_length_mask = |
996 | gen9_blt_get_cmd_length_mask; |
997 | |
998 | /* BCS Engine unsafe without parser */ |
999 | engine->flags |= I915_ENGINE_REQUIRES_CMD_PARSER; |
1000 | } else if (IS_HASWELL(engine->i915)) { |
1001 | cmd_tables = hsw_blt_ring_cmd_table; |
1002 | cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmd_table); |
1003 | } else { |
1004 | cmd_tables = gen7_blt_cmd_table; |
1005 | cmd_table_count = ARRAY_SIZE(gen7_blt_cmd_table); |
1006 | } |
1007 | |
1008 | if (GRAPHICS_VER(engine->i915) == 9) { |
1009 | engine->reg_tables = gen9_blt_reg_tables; |
1010 | engine->reg_table_count = |
1011 | ARRAY_SIZE(gen9_blt_reg_tables); |
1012 | } else if (IS_HASWELL(engine->i915)) { |
1013 | engine->reg_tables = hsw_blt_reg_tables; |
1014 | engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables); |
1015 | } else { |
1016 | engine->reg_tables = ivb_blt_reg_tables; |
1017 | engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables); |
1018 | } |
1019 | break; |
1020 | case VIDEO_ENHANCEMENT_CLASS: |
1021 | cmd_tables = hsw_vebox_cmd_table; |
1022 | cmd_table_count = ARRAY_SIZE(hsw_vebox_cmd_table); |
1023 | /* VECS can use the same length_mask function as VCS */ |
1024 | engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; |
1025 | break; |
1026 | default: |
1027 | MISSING_CASE(engine->class); |
1028 | goto out; |
1029 | } |
1030 | |
1031 | if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) { |
1032 | drm_err(&engine->i915->drm, |
1033 | "%s: command descriptions are not sorted\n" , |
1034 | engine->name); |
1035 | goto out; |
1036 | } |
1037 | if (!validate_regs_sorted(engine)) { |
1038 | drm_err(&engine->i915->drm, |
1039 | "%s: registers are not sorted\n" , engine->name); |
1040 | goto out; |
1041 | } |
1042 | |
1043 | ret = init_hash_table(engine, cmd_tables, cmd_table_count); |
1044 | if (ret) { |
1045 | drm_err(&engine->i915->drm, |
1046 | "%s: initialised failed!\n" , engine->name); |
1047 | fini_hash_table(engine); |
1048 | goto out; |
1049 | } |
1050 | |
1051 | engine->flags |= I915_ENGINE_USING_CMD_PARSER; |
1052 | |
1053 | out: |
1054 | if (intel_engine_requires_cmd_parser(engine) && |
1055 | !intel_engine_using_cmd_parser(engine)) |
1056 | return -EINVAL; |
1057 | |
1058 | return 0; |
1059 | } |
1060 | |
1061 | /** |
1062 | * intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields |
1063 | * @engine: the engine to clean up |
1064 | * |
1065 | * Releases any resources related to command parsing that may have been |
1066 | * initialized for the specified engine. |
1067 | */ |
1068 | void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine) |
1069 | { |
1070 | if (!intel_engine_using_cmd_parser(engine)) |
1071 | return; |
1072 | |
1073 | fini_hash_table(engine); |
1074 | } |
1075 | |
1076 | static const struct drm_i915_cmd_descriptor* |
1077 | find_cmd_in_table(struct intel_engine_cs *engine, |
1078 | u32 ) |
1079 | { |
1080 | struct cmd_node *desc_node; |
1081 | |
1082 | hash_for_each_possible(engine->cmd_hash, desc_node, node, |
1083 | cmd_header_key(cmd_header)) { |
1084 | const struct drm_i915_cmd_descriptor *desc = desc_node->desc; |
1085 | if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0) |
1086 | return desc; |
1087 | } |
1088 | |
1089 | return NULL; |
1090 | } |
1091 | |
1092 | /* |
1093 | * Returns a pointer to a descriptor for the command specified by cmd_header. |
1094 | * |
1095 | * The caller must supply space for a default descriptor via the default_desc |
1096 | * parameter. If no descriptor for the specified command exists in the engine's |
1097 | * command parser tables, this function fills in default_desc based on the |
1098 | * engine's default length encoding and returns default_desc. |
1099 | */ |
1100 | static const struct drm_i915_cmd_descriptor* |
1101 | find_cmd(struct intel_engine_cs *engine, |
1102 | u32 , |
1103 | const struct drm_i915_cmd_descriptor *desc, |
1104 | struct drm_i915_cmd_descriptor *default_desc) |
1105 | { |
1106 | u32 mask; |
1107 | |
1108 | if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0) |
1109 | return desc; |
1110 | |
1111 | desc = find_cmd_in_table(engine, cmd_header); |
1112 | if (desc) |
1113 | return desc; |
1114 | |
1115 | mask = engine->get_cmd_length_mask(cmd_header); |
1116 | if (!mask) |
1117 | return NULL; |
1118 | |
1119 | default_desc->cmd.value = cmd_header; |
1120 | default_desc->cmd.mask = ~0u << MIN_OPCODE_SHIFT; |
1121 | default_desc->length.mask = mask; |
1122 | default_desc->flags = CMD_DESC_SKIP; |
1123 | return default_desc; |
1124 | } |
1125 | |
1126 | static const struct drm_i915_reg_descriptor * |
1127 | __find_reg(const struct drm_i915_reg_descriptor *table, int count, u32 addr) |
1128 | { |
1129 | int start = 0, end = count; |
1130 | while (start < end) { |
1131 | int mid = start + (end - start) / 2; |
1132 | int ret = addr - i915_mmio_reg_offset(table[mid].addr); |
1133 | if (ret < 0) |
1134 | end = mid; |
1135 | else if (ret > 0) |
1136 | start = mid + 1; |
1137 | else |
1138 | return &table[mid]; |
1139 | } |
1140 | return NULL; |
1141 | } |
1142 | |
1143 | static const struct drm_i915_reg_descriptor * |
1144 | find_reg(const struct intel_engine_cs *engine, u32 addr) |
1145 | { |
1146 | const struct drm_i915_reg_table *table = engine->reg_tables; |
1147 | const struct drm_i915_reg_descriptor *reg = NULL; |
1148 | int count = engine->reg_table_count; |
1149 | |
1150 | for (; !reg && (count > 0); ++table, --count) |
1151 | reg = __find_reg(table: table->regs, count: table->num_regs, addr); |
1152 | |
1153 | return reg; |
1154 | } |
1155 | |
1156 | /* Returns a vmap'd pointer to dst_obj, which the caller must unmap */ |
1157 | static u32 *copy_batch(struct drm_i915_gem_object *dst_obj, |
1158 | struct drm_i915_gem_object *src_obj, |
1159 | unsigned long offset, unsigned long length, |
1160 | bool *needs_clflush_after) |
1161 | { |
1162 | unsigned int src_needs_clflush; |
1163 | unsigned int dst_needs_clflush; |
1164 | void *dst, *src; |
1165 | int ret; |
1166 | |
1167 | ret = i915_gem_object_prepare_write(obj: dst_obj, needs_clflush: &dst_needs_clflush); |
1168 | if (ret) |
1169 | return ERR_PTR(error: ret); |
1170 | |
1171 | dst = i915_gem_object_pin_map(obj: dst_obj, type: I915_MAP_WB); |
1172 | i915_gem_object_finish_access(obj: dst_obj); |
1173 | if (IS_ERR(ptr: dst)) |
1174 | return dst; |
1175 | |
1176 | ret = i915_gem_object_prepare_read(obj: src_obj, needs_clflush: &src_needs_clflush); |
1177 | if (ret) { |
1178 | i915_gem_object_unpin_map(obj: dst_obj); |
1179 | return ERR_PTR(error: ret); |
1180 | } |
1181 | |
1182 | src = ERR_PTR(error: -ENODEV); |
1183 | if (src_needs_clflush && i915_has_memcpy_from_wc()) { |
1184 | src = i915_gem_object_pin_map(obj: src_obj, type: I915_MAP_WC); |
1185 | if (!IS_ERR(ptr: src)) { |
1186 | i915_unaligned_memcpy_from_wc(dst, |
1187 | src: src + offset, |
1188 | len: length); |
1189 | i915_gem_object_unpin_map(obj: src_obj); |
1190 | } |
1191 | } |
1192 | if (IS_ERR(ptr: src)) { |
1193 | unsigned long x, n, remain; |
1194 | void *ptr; |
1195 | |
1196 | /* |
1197 | * We can avoid clflushing partial cachelines before the write |
1198 | * if we only every write full cache-lines. Since we know that |
1199 | * both the source and destination are in multiples of |
1200 | * PAGE_SIZE, we can simply round up to the next cacheline. |
1201 | * We don't care about copying too much here as we only |
1202 | * validate up to the end of the batch. |
1203 | */ |
1204 | remain = length; |
1205 | if (dst_needs_clflush & CLFLUSH_BEFORE) |
1206 | remain = round_up(remain, |
1207 | boot_cpu_data.x86_clflush_size); |
1208 | |
1209 | ptr = dst; |
1210 | x = offset_in_page(offset); |
1211 | for (n = offset >> PAGE_SHIFT; remain; n++) { |
1212 | int len = min(remain, PAGE_SIZE - x); |
1213 | |
1214 | src = kmap_atomic(i915_gem_object_get_page(src_obj, n)); |
1215 | if (src_needs_clflush) |
1216 | drm_clflush_virt_range(addr: src + x, length: len); |
1217 | memcpy(ptr, src + x, len); |
1218 | kunmap_atomic(src); |
1219 | |
1220 | ptr += len; |
1221 | remain -= len; |
1222 | x = 0; |
1223 | } |
1224 | } |
1225 | |
1226 | i915_gem_object_finish_access(obj: src_obj); |
1227 | |
1228 | memset32(s: dst + length, v: 0, n: (dst_obj->base.size - length) / sizeof(u32)); |
1229 | |
1230 | /* dst_obj is returned with vmap pinned */ |
1231 | *needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER; |
1232 | |
1233 | return dst; |
1234 | } |
1235 | |
1236 | static inline bool cmd_desc_is(const struct drm_i915_cmd_descriptor * const desc, |
1237 | const u32 cmd) |
1238 | { |
1239 | return desc->cmd.value == (cmd & desc->cmd.mask); |
1240 | } |
1241 | |
1242 | static bool check_cmd(const struct intel_engine_cs *engine, |
1243 | const struct drm_i915_cmd_descriptor *desc, |
1244 | const u32 *cmd, u32 length) |
1245 | { |
1246 | if (desc->flags & CMD_DESC_SKIP) |
1247 | return true; |
1248 | |
1249 | if (desc->flags & CMD_DESC_REJECT) { |
1250 | DRM_DEBUG("CMD: Rejected command: 0x%08X\n" , *cmd); |
1251 | return false; |
1252 | } |
1253 | |
1254 | if (desc->flags & CMD_DESC_REGISTER) { |
1255 | /* |
1256 | * Get the distance between individual register offset |
1257 | * fields if the command can perform more than one |
1258 | * access at a time. |
1259 | */ |
1260 | const u32 step = desc->reg.step ? desc->reg.step : length; |
1261 | u32 offset; |
1262 | |
1263 | for (offset = desc->reg.offset; offset < length; |
1264 | offset += step) { |
1265 | const u32 reg_addr = cmd[offset] & desc->reg.mask; |
1266 | const struct drm_i915_reg_descriptor *reg = |
1267 | find_reg(engine, addr: reg_addr); |
1268 | |
1269 | if (!reg) { |
1270 | DRM_DEBUG("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n" , |
1271 | reg_addr, *cmd, engine->name); |
1272 | return false; |
1273 | } |
1274 | |
1275 | /* |
1276 | * Check the value written to the register against the |
1277 | * allowed mask/value pair given in the whitelist entry. |
1278 | */ |
1279 | if (reg->mask) { |
1280 | if (cmd_desc_is(desc, MI_LOAD_REGISTER_MEM)) { |
1281 | DRM_DEBUG("CMD: Rejected LRM to masked register 0x%08X\n" , |
1282 | reg_addr); |
1283 | return false; |
1284 | } |
1285 | |
1286 | if (cmd_desc_is(desc, MI_LOAD_REGISTER_REG)) { |
1287 | DRM_DEBUG("CMD: Rejected LRR to masked register 0x%08X\n" , |
1288 | reg_addr); |
1289 | return false; |
1290 | } |
1291 | |
1292 | if (cmd_desc_is(desc, MI_LOAD_REGISTER_IMM(1)) && |
1293 | (offset + 2 > length || |
1294 | (cmd[offset + 1] & reg->mask) != reg->value)) { |
1295 | DRM_DEBUG("CMD: Rejected LRI to masked register 0x%08X\n" , |
1296 | reg_addr); |
1297 | return false; |
1298 | } |
1299 | } |
1300 | } |
1301 | } |
1302 | |
1303 | if (desc->flags & CMD_DESC_BITMASK) { |
1304 | int i; |
1305 | |
1306 | for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) { |
1307 | u32 dword; |
1308 | |
1309 | if (desc->bits[i].mask == 0) |
1310 | break; |
1311 | |
1312 | if (desc->bits[i].condition_mask != 0) { |
1313 | u32 offset = |
1314 | desc->bits[i].condition_offset; |
1315 | u32 condition = cmd[offset] & |
1316 | desc->bits[i].condition_mask; |
1317 | |
1318 | if (condition == 0) |
1319 | continue; |
1320 | } |
1321 | |
1322 | if (desc->bits[i].offset >= length) { |
1323 | DRM_DEBUG("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n" , |
1324 | *cmd, engine->name); |
1325 | return false; |
1326 | } |
1327 | |
1328 | dword = cmd[desc->bits[i].offset] & |
1329 | desc->bits[i].mask; |
1330 | |
1331 | if (dword != desc->bits[i].expected) { |
1332 | DRM_DEBUG("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n" , |
1333 | *cmd, |
1334 | desc->bits[i].mask, |
1335 | desc->bits[i].expected, |
1336 | dword, engine->name); |
1337 | return false; |
1338 | } |
1339 | } |
1340 | } |
1341 | |
1342 | return true; |
1343 | } |
1344 | |
1345 | static int check_bbstart(u32 *cmd, u32 offset, u32 length, |
1346 | u32 batch_length, |
1347 | u64 batch_addr, |
1348 | u64 shadow_addr, |
1349 | const unsigned long *jump_whitelist) |
1350 | { |
1351 | u64 jump_offset, jump_target; |
1352 | u32 target_cmd_offset, target_cmd_index; |
1353 | |
1354 | /* For igt compatibility on older platforms */ |
1355 | if (!jump_whitelist) { |
1356 | DRM_DEBUG("CMD: Rejecting BB_START for ggtt based submission\n" ); |
1357 | return -EACCES; |
1358 | } |
1359 | |
1360 | if (length != 3) { |
1361 | DRM_DEBUG("CMD: Recursive BB_START with bad length(%u)\n" , |
1362 | length); |
1363 | return -EINVAL; |
1364 | } |
1365 | |
1366 | jump_target = *(u64 *)(cmd + 1); |
1367 | jump_offset = jump_target - batch_addr; |
1368 | |
1369 | /* |
1370 | * Any underflow of jump_target is guaranteed to be outside the range |
1371 | * of a u32, so >= test catches both too large and too small |
1372 | */ |
1373 | if (jump_offset >= batch_length) { |
1374 | DRM_DEBUG("CMD: BB_START to 0x%llx jumps out of BB\n" , |
1375 | jump_target); |
1376 | return -EINVAL; |
1377 | } |
1378 | |
1379 | /* |
1380 | * This cannot overflow a u32 because we already checked jump_offset |
1381 | * is within the BB, and the batch_length is a u32 |
1382 | */ |
1383 | target_cmd_offset = lower_32_bits(jump_offset); |
1384 | target_cmd_index = target_cmd_offset / sizeof(u32); |
1385 | |
1386 | *(u64 *)(cmd + 1) = shadow_addr + target_cmd_offset; |
1387 | |
1388 | if (target_cmd_index == offset) |
1389 | return 0; |
1390 | |
1391 | if (IS_ERR(ptr: jump_whitelist)) |
1392 | return PTR_ERR(ptr: jump_whitelist); |
1393 | |
1394 | if (!test_bit(target_cmd_index, jump_whitelist)) { |
1395 | DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n" , |
1396 | jump_target); |
1397 | return -EINVAL; |
1398 | } |
1399 | |
1400 | return 0; |
1401 | } |
1402 | |
1403 | static unsigned long *alloc_whitelist(u32 batch_length) |
1404 | { |
1405 | unsigned long *jmp; |
1406 | |
1407 | /* |
1408 | * We expect batch_length to be less than 256KiB for known users, |
1409 | * i.e. we need at most an 8KiB bitmap allocation which should be |
1410 | * reasonably cheap due to kmalloc caches. |
1411 | */ |
1412 | |
1413 | /* Prefer to report transient allocation failure rather than hit oom */ |
1414 | jmp = bitmap_zalloc(DIV_ROUND_UP(batch_length, sizeof(u32)), |
1415 | GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); |
1416 | if (!jmp) |
1417 | return ERR_PTR(error: -ENOMEM); |
1418 | |
1419 | return jmp; |
1420 | } |
1421 | |
1422 | #define LENGTH_BIAS 2 |
1423 | |
1424 | /** |
1425 | * intel_engine_cmd_parser() - parse a batch buffer for privilege violations |
1426 | * @engine: the engine on which the batch is to execute |
1427 | * @batch: the batch buffer in question |
1428 | * @batch_offset: byte offset in the batch at which execution starts |
1429 | * @batch_length: length of the commands in batch_obj |
1430 | * @shadow: validated copy of the batch buffer in question |
1431 | * @trampoline: true if we need to trampoline into privileged execution |
1432 | * |
1433 | * Parses the specified batch buffer looking for privilege violations as |
1434 | * described in the overview. |
1435 | * |
1436 | * Return: non-zero if the parser finds violations or otherwise fails; -EACCES |
1437 | * if the batch appears legal but should use hardware parsing |
1438 | */ |
1439 | |
1440 | int intel_engine_cmd_parser(struct intel_engine_cs *engine, |
1441 | struct i915_vma *batch, |
1442 | unsigned long batch_offset, |
1443 | unsigned long batch_length, |
1444 | struct i915_vma *shadow, |
1445 | bool trampoline) |
1446 | { |
1447 | u32 *cmd, *batch_end, offset = 0; |
1448 | struct drm_i915_cmd_descriptor default_desc = noop_desc; |
1449 | const struct drm_i915_cmd_descriptor *desc = &default_desc; |
1450 | bool needs_clflush_after = false; |
1451 | unsigned long *jump_whitelist; |
1452 | u64 batch_addr, shadow_addr; |
1453 | int ret = 0; |
1454 | |
1455 | GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd))); |
1456 | GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd))); |
1457 | GEM_BUG_ON(range_overflows_t(u64, batch_offset, batch_length, |
1458 | batch->size)); |
1459 | GEM_BUG_ON(!batch_length); |
1460 | |
1461 | cmd = copy_batch(dst_obj: shadow->obj, src_obj: batch->obj, |
1462 | offset: batch_offset, length: batch_length, |
1463 | needs_clflush_after: &needs_clflush_after); |
1464 | if (IS_ERR(ptr: cmd)) { |
1465 | DRM_DEBUG("CMD: Failed to copy batch\n" ); |
1466 | return PTR_ERR(ptr: cmd); |
1467 | } |
1468 | |
1469 | jump_whitelist = NULL; |
1470 | if (!trampoline) |
1471 | /* Defer failure until attempted use */ |
1472 | jump_whitelist = alloc_whitelist(batch_length); |
1473 | |
1474 | shadow_addr = gen8_canonical_addr(address: i915_vma_offset(vma: shadow)); |
1475 | batch_addr = gen8_canonical_addr(address: i915_vma_offset(vma: batch) + batch_offset); |
1476 | |
1477 | /* |
1478 | * We use the batch length as size because the shadow object is as |
1479 | * large or larger and copy_batch() will write MI_NOPs to the extra |
1480 | * space. Parsing should be faster in some cases this way. |
1481 | */ |
1482 | batch_end = cmd + batch_length / sizeof(*batch_end); |
1483 | do { |
1484 | u32 length; |
1485 | |
1486 | if (*cmd == MI_BATCH_BUFFER_END) |
1487 | break; |
1488 | |
1489 | desc = find_cmd(engine, cmd_header: *cmd, desc, default_desc: &default_desc); |
1490 | if (!desc) { |
1491 | DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n" , *cmd); |
1492 | ret = -EINVAL; |
1493 | break; |
1494 | } |
1495 | |
1496 | if (desc->flags & CMD_DESC_FIXED) |
1497 | length = desc->length.fixed; |
1498 | else |
1499 | length = (*cmd & desc->length.mask) + LENGTH_BIAS; |
1500 | |
1501 | if ((batch_end - cmd) < length) { |
1502 | DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n" , |
1503 | *cmd, |
1504 | length, |
1505 | batch_end - cmd); |
1506 | ret = -EINVAL; |
1507 | break; |
1508 | } |
1509 | |
1510 | if (!check_cmd(engine, desc, cmd, length)) { |
1511 | ret = -EACCES; |
1512 | break; |
1513 | } |
1514 | |
1515 | if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) { |
1516 | ret = check_bbstart(cmd, offset, length, batch_length, |
1517 | batch_addr, shadow_addr, |
1518 | jump_whitelist); |
1519 | break; |
1520 | } |
1521 | |
1522 | if (!IS_ERR_OR_NULL(ptr: jump_whitelist)) |
1523 | __set_bit(offset, jump_whitelist); |
1524 | |
1525 | cmd += length; |
1526 | offset += length; |
1527 | if (cmd >= batch_end) { |
1528 | DRM_DEBUG("CMD: Got to the end of the buffer w/o a BBE cmd!\n" ); |
1529 | ret = -EINVAL; |
1530 | break; |
1531 | } |
1532 | } while (1); |
1533 | |
1534 | if (trampoline) { |
1535 | /* |
1536 | * With the trampoline, the shadow is executed twice. |
1537 | * |
1538 | * 1 - starting at offset 0, in privileged mode |
1539 | * 2 - starting at offset batch_len, as non-privileged |
1540 | * |
1541 | * Only if the batch is valid and safe to execute, do we |
1542 | * allow the first privileged execution to proceed. If not, |
1543 | * we terminate the first batch and use the second batchbuffer |
1544 | * entry to chain to the original unsafe non-privileged batch, |
1545 | * leaving it to the HW to validate. |
1546 | */ |
1547 | *batch_end = MI_BATCH_BUFFER_END; |
1548 | |
1549 | if (ret) { |
1550 | /* Batch unsafe to execute with privileges, cancel! */ |
1551 | cmd = page_mask_bits(shadow->obj->mm.mapping); |
1552 | *cmd = MI_BATCH_BUFFER_END; |
1553 | |
1554 | /* If batch is unsafe but valid, jump to the original */ |
1555 | if (ret == -EACCES) { |
1556 | unsigned int flags; |
1557 | |
1558 | flags = MI_BATCH_NON_SECURE_I965; |
1559 | if (IS_HASWELL(engine->i915)) |
1560 | flags = MI_BATCH_NON_SECURE_HSW; |
1561 | |
1562 | GEM_BUG_ON(!IS_GRAPHICS_VER(engine->i915, 6, 7)); |
1563 | __gen6_emit_bb_start(cs: batch_end, |
1564 | addr: batch_addr, |
1565 | flags); |
1566 | |
1567 | ret = 0; /* allow execution */ |
1568 | } |
1569 | } |
1570 | } |
1571 | |
1572 | i915_gem_object_flush_map(obj: shadow->obj); |
1573 | |
1574 | if (!IS_ERR_OR_NULL(ptr: jump_whitelist)) |
1575 | kfree(objp: jump_whitelist); |
1576 | i915_gem_object_unpin_map(obj: shadow->obj); |
1577 | return ret; |
1578 | } |
1579 | |
1580 | /** |
1581 | * i915_cmd_parser_get_version() - get the cmd parser version number |
1582 | * @dev_priv: i915 device private |
1583 | * |
1584 | * The cmd parser maintains a simple increasing integer version number suitable |
1585 | * for passing to userspace clients to determine what operations are permitted. |
1586 | * |
1587 | * Return: the current version number of the cmd parser |
1588 | */ |
1589 | int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv) |
1590 | { |
1591 | struct intel_engine_cs *engine; |
1592 | bool active = false; |
1593 | |
1594 | /* If the command parser is not enabled, report 0 - unsupported */ |
1595 | for_each_uabi_engine(engine, dev_priv) { |
1596 | if (intel_engine_using_cmd_parser(engine)) { |
1597 | active = true; |
1598 | break; |
1599 | } |
1600 | } |
1601 | if (!active) |
1602 | return 0; |
1603 | |
1604 | /* |
1605 | * Command parser version history |
1606 | * |
1607 | * 1. Initial version. Checks batches and reports violations, but leaves |
1608 | * hardware parsing enabled (so does not allow new use cases). |
1609 | * 2. Allow access to the MI_PREDICATE_SRC0 and |
1610 | * MI_PREDICATE_SRC1 registers. |
1611 | * 3. Allow access to the GPGPU_THREADS_DISPATCHED register. |
1612 | * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3. |
1613 | * 5. GPGPU dispatch compute indirect registers. |
1614 | * 6. TIMESTAMP register and Haswell CS GPR registers |
1615 | * 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers. |
1616 | * 8. Don't report cmd_check() failures as EINVAL errors to userspace; |
1617 | * rely on the HW to NOOP disallowed commands as it would without |
1618 | * the parser enabled. |
1619 | * 9. Don't whitelist or handle oacontrol specially, as ownership |
1620 | * for oacontrol state is moving to i915-perf. |
1621 | * 10. Support for Gen9 BCS Parsing |
1622 | */ |
1623 | return 10; |
1624 | } |
1625 | |