1	/*
2	* Copyright © 2013 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*
23	* Authors:
24	* Brad Volkin <bradley.d.volkin@intel.com>
25	*
26	*/
27
28	#include <linux/highmem.h>
29
30	#include <drm/drm_cache.h>
31	#include <drm/drm_print.h>
32
33	#include "gt/intel_engine.h"
34	#include "gt/intel_engine_regs.h"
35	#include "gt/intel_gpu_commands.h"
36	#include "gt/intel_gt_regs.h"
37
38	#include "i915_cmd_parser.h"
39	#include "i915_drv.h"
40	#include "i915_memcpy.h"
41	#include "i915_reg.h"
42
43	/**
44	* DOC: batch buffer command parser
45	*
46	* Motivation:
47	* Certain OpenGL features (e.g. transform feedback, performance monitoring)
48	* require userspace code to submit batches containing commands such as
49	* MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some
50	* generations of the hardware will noop these commands in "unsecure" batches
51	* (which includes all userspace batches submitted via i915) even though the
52	* commands may be safe and represent the intended programming model of the
53	* device.
54	*
55	* The software command parser is similar in operation to the command parsing
56	* done in hardware for unsecure batches. However, the software parser allows
57	* some operations that would be noop'd by hardware, if the parser determines
58	* the operation is safe, and submits the batch as "secure" to prevent hardware
59	* parsing.
60	*
61	* Threats:
62	* At a high level, the hardware (and software) checks attempt to prevent
63	* granting userspace undue privileges. There are three categories of privilege.
64	*
65	* First, commands which are explicitly defined as privileged or which should
66	* only be used by the kernel driver. The parser rejects such commands
67	*
68	* Second, commands which access registers. To support correct/enhanced
69	* userspace functionality, particularly certain OpenGL extensions, the parser
70	* provides a whitelist of registers which userspace may safely access
71	*
72	* Third, commands which access privileged memory (i.e. GGTT, HWS page, etc).
73	* The parser always rejects such commands.
74	*
75	* The majority of the problematic commands fall in the MI_* range, with only a
76	* few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW).
77	*
78	* Implementation:
79	* Each engine maintains tables of commands and registers which the parser
80	* uses in scanning batch buffers submitted to that engine.
81	*
82	* Since the set of commands that the parser must check for is significantly
83	* smaller than the number of commands supported, the parser tables contain only
84	* those commands required by the parser. This generally works because command
85	* opcode ranges have standard command length encodings. So for commands that
86	* the parser does not need to check, it can easily skip them. This is
87	* implemented via a per-engine length decoding vfunc.
88	*
89	* Unfortunately, there are a number of commands that do not follow the standard
90	* length encoding for their opcode range, primarily amongst the MI_* commands.
91	* To handle this, the parser provides a way to define explicit "skip" entries
92	* in the per-engine command tables.
93	*
94	* Other command table entries map fairly directly to high level categories
95	* mentioned above: rejected, register whitelist. The parser implements a number
96	* of checks, including the privileged memory checks, via a general bitmasking
97	* mechanism.
98	*/
99
100	/*
101	* A command that requires special handling by the command parser.
102	*/
103	struct drm_i915_cmd_descriptor {
104	/*
105	* Flags describing how the command parser processes the command.
106	*
107	* CMD_DESC_FIXED: The command has a fixed length if this is set,
108	* a length mask if not set
109	* CMD_DESC_SKIP: The command is allowed but does not follow the
110	* standard length encoding for the opcode range in
111	* which it falls
112	* CMD_DESC_REJECT: The command is never allowed
113	* CMD_DESC_REGISTER: The command should be checked against the
114	* register whitelist for the appropriate ring
115	*/
116	u32 flags;
117	#define CMD_DESC_FIXED (1<<0)
118	#define CMD_DESC_SKIP (1<<1)
119	#define CMD_DESC_REJECT (1<<2)
120	#define CMD_DESC_REGISTER (1<<3)
121	#define CMD_DESC_BITMASK (1<<4)
122
123	/*
124	* The command's unique identification bits and the bitmask to get them.
125	* This isn't strictly the opcode field as defined in the spec and may
126	* also include type, subtype, and/or subop fields.
127	*/
128	struct {
129	u32 value;
130	u32 mask;
131	} cmd;
132
133	/*
134	* The command's length. The command is either fixed length (i.e. does
135	* not include a length field) or has a length field mask. The flag
136	* CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has
137	* a length mask. All command entries in a command table must include
138	* length information.
139	*/
140	union {
141	u32 fixed;
142	u32 mask;
143	} length;
144
145	/*
146	* Describes where to find a register address in the command to check
147	* against the ring's register whitelist. Only valid if flags has the
148	* CMD_DESC_REGISTER bit set.
149	*
150	* A non-zero step value implies that the command may access multiple
151	* registers in sequence (e.g. LRI), in that case step gives the
152	* distance in dwords between individual offset fields.
153	*/
154	struct {
155	u32 offset;
156	u32 mask;
157	u32 step;
158	} reg;
159
160	#define MAX_CMD_DESC_BITMASKS 3
161	/*
162	* Describes command checks where a particular dword is masked and
163	* compared against an expected value. If the command does not match
164	* the expected value, the parser rejects it. Only valid if flags has
165	* the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero
166	* are valid.
167	*
168	* If the check specifies a non-zero condition_mask then the parser
169	* only performs the check when the bits specified by condition_mask
170	* are non-zero.
171	*/
172	struct {
173	u32 offset;
174	u32 mask;
175	u32 expected;
176	u32 condition_offset;
177	u32 condition_mask;
178	} bits[MAX_CMD_DESC_BITMASKS];
179	};
180
181	/*
182	* A table of commands requiring special handling by the command parser.
183	*
184	* Each engine has an array of tables. Each table consists of an array of
185	* command descriptors, which must be sorted with command opcodes in
186	* ascending order.
187	*/
188	struct drm_i915_cmd_table {
189	const struct drm_i915_cmd_descriptor *table;
190	int count;
191	};
192
193	#define STD_MI_OPCODE_SHIFT (32 - 9)
194	#define STD_3D_OPCODE_SHIFT (32 - 16)
195	#define STD_2D_OPCODE_SHIFT (32 - 10)
196	#define STD_MFX_OPCODE_SHIFT (32 - 16)
197	#define MIN_OPCODE_SHIFT 16
198
199	#define CMD(op, opm, f, lm, fl, ...) \
200	{ \
201	.flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \
202	.cmd = { (op & ~0u << (opm)), ~0u << (opm) }, \
203	.length = { (lm) }, \
204	__VA_ARGS__ \
205	}
206
207	/* Convenience macros to compress the tables */
208	#define SMI STD_MI_OPCODE_SHIFT
209	#define S3D STD_3D_OPCODE_SHIFT
210	#define S2D STD_2D_OPCODE_SHIFT
211	#define SMFX STD_MFX_OPCODE_SHIFT
212	#define F true
213	#define S CMD_DESC_SKIP
214	#define R CMD_DESC_REJECT
215	#define W CMD_DESC_REGISTER
216	#define B CMD_DESC_BITMASK
217
218	/* Command Mask Fixed Len Action
219	---------------------------------------------------------- */
220	static const struct drm_i915_cmd_descriptor gen7_common_cmds[] = {
221	CMD( MI_NOOP, SMI, F, 1, S ),
222	CMD( MI_USER_INTERRUPT, SMI, F, 1, R ),
223	CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, R ),
224	CMD( MI_ARB_CHECK, SMI, F, 1, S ),
225	CMD( MI_REPORT_HEAD, SMI, F, 1, S ),
226	CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ),
227	CMD( MI_SEMAPHORE_MBOX, SMI, !F, 0xFF, R ),
228	CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ),
229	CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W,
230	.reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ),
231	CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B,
232	.reg = { .offset = 1, .mask = 0x007FFFFC },
233	.bits = {{
234	.offset = 0,
235	.mask = MI_GLOBAL_GTT,
236	.expected = 0,
237	}}, ),
238	CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B,
239	.reg = { .offset = 1, .mask = 0x007FFFFC },
240	.bits = {{
241	.offset = 0,
242	.mask = MI_GLOBAL_GTT,
243	.expected = 0,
244	}}, ),
245	/*
246	* MI_BATCH_BUFFER_START requires some special handling. It's not
247	* really a 'skip' action but it doesn't seem like it's worth adding
248	* a new action. See intel_engine_cmd_parser().
249	*/
250	CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ),
251	};
252
253	static const struct drm_i915_cmd_descriptor gen7_render_cmds[] = {
254	CMD( MI_FLUSH, SMI, F, 1, S ),
255	CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),
256	CMD( MI_PREDICATE, SMI, F, 1, S ),
257	CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ),
258	CMD( MI_SET_APPID, SMI, F, 1, S ),
259	CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ),
260	CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ),
261	CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ),
262	CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B,
263	.bits = {{
264	.offset = 0,
265	.mask = MI_GLOBAL_GTT,
266	.expected = 0,
267	}}, ),
268	CMD( MI_UPDATE_GTT, SMI, !F, 0xFF, R ),
269	CMD( MI_CLFLUSH, SMI, !F, 0x3FF, B,
270	.bits = {{
271	.offset = 0,
272	.mask = MI_GLOBAL_GTT,
273	.expected = 0,
274	}}, ),
275	CMD( MI_REPORT_PERF_COUNT, SMI, !F, 0x3F, B,
276	.bits = {{
277	.offset = 1,
278	.mask = MI_REPORT_PERF_COUNT_GGTT,
279	.expected = 0,
280	}}, ),
281	CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B,
282	.bits = {{
283	.offset = 0,
284	.mask = MI_GLOBAL_GTT,
285	.expected = 0,
286	}}, ),
287	CMD( GFX_OP_3DSTATE_VF_STATISTICS, S3D, F, 1, S ),
288	CMD( PIPELINE_SELECT, S3D, F, 1, S ),
289	CMD( MEDIA_VFE_STATE, S3D, !F, 0xFFFF, B,
290	.bits = {{
291	.offset = 2,
292	.mask = MEDIA_VFE_STATE_MMIO_ACCESS_MASK,
293	.expected = 0,
294	}}, ),
295	CMD( GPGPU_OBJECT, S3D, !F, 0xFF, S ),
296	CMD( GPGPU_WALKER, S3D, !F, 0xFF, S ),
297	CMD( GFX_OP_3DSTATE_SO_DECL_LIST, S3D, !F, 0x1FF, S ),
298	CMD( GFX_OP_PIPE_CONTROL(5), S3D, !F, 0xFF, B,
299	.bits = {{
300	.offset = 1,
301	.mask = (PIPE_CONTROL_MMIO_WRITE | PIPE_CONTROL_NOTIFY),
302	.expected = 0,
303	},
304	{
305	.offset = 1,
306	.mask = (PIPE_CONTROL_GLOBAL_GTT_IVB |
307	PIPE_CONTROL_STORE_DATA_INDEX),
308	.expected = 0,
309	.condition_offset = 1,
310	.condition_mask = PIPE_CONTROL_POST_SYNC_OP_MASK,
311	}}, ),
312	};
313
314	static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = {
315	CMD( MI_SET_PREDICATE, SMI, F, 1, S ),
316	CMD( MI_RS_CONTROL, SMI, F, 1, S ),
317	CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ),
318	CMD( MI_SET_APPID, SMI, F, 1, S ),
319	CMD( MI_RS_CONTEXT, SMI, F, 1, S ),
320	CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ),
321	CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ),
322	CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W,
323	.reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ),
324	CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ),
325	CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ),
326	CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ),
327	CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_VS, S3D, !F, 0x7FF, S ),
328	CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_PS, S3D, !F, 0x7FF, S ),
329
330	CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS, S3D, !F, 0x1FF, S ),
331	CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS, S3D, !F, 0x1FF, S ),
332	CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS, S3D, !F, 0x1FF, S ),
333	CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS, S3D, !F, 0x1FF, S ),
334	CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ),
335	};
336
337	static const struct drm_i915_cmd_descriptor gen7_video_cmds[] = {
338	CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),
339	CMD( MI_SET_APPID, SMI, F, 1, S ),
340	CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B,
341	.bits = {{
342	.offset = 0,
343	.mask = MI_GLOBAL_GTT,
344	.expected = 0,
345	}}, ),
346	CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ),
347	CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B,
348	.bits = {{
349	.offset = 0,
350	.mask = MI_FLUSH_DW_NOTIFY,
351	.expected = 0,
352	},
353	{
354	.offset = 1,
355	.mask = MI_FLUSH_DW_USE_GTT,
356	.expected = 0,
357	.condition_offset = 0,
358	.condition_mask = MI_FLUSH_DW_OP_MASK,
359	},
360	{
361	.offset = 0,
362	.mask = MI_FLUSH_DW_STORE_INDEX,
363	.expected = 0,
364	.condition_offset = 0,
365	.condition_mask = MI_FLUSH_DW_OP_MASK,
366	}}, ),
367	CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B,
368	.bits = {{
369	.offset = 0,
370	.mask = MI_GLOBAL_GTT,
371	.expected = 0,
372	}}, ),
373	/*
374	* MFX_WAIT doesn't fit the way we handle length for most commands.
375	* It has a length field but it uses a non-standard length bias.
376	* It is always 1 dword though, so just treat it as fixed length.
377	*/
378	CMD( MFX_WAIT, SMFX, F, 1, S ),
379	};
380
381	static const struct drm_i915_cmd_descriptor gen7_vecs_cmds[] = {
382	CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),
383	CMD( MI_SET_APPID, SMI, F, 1, S ),
384	CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B,
385	.bits = {{
386	.offset = 0,
387	.mask = MI_GLOBAL_GTT,
388	.expected = 0,
389	}}, ),
390	CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ),
391	CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B,
392	.bits = {{
393	.offset = 0,
394	.mask = MI_FLUSH_DW_NOTIFY,
395	.expected = 0,
396	},
397	{
398	.offset = 1,
399	.mask = MI_FLUSH_DW_USE_GTT,
400	.expected = 0,
401	.condition_offset = 0,
402	.condition_mask = MI_FLUSH_DW_OP_MASK,
403	},
404	{
405	.offset = 0,
406	.mask = MI_FLUSH_DW_STORE_INDEX,
407	.expected = 0,
408	.condition_offset = 0,
409	.condition_mask = MI_FLUSH_DW_OP_MASK,
410	}}, ),
411	CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B,
412	.bits = {{
413	.offset = 0,
414	.mask = MI_GLOBAL_GTT,
415	.expected = 0,
416	}}, ),
417	};
418
419	static const struct drm_i915_cmd_descriptor gen7_blt_cmds[] = {
420	CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ),
421	CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B,
422	.bits = {{
423	.offset = 0,
424	.mask = MI_GLOBAL_GTT,
425	.expected = 0,
426	}}, ),
427	CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ),
428	CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B,
429	.bits = {{
430	.offset = 0,
431	.mask = MI_FLUSH_DW_NOTIFY,
432	.expected = 0,
433	},
434	{
435	.offset = 1,
436	.mask = MI_FLUSH_DW_USE_GTT,
437	.expected = 0,
438	.condition_offset = 0,
439	.condition_mask = MI_FLUSH_DW_OP_MASK,
440	},
441	{
442	.offset = 0,
443	.mask = MI_FLUSH_DW_STORE_INDEX,
444	.expected = 0,
445	.condition_offset = 0,
446	.condition_mask = MI_FLUSH_DW_OP_MASK,
447	}}, ),
448	CMD( COLOR_BLT, S2D, !F, 0x3F, S ),
449	CMD( SRC_COPY_BLT, S2D, !F, 0x3F, S ),
450	};
451
452	static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = {
453	CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ),
454	CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ),
455	};
456
457	/*
458	* For Gen9 we can still rely on the h/w to enforce cmd security, and only
459	* need to re-enforce the register access checks. We therefore only need to
460	* teach the cmdparser how to find the end of each command, and identify
461	* register accesses. The table doesn't need to reject any commands, and so
462	* the only commands listed here are:
463	* 1) Those that touch registers
464	* 2) Those that do not have the default 8-bit length
465	*
466	* Note that the default MI length mask chosen for this table is 0xFF, not
467	* the 0x3F used on older devices. This is because the vast majority of MI
468	* cmds on Gen9 use a standard 8-bit Length field.
469	* All the Gen9 blitter instructions are standard 0xFF length mask, and
470	* none allow access to non-general registers, so in fact no BLT cmds are
471	* included in the table at all.
472	*
473	*/
474	static const struct drm_i915_cmd_descriptor gen9_blt_cmds[] = {
475	CMD( MI_NOOP, SMI, F, 1, S ),
476	CMD( MI_USER_INTERRUPT, SMI, F, 1, S ),
477	CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, S ),
478	CMD( MI_FLUSH, SMI, F, 1, S ),
479	CMD( MI_ARB_CHECK, SMI, F, 1, S ),
480	CMD( MI_REPORT_HEAD, SMI, F, 1, S ),
481	CMD( MI_ARB_ON_OFF, SMI, F, 1, S ),
482	CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ),
483	CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, S ),
484	CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, S ),
485	CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, S ),
486	CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W,
487	.reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ),
488	CMD( MI_UPDATE_GTT, SMI, !F, 0x3FF, S ),
489	CMD( MI_STORE_REGISTER_MEM_GEN8, SMI, F, 4, W,
490	.reg = { .offset = 1, .mask = 0x007FFFFC } ),
491	CMD( MI_FLUSH_DW, SMI, !F, 0x3F, S ),
492	CMD( MI_LOAD_REGISTER_MEM_GEN8, SMI, F, 4, W,
493	.reg = { .offset = 1, .mask = 0x007FFFFC } ),
494	CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W,
495	.reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ),
496
497	/*
498	* We allow BB_START but apply further checks. We just sanitize the
499	* basic fields here.
500	*/
501	#define MI_BB_START_OPERAND_MASK GENMASK(SMI-1, 0)
502	#define MI_BB_START_OPERAND_EXPECT (MI_BATCH_PPGTT_HSW | 1)
503	CMD( MI_BATCH_BUFFER_START_GEN8, SMI, !F, 0xFF, B,
504	.bits = {{
505	.offset = 0,
506	.mask = MI_BB_START_OPERAND_MASK,
507	.expected = MI_BB_START_OPERAND_EXPECT,
508	}}, ),
509	};
510
511	static const struct drm_i915_cmd_descriptor noop_desc =
512	CMD(MI_NOOP, SMI, F, 1, S);
513
514	#undef CMD
515	#undef SMI
516	#undef S3D
517	#undef S2D
518	#undef SMFX
519	#undef F
520	#undef S
521	#undef R
522	#undef W
523	#undef B
524
525	static const struct drm_i915_cmd_table gen7_render_cmd_table[] = {
526	{ gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
527	{ gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) },
528	};
529
530	static const struct drm_i915_cmd_table hsw_render_ring_cmd_table[] = {
531	{ gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
532	{ gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) },
533	{ hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) },
534	};
535
536	static const struct drm_i915_cmd_table gen7_video_cmd_table[] = {
537	{ gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
538	{ gen7_video_cmds, ARRAY_SIZE(gen7_video_cmds) },
539	};
540
541	static const struct drm_i915_cmd_table hsw_vebox_cmd_table[] = {
542	{ gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
543	{ gen7_vecs_cmds, ARRAY_SIZE(gen7_vecs_cmds) },
544	};
545
546	static const struct drm_i915_cmd_table gen7_blt_cmd_table[] = {
547	{ gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
548	{ gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) },
549	};
550
551	static const struct drm_i915_cmd_table hsw_blt_ring_cmd_table[] = {
552	{ gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },
553	{ gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) },
554	{ hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) },
555	};
556
557	static const struct drm_i915_cmd_table gen9_blt_cmd_table[] = {
558	{ gen9_blt_cmds, ARRAY_SIZE(gen9_blt_cmds) },
559	};
560
561
562	/*
563	* Register whitelists, sorted by increasing register offset.
564	*/
565
566	/*
567	* An individual whitelist entry granting access to register addr. If
568	* mask is non-zero the argument of immediate register writes will be
569	* AND-ed with mask, and the command will be rejected if the result
570	* doesn't match value.
571	*
572	* Registers with non-zero mask are only allowed to be written using
573	* LRI.
574	*/
575	struct drm_i915_reg_descriptor {
576	i915_reg_t addr;
577	u32 mask;
578	u32 value;
579	};
580
581	/* Convenience macro for adding 32-bit registers. */
582	#define REG32(_reg, ...) \
583	{ .addr = (_reg), __VA_ARGS__ }
584
585	#define REG32_IDX(_reg, idx) \
586	{ .addr = _reg(idx) }
587
588	/*
589	* Convenience macro for adding 64-bit registers.
590	*
591	* Some registers that userspace accesses are 64 bits. The register
592	* access commands only allow 32-bit accesses. Hence, we have to include
593	* entries for both halves of the 64-bit registers.
594	*/
595	#define REG64(_reg) \
596	{ .addr = _reg }, \
597	{ .addr = _reg ## _UDW }
598
599	#define REG64_IDX(_reg, idx) \
600	{ .addr = _reg(idx) }, \
601	{ .addr = _reg ## _UDW(idx) }
602
603	#define REG64_BASE_IDX(_reg, base, idx) \
604	{ .addr = _reg(base, idx) }, \
605	{ .addr = _reg ## _UDW(base, idx) }
606
607	static const struct drm_i915_reg_descriptor gen7_render_regs[] = {
608	REG64(GPGPU_THREADS_DISPATCHED),
609	REG64(HS_INVOCATION_COUNT),
610	REG64(DS_INVOCATION_COUNT),
611	REG64(IA_VERTICES_COUNT),
612	REG64(IA_PRIMITIVES_COUNT),
613	REG64(VS_INVOCATION_COUNT),
614	REG64(GS_INVOCATION_COUNT),
615	REG64(GS_PRIMITIVES_COUNT),
616	REG64(CL_INVOCATION_COUNT),
617	REG64(CL_PRIMITIVES_COUNT),
618	REG64(PS_INVOCATION_COUNT),
619	REG64(PS_DEPTH_COUNT),
620	REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),
621	REG64_IDX(MI_PREDICATE_SRC0, RENDER_RING_BASE),
622	REG64_IDX(MI_PREDICATE_SRC1, RENDER_RING_BASE),
623	REG32(GEN7_3DPRIM_END_OFFSET),
624	REG32(GEN7_3DPRIM_START_VERTEX),
625	REG32(GEN7_3DPRIM_VERTEX_COUNT),
626	REG32(GEN7_3DPRIM_INSTANCE_COUNT),
627	REG32(GEN7_3DPRIM_START_INSTANCE),
628	REG32(GEN7_3DPRIM_BASE_VERTEX),
629	REG32(GEN7_GPGPU_DISPATCHDIMX),
630	REG32(GEN7_GPGPU_DISPATCHDIMY),
631	REG32(GEN7_GPGPU_DISPATCHDIMZ),
632	REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
633	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0),
634	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1),
635	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2),
636	REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3),
637	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0),
638	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1),
639	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2),
640	REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3),
641	REG32(GEN7_SO_WRITE_OFFSET(0)),
642	REG32(GEN7_SO_WRITE_OFFSET(1)),
643	REG32(GEN7_SO_WRITE_OFFSET(2)),
644	REG32(GEN7_SO_WRITE_OFFSET(3)),
645	REG32(GEN7_L3SQCREG1),
646	REG32(GEN7_L3CNTLREG2),
647	REG32(GEN7_L3CNTLREG3),
648	REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
649	};
650
651	static const struct drm_i915_reg_descriptor hsw_render_regs[] = {
652	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 0),
653	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 1),
654	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 2),
655	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 3),
656	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 4),
657	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 5),
658	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 6),
659	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 7),
660	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 8),
661	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 9),
662	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 10),
663	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 11),
664	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 12),
665	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 13),
666	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 14),
667	REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 15),
668	REG32(HSW_SCRATCH1,
669	.mask = ~HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE,
670	.value = 0),
671	REG32(HSW_ROW_CHICKEN3,
672	.mask = ~(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE << 16 |
673	HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
674	.value = 0),
675	};
676
677	static const struct drm_i915_reg_descriptor gen7_blt_regs[] = {
678	REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),
679	REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
680	REG32(BCS_SWCTRL),
681	REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
682	};
683
684	static const struct drm_i915_reg_descriptor gen9_blt_regs[] = {
685	REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),
686	REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
687	REG32(BCS_SWCTRL),
688	REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
689	REG32_IDX(RING_CTX_TIMESTAMP, BLT_RING_BASE),
690	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 0),
691	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 1),
692	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 2),
693	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 3),
694	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 4),
695	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 5),
696	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 6),
697	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 7),
698	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 8),
699	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 9),
700	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 10),
701	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 11),
702	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 12),
703	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 13),
704	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 14),
705	REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 15),
706	};
707
708	#undef REG64
709	#undef REG32
710
711	struct drm_i915_reg_table {
712	const struct drm_i915_reg_descriptor *regs;
713	int num_regs;
714	};
715
716	static const struct drm_i915_reg_table ivb_render_reg_tables[] = {
717	{ gen7_render_regs, ARRAY_SIZE(gen7_render_regs) },
718	};
719
720	static const struct drm_i915_reg_table ivb_blt_reg_tables[] = {
721	{ gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) },
722	};
723
724	static const struct drm_i915_reg_table hsw_render_reg_tables[] = {
725	{ gen7_render_regs, ARRAY_SIZE(gen7_render_regs) },
726	{ hsw_render_regs, ARRAY_SIZE(hsw_render_regs) },
727	};
728
729	static const struct drm_i915_reg_table hsw_blt_reg_tables[] = {
730	{ gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) },
731	};
732
733	static const struct drm_i915_reg_table gen9_blt_reg_tables[] = {
734	{ gen9_blt_regs, ARRAY_SIZE(gen9_blt_regs) },
735	};
736
737	static u32 gen7_render_get_cmd_length_mask(u32 cmd_header)
738	{
739	u32 client = cmd_header >> INSTR_CLIENT_SHIFT;
740	u32 subclient =
741	(cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT;
742
743	if (client == INSTR_MI_CLIENT)
744	return 0x3F;
745	else if (client == INSTR_RC_CLIENT) {
746	if (subclient == INSTR_MEDIA_SUBCLIENT)
747	return 0xFFFF;
748	else
749	return 0xFF;
750	}
751
752	DRM_DEBUG("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header);
753	return 0;
754	}
755
756	static u32 gen7_bsd_get_cmd_length_mask(u32 cmd_header)
757	{
758	u32 client = cmd_header >> INSTR_CLIENT_SHIFT;
759	u32 subclient =
760	(cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT;
761	u32 op = (cmd_header & INSTR_26_TO_24_MASK) >> INSTR_26_TO_24_SHIFT;
762
763	if (client == INSTR_MI_CLIENT)
764	return 0x3F;
765	else if (client == INSTR_RC_CLIENT) {
766	if (subclient == INSTR_MEDIA_SUBCLIENT) {
767	if (op == 6)
768	return 0xFFFF;
769	else
770	return 0xFFF;
771	} else
772	return 0xFF;
773	}
774
775	DRM_DEBUG("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header);
776	return 0;
777	}
778
779	static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)
780	{
781	u32 client = cmd_header >> INSTR_CLIENT_SHIFT;
782
783	if (client == INSTR_MI_CLIENT)
784	return 0x3F;
785	else if (client == INSTR_BC_CLIENT)
786	return 0xFF;
787
788	DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
789	return 0;
790	}
791
792	static u32 gen9_blt_get_cmd_length_mask(u32 cmd_header)
793	{
794	u32 client = cmd_header >> INSTR_CLIENT_SHIFT;
795
796	if (client == INSTR_MI_CLIENT || client == INSTR_BC_CLIENT)
797	return 0xFF;
798
799	DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
800	return 0;
801	}
802
803	static bool validate_cmds_sorted(const struct intel_engine_cs *engine,
804	const struct drm_i915_cmd_table *cmd_tables,
805	int cmd_table_count)
806	{
807	int i;
808	bool ret = true;
809
810	if (!cmd_tables || cmd_table_count == 0)
811	return true;
812
813	for (i = 0; i < cmd_table_count; i++) {
814	const struct drm_i915_cmd_table *table = &cmd_tables[i];
815	u32 previous = 0;
816	int j;
817
818	for (j = 0; j < table->count; j++) {
819	const struct drm_i915_cmd_descriptor *desc =
820	&table->table[j];
821	u32 curr = desc->cmd.value & desc->cmd.mask;
822
823	if (curr < previous) {
824	drm_err(&engine->i915->drm,
825	"CMD: %s [%d] command table not sorted: "
826	"table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
827	engine->name, engine->id,
828	i, j, curr, previous);
829	ret = false;
830	}
831
832	previous = curr;
833	}
834	}
835
836	return ret;
837	}
838
839	static bool check_sorted(const struct intel_engine_cs *engine,
840	const struct drm_i915_reg_descriptor *reg_table,
841	int reg_count)
842	{
843	int i;
844	u32 previous = 0;
845	bool ret = true;
846
847	for (i = 0; i < reg_count; i++) {
848	u32 curr = i915_mmio_reg_offset(reg_table[i].addr);
849
850	if (curr < previous) {
851	drm_err(&engine->i915->drm,
852	"CMD: %s [%d] register table not sorted: "
853	"entry=%d reg=0x%08X prev=0x%08X\n",
854	engine->name, engine->id,
855	i, curr, previous);
856	ret = false;
857	}
858
859	previous = curr;
860	}
861
862	return ret;
863	}
864
865	static bool validate_regs_sorted(struct intel_engine_cs *engine)
866	{
867	int i;
868	const struct drm_i915_reg_table *table;
869
870	for (i = 0; i < engine->reg_table_count; i++) {
871	table = &engine->reg_tables[i];
872	if (!check_sorted(engine, reg_table: table->regs, reg_count: table->num_regs))
873	return false;
874	}
875
876	return true;
877	}
878
879	struct cmd_node {
880	const struct drm_i915_cmd_descriptor *desc;
881	struct hlist_node node;
882	};
883
884	/*
885	* Different command ranges have different numbers of bits for the opcode. For
886	* example, MI commands use bits 31:23 while 3D commands use bits 31:16. The
887	* problem is that, for example, MI commands use bits 22:16 for other fields
888	* such as GGTT vs PPGTT bits. If we include those bits in the mask then when
889	* we mask a command from a batch it could hash to the wrong bucket due to
890	* non-opcode bits being set. But if we don't include those bits, some 3D
891	* commands may hash to the same bucket due to not including opcode bits that
892	* make the command unique. For now, we will risk hashing to the same bucket.
893	*/
894	static inline u32 cmd_header_key(u32 x)
895	{
896	switch (x >> INSTR_CLIENT_SHIFT) {
897	default:
898	case INSTR_MI_CLIENT:
899	return x >> STD_MI_OPCODE_SHIFT;
900	case INSTR_RC_CLIENT:
901	return x >> STD_3D_OPCODE_SHIFT;
902	case INSTR_BC_CLIENT:
903	return x >> STD_2D_OPCODE_SHIFT;
904	}
905	}
906
907	static int init_hash_table(struct intel_engine_cs *engine,
908	const struct drm_i915_cmd_table *cmd_tables,
909	int cmd_table_count)
910	{
911	int i, j;
912
913	hash_init(engine->cmd_hash);
914
915	for (i = 0; i < cmd_table_count; i++) {
916	const struct drm_i915_cmd_table *table = &cmd_tables[i];
917
918	for (j = 0; j < table->count; j++) {
919	const struct drm_i915_cmd_descriptor *desc =
920	&table->table[j];
921	struct cmd_node *desc_node =
922	kmalloc(sizeof(*desc_node), GFP_KERNEL);
923
924	if (!desc_node)
925	return -ENOMEM;
926
927	desc_node->desc = desc;
928	hash_add(engine->cmd_hash, &desc_node->node,
929	cmd_header_key(desc->cmd.value));
930	}
931	}
932
933	return 0;
934	}
935
936	static void fini_hash_table(struct intel_engine_cs *engine)
937	{
938	struct hlist_node *tmp;
939	struct cmd_node *desc_node;
940	int i;
941
942	hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) {
943	hash_del(node: &desc_node->node);
944	kfree(objp: desc_node);
945	}
946	}
947
948	/**
949	* intel_engine_init_cmd_parser() - set cmd parser related fields for an engine
950	* @engine: the engine to initialize
951	*
952	* Optionally initializes fields related to batch buffer command parsing in the
953	* struct intel_engine_cs based on whether the platform requires software
954	* command parsing.
955	*/
956	int intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
957	{
958	const struct drm_i915_cmd_table *cmd_tables;
959	int cmd_table_count;
960	int ret;
961
962	if (GRAPHICS_VER(engine->i915) != 7 && !(GRAPHICS_VER(engine->i915) == 9 &&
963	engine->class == COPY_ENGINE_CLASS))
964	return 0;
965
966	switch (engine->class) {
967	case RENDER_CLASS:
968	if (IS_HASWELL(engine->i915)) {
969	cmd_tables = hsw_render_ring_cmd_table;
970	cmd_table_count =
971	ARRAY_SIZE(hsw_render_ring_cmd_table);
972	} else {
973	cmd_tables = gen7_render_cmd_table;
974	cmd_table_count = ARRAY_SIZE(gen7_render_cmd_table);
975	}
976
977	if (IS_HASWELL(engine->i915)) {
978	engine->reg_tables = hsw_render_reg_tables;
979	engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables);
980	} else {
981	engine->reg_tables = ivb_render_reg_tables;
982	engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables);
983	}
984	engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
985	break;
986	case VIDEO_DECODE_CLASS:
987	cmd_tables = gen7_video_cmd_table;
988	cmd_table_count = ARRAY_SIZE(gen7_video_cmd_table);
989	engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
990	break;
991	case COPY_ENGINE_CLASS:
992	engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
993	if (GRAPHICS_VER(engine->i915) == 9) {
994	cmd_tables = gen9_blt_cmd_table;
995	cmd_table_count = ARRAY_SIZE(gen9_blt_cmd_table);
996	engine->get_cmd_length_mask =
997	gen9_blt_get_cmd_length_mask;
998
999	/* BCS Engine unsafe without parser */
1000	engine->flags |= I915_ENGINE_REQUIRES_CMD_PARSER;
1001	} else if (IS_HASWELL(engine->i915)) {
1002	cmd_tables = hsw_blt_ring_cmd_table;
1003	cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmd_table);
1004	} else {
1005	cmd_tables = gen7_blt_cmd_table;
1006	cmd_table_count = ARRAY_SIZE(gen7_blt_cmd_table);
1007	}
1008
1009	if (GRAPHICS_VER(engine->i915) == 9) {
1010	engine->reg_tables = gen9_blt_reg_tables;
1011	engine->reg_table_count =
1012	ARRAY_SIZE(gen9_blt_reg_tables);
1013	} else if (IS_HASWELL(engine->i915)) {
1014	engine->reg_tables = hsw_blt_reg_tables;
1015	engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables);
1016	} else {
1017	engine->reg_tables = ivb_blt_reg_tables;
1018	engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables);
1019	}
1020	break;
1021	case VIDEO_ENHANCEMENT_CLASS:
1022	cmd_tables = hsw_vebox_cmd_table;
1023	cmd_table_count = ARRAY_SIZE(hsw_vebox_cmd_table);
1024	/* VECS can use the same length_mask function as VCS */
1025	engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
1026	break;
1027	default:
1028	MISSING_CASE(engine->class);
1029	goto out;
1030	}
1031
1032	if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) {
1033	drm_err(&engine->i915->drm,
1034	"%s: command descriptions are not sorted\n",
1035	engine->name);
1036	goto out;
1037	}
1038	if (!validate_regs_sorted(engine)) {
1039	drm_err(&engine->i915->drm,
1040	"%s: registers are not sorted\n", engine->name);
1041	goto out;
1042	}
1043
1044	ret = init_hash_table(engine, cmd_tables, cmd_table_count);
1045	if (ret) {
1046	drm_err(&engine->i915->drm,
1047	"%s: initialised failed!\n", engine->name);
1048	fini_hash_table(engine);
1049	goto out;
1050	}
1051
1052	engine->flags |= I915_ENGINE_USING_CMD_PARSER;
1053
1054	out:
1055	if (intel_engine_requires_cmd_parser(engine) &&
1056	!intel_engine_using_cmd_parser(engine))
1057	return -EINVAL;
1058
1059	return 0;
1060	}
1061
1062	/**
1063	* intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields
1064	* @engine: the engine to clean up
1065	*
1066	* Releases any resources related to command parsing that may have been
1067	* initialized for the specified engine.
1068	*/
1069	void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine)
1070	{
1071	if (!intel_engine_using_cmd_parser(engine))
1072	return;
1073
1074	fini_hash_table(engine);
1075	}
1076
1077	static const struct drm_i915_cmd_descriptor*
1078	find_cmd_in_table(struct intel_engine_cs *engine,
1079	u32 cmd_header)
1080	{
1081	struct cmd_node *desc_node;
1082
1083	hash_for_each_possible(engine->cmd_hash, desc_node, node,
1084	cmd_header_key(cmd_header)) {
1085	const struct drm_i915_cmd_descriptor *desc = desc_node->desc;
1086	if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0)
1087	return desc;
1088	}
1089
1090	return NULL;
1091	}
1092
1093	/*
1094	* Returns a pointer to a descriptor for the command specified by cmd_header.
1095	*
1096	* The caller must supply space for a default descriptor via the default_desc
1097	* parameter. If no descriptor for the specified command exists in the engine's
1098	* command parser tables, this function fills in default_desc based on the
1099	* engine's default length encoding and returns default_desc.
1100	*/
1101	static const struct drm_i915_cmd_descriptor*
1102	find_cmd(struct intel_engine_cs *engine,
1103	u32 cmd_header,
1104	const struct drm_i915_cmd_descriptor *desc,
1105	struct drm_i915_cmd_descriptor *default_desc)
1106	{
1107	u32 mask;
1108
1109	if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0)
1110	return desc;
1111
1112	desc = find_cmd_in_table(engine, cmd_header);
1113	if (desc)
1114	return desc;
1115
1116	mask = engine->get_cmd_length_mask(cmd_header);
1117	if (!mask)
1118	return NULL;
1119
1120	default_desc->cmd.value = cmd_header;
1121	default_desc->cmd.mask = ~0u << MIN_OPCODE_SHIFT;
1122	default_desc->length.mask = mask;
1123	default_desc->flags = CMD_DESC_SKIP;
1124	return default_desc;
1125	}
1126
1127	static const struct drm_i915_reg_descriptor *
1128	__find_reg(const struct drm_i915_reg_descriptor *table, int count, u32 addr)
1129	{
1130	int start = 0, end = count;
1131	while (start < end) {
1132	int mid = start + (end - start) / 2;
1133	int ret = addr - i915_mmio_reg_offset(table[mid].addr);
1134	if (ret < 0)
1135	end = mid;
1136	else if (ret > 0)
1137	start = mid + 1;
1138	else
1139	return &table[mid];
1140	}
1141	return NULL;
1142	}
1143
1144	static const struct drm_i915_reg_descriptor *
1145	find_reg(const struct intel_engine_cs *engine, u32 addr)
1146	{
1147	const struct drm_i915_reg_table *table = engine->reg_tables;
1148	const struct drm_i915_reg_descriptor *reg = NULL;
1149	int count = engine->reg_table_count;
1150
1151	for (; !reg && (count > 0); ++table, --count)
1152	reg = __find_reg(table: table->regs, count: table->num_regs, addr);
1153
1154	return reg;
1155	}
1156
1157	/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
1158	static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
1159	struct drm_i915_gem_object *src_obj,
1160	unsigned long offset, unsigned long length,
1161	bool *needs_clflush_after)
1162	{
1163	unsigned int src_needs_clflush;
1164	unsigned int dst_needs_clflush;
1165	void *dst, *src;
1166	int ret;
1167
1168	ret = i915_gem_object_prepare_write(obj: dst_obj, needs_clflush: &dst_needs_clflush);
1169	if (ret)
1170	return ERR_PTR(error: ret);
1171
1172	dst = i915_gem_object_pin_map(obj: dst_obj, type: I915_MAP_WB);
1173	i915_gem_object_finish_access(obj: dst_obj);
1174	if (IS_ERR(ptr: dst))
1175	return dst;
1176
1177	ret = i915_gem_object_prepare_read(obj: src_obj, needs_clflush: &src_needs_clflush);
1178	if (ret) {
1179	i915_gem_object_unpin_map(obj: dst_obj);
1180	return ERR_PTR(error: ret);
1181	}
1182
1183	src = ERR_PTR(error: -ENODEV);
1184	if (src_needs_clflush && i915_has_memcpy_from_wc()) {
1185	src = i915_gem_object_pin_map(obj: src_obj, type: I915_MAP_WC);
1186	if (!IS_ERR(ptr: src)) {
1187	i915_unaligned_memcpy_from_wc(dst,
1188	src: src + offset,
1189	len: length);
1190	i915_gem_object_unpin_map(obj: src_obj);
1191	}
1192	}
1193	if (IS_ERR(ptr: src)) {
1194	unsigned long x, n, remain;
1195	void *ptr;
1196
1197	/*
1198	* We can avoid clflushing partial cachelines before the write
1199	* if we only every write full cache-lines. Since we know that
1200	* both the source and destination are in multiples of
1201	* PAGE_SIZE, we can simply round up to the next cacheline.
1202	* We don't care about copying too much here as we only
1203	* validate up to the end of the batch.
1204	*/
1205	remain = length;
1206	if (dst_needs_clflush & CLFLUSH_BEFORE)
1207	remain = round_up(remain,
1208	boot_cpu_data.x86_clflush_size);
1209
1210	ptr = dst;
1211	x = offset_in_page(offset);
1212	for (n = offset >> PAGE_SHIFT; remain; n++) {
1213	int len = min(remain, PAGE_SIZE - x);
1214
1215	src = kmap_local_page(i915_gem_object_get_page(src_obj, n));
1216	if (src_needs_clflush)
1217	drm_clflush_virt_range(addr: src + x, length: len);
1218	memcpy(ptr, src + x, len);
1219	kunmap_local(src);
1220
1221	ptr += len;
1222	remain -= len;
1223	x = 0;
1224	}
1225	}
1226
1227	i915_gem_object_finish_access(obj: src_obj);
1228
1229	memset32(s: dst + length, v: 0, n: (dst_obj->base.size - length) / sizeof(u32));
1230
1231	/* dst_obj is returned with vmap pinned */
1232	*needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER;
1233
1234	return dst;
1235	}
1236
1237	static inline bool cmd_desc_is(const struct drm_i915_cmd_descriptor * const desc,
1238	const u32 cmd)
1239	{
1240	return desc->cmd.value == (cmd & desc->cmd.mask);
1241	}
1242
1243	static bool check_cmd(const struct intel_engine_cs *engine,
1244	const struct drm_i915_cmd_descriptor *desc,
1245	const u32 *cmd, u32 length)
1246	{
1247	if (desc->flags & CMD_DESC_SKIP)
1248	return true;
1249
1250	if (desc->flags & CMD_DESC_REJECT) {
1251	DRM_DEBUG("CMD: Rejected command: 0x%08X\n", *cmd);
1252	return false;
1253	}
1254
1255	if (desc->flags & CMD_DESC_REGISTER) {
1256	/*
1257	* Get the distance between individual register offset
1258	* fields if the command can perform more than one
1259	* access at a time.
1260	*/
1261	const u32 step = desc->reg.step ? desc->reg.step : length;
1262	u32 offset;
1263
1264	for (offset = desc->reg.offset; offset < length;
1265	offset += step) {
1266	const u32 reg_addr = cmd[offset] & desc->reg.mask;
1267	const struct drm_i915_reg_descriptor *reg =
1268	find_reg(engine, addr: reg_addr);
1269
1270	if (!reg) {
1271	DRM_DEBUG("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n",
1272	reg_addr, *cmd, engine->name);
1273	return false;
1274	}
1275
1276	/*
1277	* Check the value written to the register against the
1278	* allowed mask/value pair given in the whitelist entry.
1279	*/
1280	if (reg->mask) {
1281	if (cmd_desc_is(desc, MI_LOAD_REGISTER_MEM)) {
1282	DRM_DEBUG("CMD: Rejected LRM to masked register 0x%08X\n",
1283	reg_addr);
1284	return false;
1285	}
1286
1287	if (cmd_desc_is(desc, MI_LOAD_REGISTER_REG)) {
1288	DRM_DEBUG("CMD: Rejected LRR to masked register 0x%08X\n",
1289	reg_addr);
1290	return false;
1291	}
1292
1293	if (cmd_desc_is(desc, MI_LOAD_REGISTER_IMM(1)) &&
1294	(offset + 2 > length ||
1295	(cmd[offset + 1] & reg->mask) != reg->value)) {
1296	DRM_DEBUG("CMD: Rejected LRI to masked register 0x%08X\n",
1297	reg_addr);
1298	return false;
1299	}
1300	}
1301	}
1302	}
1303
1304	if (desc->flags & CMD_DESC_BITMASK) {
1305	int i;
1306
1307	for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) {
1308	u32 dword;
1309
1310	if (desc->bits[i].mask == 0)
1311	break;
1312
1313	if (desc->bits[i].condition_mask != 0) {
1314	u32 offset =
1315	desc->bits[i].condition_offset;
1316	u32 condition = cmd[offset] &
1317	desc->bits[i].condition_mask;
1318
1319	if (condition == 0)
1320	continue;
1321	}
1322
1323	if (desc->bits[i].offset >= length) {
1324	DRM_DEBUG("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n",
1325	*cmd, engine->name);
1326	return false;
1327	}
1328
1329	dword = cmd[desc->bits[i].offset] &
1330	desc->bits[i].mask;
1331
1332	if (dword != desc->bits[i].expected) {
1333	DRM_DEBUG("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n",
1334	*cmd,
1335	desc->bits[i].mask,
1336	desc->bits[i].expected,
1337	dword, engine->name);
1338	return false;
1339	}
1340	}
1341	}
1342
1343	return true;
1344	}
1345
1346	static int check_bbstart(u32 *cmd, u32 offset, u32 length,
1347	u32 batch_length,
1348	u64 batch_addr,
1349	u64 shadow_addr,
1350	const unsigned long *jump_whitelist)
1351	{
1352	u64 jump_offset, jump_target;
1353	u32 target_cmd_offset, target_cmd_index;
1354
1355	/* For igt compatibility on older platforms */
1356	if (!jump_whitelist) {
1357	DRM_DEBUG("CMD: Rejecting BB_START for ggtt based submission\n");
1358	return -EACCES;
1359	}
1360
1361	if (length != 3) {
1362	DRM_DEBUG("CMD: Recursive BB_START with bad length(%u)\n",
1363	length);
1364	return -EINVAL;
1365	}
1366
1367	jump_target = *(u64 *)(cmd + 1);
1368	jump_offset = jump_target - batch_addr;
1369
1370	/*
1371	* Any underflow of jump_target is guaranteed to be outside the range
1372	* of a u32, so >= test catches both too large and too small
1373	*/
1374	if (jump_offset >= batch_length) {
1375	DRM_DEBUG("CMD: BB_START to 0x%llx jumps out of BB\n",
1376	jump_target);
1377	return -EINVAL;
1378	}
1379
1380	/*
1381	* This cannot overflow a u32 because we already checked jump_offset
1382	* is within the BB, and the batch_length is a u32
1383	*/
1384	target_cmd_offset = lower_32_bits(jump_offset);
1385	target_cmd_index = target_cmd_offset / sizeof(u32);
1386
1387	*(u64 *)(cmd + 1) = shadow_addr + target_cmd_offset;
1388
1389	if (target_cmd_index == offset)
1390	return 0;
1391
1392	if (IS_ERR(ptr: jump_whitelist))
1393	return PTR_ERR(ptr: jump_whitelist);
1394
1395	if (!test_bit(target_cmd_index, jump_whitelist)) {
1396	DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",
1397	jump_target);
1398	return -EINVAL;
1399	}
1400
1401	return 0;
1402	}
1403
1404	static unsigned long *alloc_whitelist(u32 batch_length)
1405	{
1406	unsigned long *jmp;
1407
1408	/*
1409	* We expect batch_length to be less than 256KiB for known users,
1410	* i.e. we need at most an 8KiB bitmap allocation which should be
1411	* reasonably cheap due to kmalloc caches.
1412	*/
1413
1414	/* Prefer to report transient allocation failure rather than hit oom */
1415	jmp = bitmap_zalloc(DIV_ROUND_UP(batch_length, sizeof(u32)),
1416	GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
1417	if (!jmp)
1418	return ERR_PTR(error: -ENOMEM);
1419
1420	return jmp;
1421	}
1422
1423	#define LENGTH_BIAS 2
1424
1425	/**
1426	* intel_engine_cmd_parser() - parse a batch buffer for privilege violations
1427	* @engine: the engine on which the batch is to execute
1428	* @batch: the batch buffer in question
1429	* @batch_offset: byte offset in the batch at which execution starts
1430	* @batch_length: length of the commands in batch_obj
1431	* @shadow: validated copy of the batch buffer in question
1432	* @trampoline: true if we need to trampoline into privileged execution
1433	*
1434	* Parses the specified batch buffer looking for privilege violations as
1435	* described in the overview.
1436	*
1437	* Return: non-zero if the parser finds violations or otherwise fails; -EACCES
1438	* if the batch appears legal but should use hardware parsing
1439	*/
1440
1441	int intel_engine_cmd_parser(struct intel_engine_cs *engine,
1442	struct i915_vma *batch,
1443	unsigned long batch_offset,
1444	unsigned long batch_length,
1445	struct i915_vma *shadow,
1446	bool trampoline)
1447	{
1448	u32 *cmd, *batch_end, offset = 0;
1449	struct drm_i915_cmd_descriptor default_desc = noop_desc;
1450	const struct drm_i915_cmd_descriptor *desc = &default_desc;
1451	bool needs_clflush_after = false;
1452	unsigned long *jump_whitelist;
1453	u64 batch_addr, shadow_addr;
1454	int ret = 0;
1455
1456	GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd)));
1457	GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd)));
1458	GEM_BUG_ON(range_overflows_t(u64, batch_offset, batch_length,
1459	batch->size));
1460	GEM_BUG_ON(!batch_length);
1461
1462	cmd = copy_batch(dst_obj: shadow->obj, src_obj: batch->obj,
1463	offset: batch_offset, length: batch_length,
1464	needs_clflush_after: &needs_clflush_after);
1465	if (IS_ERR(ptr: cmd)) {
1466	DRM_DEBUG("CMD: Failed to copy batch\n");
1467	return PTR_ERR(ptr: cmd);
1468	}
1469
1470	jump_whitelist = NULL;
1471	if (!trampoline)
1472	/* Defer failure until attempted use */
1473	jump_whitelist = alloc_whitelist(batch_length);
1474
1475	shadow_addr = gen8_canonical_addr(address: i915_vma_offset(vma: shadow));
1476	batch_addr = gen8_canonical_addr(address: i915_vma_offset(vma: batch) + batch_offset);
1477
1478	/*
1479	* We use the batch length as size because the shadow object is as
1480	* large or larger and copy_batch() will write MI_NOPs to the extra
1481	* space. Parsing should be faster in some cases this way.
1482	*/
1483	batch_end = cmd + batch_length / sizeof(*batch_end);
1484	do {
1485	u32 length;
1486
1487	if (*cmd == MI_BATCH_BUFFER_END)
1488	break;
1489
1490	desc = find_cmd(engine, cmd_header: *cmd, desc, default_desc: &default_desc);
1491	if (!desc) {
1492	DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd);
1493	ret = -EINVAL;
1494	break;
1495	}
1496
1497	if (desc->flags & CMD_DESC_FIXED)
1498	length = desc->length.fixed;
1499	else
1500	length = (*cmd & desc->length.mask) + LENGTH_BIAS;
1501
1502	if ((batch_end - cmd) < length) {
1503	DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n",
1504	*cmd,
1505	length,
1506	batch_end - cmd);
1507	ret = -EINVAL;
1508	break;
1509	}
1510
1511	if (!check_cmd(engine, desc, cmd, length)) {
1512	ret = -EACCES;
1513	break;
1514	}
1515
1516	if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) {
1517	ret = check_bbstart(cmd, offset, length, batch_length,
1518	batch_addr, shadow_addr,
1519	jump_whitelist);
1520	break;
1521	}
1522
1523	if (!IS_ERR_OR_NULL(ptr: jump_whitelist))
1524	__set_bit(offset, jump_whitelist);
1525
1526	cmd += length;
1527	offset += length;
1528	if (cmd >= batch_end) {
1529	DRM_DEBUG("CMD: Got to the end of the buffer w/o a BBE cmd!\n");
1530	ret = -EINVAL;
1531	break;
1532	}
1533	} while (1);
1534
1535	if (trampoline) {
1536	/*
1537	* With the trampoline, the shadow is executed twice.
1538	*
1539	* 1 - starting at offset 0, in privileged mode
1540	* 2 - starting at offset batch_len, as non-privileged
1541	*
1542	* Only if the batch is valid and safe to execute, do we
1543	* allow the first privileged execution to proceed. If not,
1544	* we terminate the first batch and use the second batchbuffer
1545	* entry to chain to the original unsafe non-privileged batch,
1546	* leaving it to the HW to validate.
1547	*/
1548	*batch_end = MI_BATCH_BUFFER_END;
1549
1550	if (ret) {
1551	/* Batch unsafe to execute with privileges, cancel! */
1552	cmd = page_mask_bits(shadow->obj->mm.mapping);
1553	*cmd = MI_BATCH_BUFFER_END;
1554
1555	/* If batch is unsafe but valid, jump to the original */
1556	if (ret == -EACCES) {
1557	unsigned int flags;
1558
1559	flags = MI_BATCH_NON_SECURE_I965;
1560	if (IS_HASWELL(engine->i915))
1561	flags = MI_BATCH_NON_SECURE_HSW;
1562
1563	GEM_BUG_ON(!IS_GRAPHICS_VER(engine->i915, 6, 7));
1564	__gen6_emit_bb_start(cs: batch_end,
1565	addr: batch_addr,
1566	flags);
1567
1568	ret = 0; /* allow execution */
1569	}
1570	}
1571	}
1572
1573	i915_gem_object_flush_map(obj: shadow->obj);
1574
1575	if (!IS_ERR_OR_NULL(ptr: jump_whitelist))
1576	kfree(objp: jump_whitelist);
1577	i915_gem_object_unpin_map(obj: shadow->obj);
1578	return ret;
1579	}
1580
1581	/**
1582	* i915_cmd_parser_get_version() - get the cmd parser version number
1583	* @dev_priv: i915 device private
1584	*
1585	* The cmd parser maintains a simple increasing integer version number suitable
1586	* for passing to userspace clients to determine what operations are permitted.
1587	*
1588	* Return: the current version number of the cmd parser
1589	*/
1590	int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
1591	{
1592	struct intel_engine_cs *engine;
1593	bool active = false;
1594
1595	/* If the command parser is not enabled, report 0 - unsupported */
1596	for_each_uabi_engine(engine, dev_priv) {
1597	if (intel_engine_using_cmd_parser(engine)) {
1598	active = true;
1599	break;
1600	}
1601	}
1602	if (!active)
1603	return 0;
1604
1605	/*
1606	* Command parser version history
1607	*
1608	* 1. Initial version. Checks batches and reports violations, but leaves
1609	* hardware parsing enabled (so does not allow new use cases).
1610	* 2. Allow access to the MI_PREDICATE_SRC0 and
1611	* MI_PREDICATE_SRC1 registers.
1612	* 3. Allow access to the GPGPU_THREADS_DISPATCHED register.
1613	* 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3.
1614	* 5. GPGPU dispatch compute indirect registers.
1615	* 6. TIMESTAMP register and Haswell CS GPR registers
1616	* 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers.
1617	* 8. Don't report cmd_check() failures as EINVAL errors to userspace;
1618	* rely on the HW to NOOP disallowed commands as it would without
1619	* the parser enabled.
1620	* 9. Don't whitelist or handle oacontrol specially, as ownership
1621	* for oacontrol state is moving to i915-perf.
1622	* 10. Support for Gen9 BCS Parsing
1623	*/
1624	return 10;
1625	}
1626