1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2018 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/prime_numbers.h> |
7 | |
8 | #include "gem/i915_gem_internal.h" |
9 | |
10 | #include "i915_selftest.h" |
11 | #include "intel_engine_heartbeat.h" |
12 | #include "intel_engine_pm.h" |
13 | #include "intel_reset.h" |
14 | #include "intel_ring.h" |
15 | #include "selftest_engine_heartbeat.h" |
16 | #include "selftests/i915_random.h" |
17 | #include "selftests/igt_flush_test.h" |
18 | #include "selftests/igt_live_test.h" |
19 | #include "selftests/igt_spinner.h" |
20 | #include "selftests/lib_sw_fence.h" |
21 | #include "shmem_utils.h" |
22 | |
23 | #include "gem/selftests/igt_gem_utils.h" |
24 | #include "gem/selftests/mock_context.h" |
25 | |
26 | #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) |
27 | #define NUM_GPR 16 |
28 | #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ |
29 | |
30 | #define MI_INSTR(0x22, 0) |
31 | #define LRI_LENGTH_MASK GENMASK(7, 0) |
32 | |
33 | static struct i915_vma *create_scratch(struct intel_gt *gt) |
34 | { |
35 | return __vm_create_scratch_for_read_pinned(vm: >->ggtt->vm, PAGE_SIZE); |
36 | } |
37 | |
38 | static bool is_active(struct i915_request *rq) |
39 | { |
40 | if (i915_request_is_active(rq)) |
41 | return true; |
42 | |
43 | if (i915_request_on_hold(rq)) |
44 | return true; |
45 | |
46 | if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) |
47 | return true; |
48 | |
49 | return false; |
50 | } |
51 | |
52 | static int wait_for_submit(struct intel_engine_cs *engine, |
53 | struct i915_request *rq, |
54 | unsigned long timeout) |
55 | { |
56 | /* Ignore our own attempts to suppress excess tasklets */ |
57 | tasklet_hi_schedule(t: &engine->sched_engine->tasklet); |
58 | |
59 | timeout += jiffies; |
60 | do { |
61 | bool done = time_after(jiffies, timeout); |
62 | |
63 | if (i915_request_completed(rq)) /* that was quick! */ |
64 | return 0; |
65 | |
66 | /* Wait until the HW has acknowleged the submission (or err) */ |
67 | intel_engine_flush_submission(engine); |
68 | if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) |
69 | return 0; |
70 | |
71 | if (done) |
72 | return -ETIME; |
73 | |
74 | cond_resched(); |
75 | } while (1); |
76 | } |
77 | |
78 | static int emit_semaphore_signal(struct intel_context *ce, void *slot) |
79 | { |
80 | const u32 offset = |
81 | i915_ggtt_offset(vma: ce->engine->status_page.vma) + |
82 | offset_in_page(slot); |
83 | struct i915_request *rq; |
84 | u32 *cs; |
85 | |
86 | rq = intel_context_create_request(ce); |
87 | if (IS_ERR(ptr: rq)) |
88 | return PTR_ERR(ptr: rq); |
89 | |
90 | cs = intel_ring_begin(rq, num_dwords: 4); |
91 | if (IS_ERR(ptr: cs)) { |
92 | i915_request_add(rq); |
93 | return PTR_ERR(ptr: cs); |
94 | } |
95 | |
96 | *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; |
97 | *cs++ = offset; |
98 | *cs++ = 0; |
99 | *cs++ = 1; |
100 | |
101 | intel_ring_advance(rq, cs); |
102 | |
103 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; |
104 | i915_request_add(rq); |
105 | return 0; |
106 | } |
107 | |
108 | static int context_flush(struct intel_context *ce, long timeout) |
109 | { |
110 | struct i915_request *rq; |
111 | struct dma_fence *fence; |
112 | int err = 0; |
113 | |
114 | rq = intel_engine_create_kernel_request(engine: ce->engine); |
115 | if (IS_ERR(ptr: rq)) |
116 | return PTR_ERR(ptr: rq); |
117 | |
118 | fence = i915_active_fence_get(active: &ce->timeline->last_request); |
119 | if (fence) { |
120 | i915_request_await_dma_fence(rq, fence); |
121 | dma_fence_put(fence); |
122 | } |
123 | |
124 | rq = i915_request_get(rq); |
125 | i915_request_add(rq); |
126 | if (i915_request_wait(rq, flags: 0, timeout) < 0) |
127 | err = -ETIME; |
128 | i915_request_put(rq); |
129 | |
130 | rmb(); /* We know the request is written, make sure all state is too! */ |
131 | return err; |
132 | } |
133 | |
134 | static int get_lri_mask(struct intel_engine_cs *engine, u32 lri) |
135 | { |
136 | if ((lri & MI_LRI_LRM_CS_MMIO) == 0) |
137 | return ~0u; |
138 | |
139 | if (GRAPHICS_VER(engine->i915) < 12) |
140 | return 0xfff; |
141 | |
142 | switch (engine->class) { |
143 | default: |
144 | case RENDER_CLASS: |
145 | case COMPUTE_CLASS: |
146 | return 0x07ff; |
147 | case COPY_ENGINE_CLASS: |
148 | return 0x0fff; |
149 | case VIDEO_DECODE_CLASS: |
150 | case VIDEO_ENHANCEMENT_CLASS: |
151 | return 0x3fff; |
152 | } |
153 | } |
154 | |
155 | static int live_lrc_layout(void *arg) |
156 | { |
157 | struct intel_gt *gt = arg; |
158 | struct intel_engine_cs *engine; |
159 | enum intel_engine_id id; |
160 | u32 *lrc; |
161 | int err; |
162 | |
163 | /* |
164 | * Check the registers offsets we use to create the initial reg state |
165 | * match the layout saved by HW. |
166 | */ |
167 | |
168 | lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ |
169 | if (!lrc) |
170 | return -ENOMEM; |
171 | GEM_BUG_ON(offset_in_page(lrc)); |
172 | |
173 | err = 0; |
174 | for_each_engine(engine, gt, id) { |
175 | u32 *hw; |
176 | int dw; |
177 | |
178 | if (!engine->default_state) |
179 | continue; |
180 | |
181 | hw = shmem_pin_map(file: engine->default_state); |
182 | if (!hw) { |
183 | err = -ENOMEM; |
184 | break; |
185 | } |
186 | hw += LRC_STATE_OFFSET / sizeof(*hw); |
187 | |
188 | __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), |
189 | ce: engine->kernel_context, engine, inhibit: true); |
190 | |
191 | dw = 0; |
192 | do { |
193 | u32 lri = READ_ONCE(hw[dw]); |
194 | u32 lri_mask; |
195 | |
196 | if (lri == 0) { |
197 | dw++; |
198 | continue; |
199 | } |
200 | |
201 | if (lrc[dw] == 0) { |
202 | pr_debug("%s: skipped instruction %x at dword %d\n" , |
203 | engine->name, lri, dw); |
204 | dw++; |
205 | continue; |
206 | } |
207 | |
208 | if ((lri & GENMASK(31, 23)) != LRI_HEADER) { |
209 | pr_err("%s: Expected LRI command at dword %d, found %08x\n" , |
210 | engine->name, dw, lri); |
211 | err = -EINVAL; |
212 | break; |
213 | } |
214 | |
215 | if (lrc[dw] != lri) { |
216 | pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n" , |
217 | engine->name, dw, lri, lrc[dw]); |
218 | err = -EINVAL; |
219 | break; |
220 | } |
221 | |
222 | /* |
223 | * When bit 19 of MI_LOAD_REGISTER_IMM instruction |
224 | * opcode is set on Gen12+ devices, HW does not |
225 | * care about certain register address offsets, and |
226 | * instead check the following for valid address |
227 | * ranges on specific engines: |
228 | * RCS && CCS: BITS(0 - 10) |
229 | * BCS: BITS(0 - 11) |
230 | * VECS && VCS: BITS(0 - 13) |
231 | */ |
232 | lri_mask = get_lri_mask(engine, lri); |
233 | |
234 | lri &= 0x7f; |
235 | lri++; |
236 | dw++; |
237 | |
238 | while (lri) { |
239 | u32 offset = READ_ONCE(hw[dw]); |
240 | |
241 | if ((offset ^ lrc[dw]) & lri_mask) { |
242 | pr_err("%s: Different registers found at dword %d, expected %x, found %x\n" , |
243 | engine->name, dw, offset, lrc[dw]); |
244 | err = -EINVAL; |
245 | break; |
246 | } |
247 | |
248 | /* |
249 | * Skip over the actual register value as we |
250 | * expect that to differ. |
251 | */ |
252 | dw += 2; |
253 | lri -= 2; |
254 | } |
255 | } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); |
256 | |
257 | if (err) { |
258 | pr_info("%s: HW register image:\n" , engine->name); |
259 | igt_hexdump(buf: hw, PAGE_SIZE); |
260 | |
261 | pr_info("%s: SW register image:\n" , engine->name); |
262 | igt_hexdump(buf: lrc, PAGE_SIZE); |
263 | } |
264 | |
265 | shmem_unpin_map(file: engine->default_state, ptr: hw); |
266 | if (err) |
267 | break; |
268 | } |
269 | |
270 | free_page((unsigned long)lrc); |
271 | return err; |
272 | } |
273 | |
274 | static int find_offset(const u32 *lri, u32 offset) |
275 | { |
276 | int i; |
277 | |
278 | for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) |
279 | if (lri[i] == offset) |
280 | return i; |
281 | |
282 | return -1; |
283 | } |
284 | |
285 | static int live_lrc_fixed(void *arg) |
286 | { |
287 | struct intel_gt *gt = arg; |
288 | struct intel_engine_cs *engine; |
289 | enum intel_engine_id id; |
290 | int err = 0; |
291 | |
292 | /* |
293 | * Check the assumed register offsets match the actual locations in |
294 | * the context image. |
295 | */ |
296 | |
297 | for_each_engine(engine, gt, id) { |
298 | const struct { |
299 | u32 reg; |
300 | u32 offset; |
301 | const char *name; |
302 | } tbl[] = { |
303 | { |
304 | i915_mmio_reg_offset(RING_START(engine->mmio_base)), |
305 | CTX_RING_START - 1, |
306 | "RING_START" |
307 | }, |
308 | { |
309 | i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), |
310 | CTX_RING_CTL - 1, |
311 | "RING_CTL" |
312 | }, |
313 | { |
314 | i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), |
315 | CTX_RING_HEAD - 1, |
316 | "RING_HEAD" |
317 | }, |
318 | { |
319 | i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), |
320 | CTX_RING_TAIL - 1, |
321 | "RING_TAIL" |
322 | }, |
323 | { |
324 | i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), |
325 | lrc_ring_mi_mode(engine), |
326 | "RING_MI_MODE" |
327 | }, |
328 | { |
329 | i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), |
330 | CTX_BB_STATE - 1, |
331 | "BB_STATE" |
332 | }, |
333 | { |
334 | i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), |
335 | lrc_ring_wa_bb_per_ctx(engine), |
336 | "RING_BB_PER_CTX_PTR" |
337 | }, |
338 | { |
339 | i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), |
340 | lrc_ring_indirect_ptr(engine), |
341 | "RING_INDIRECT_CTX_PTR" |
342 | }, |
343 | { |
344 | i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), |
345 | lrc_ring_indirect_offset(engine), |
346 | "RING_INDIRECT_CTX_OFFSET" |
347 | }, |
348 | { |
349 | i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), |
350 | CTX_TIMESTAMP - 1, |
351 | "RING_CTX_TIMESTAMP" |
352 | }, |
353 | { |
354 | i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), |
355 | lrc_ring_gpr0(engine), |
356 | "RING_CS_GPR0" |
357 | }, |
358 | { |
359 | i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), |
360 | lrc_ring_cmd_buf_cctl(engine), |
361 | "RING_CMD_BUF_CCTL" |
362 | }, |
363 | { |
364 | i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)), |
365 | lrc_ring_bb_offset(engine), |
366 | "RING_BB_OFFSET" |
367 | }, |
368 | { }, |
369 | }, *t; |
370 | u32 *hw; |
371 | |
372 | if (!engine->default_state) |
373 | continue; |
374 | |
375 | hw = shmem_pin_map(file: engine->default_state); |
376 | if (!hw) { |
377 | err = -ENOMEM; |
378 | break; |
379 | } |
380 | hw += LRC_STATE_OFFSET / sizeof(*hw); |
381 | |
382 | for (t = tbl; t->name; t++) { |
383 | int dw = find_offset(lri: hw, offset: t->reg); |
384 | |
385 | if (dw != t->offset) { |
386 | pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n" , |
387 | engine->name, |
388 | t->name, |
389 | t->reg, |
390 | dw, |
391 | t->offset); |
392 | err = -EINVAL; |
393 | } |
394 | } |
395 | |
396 | shmem_unpin_map(file: engine->default_state, ptr: hw); |
397 | } |
398 | |
399 | return err; |
400 | } |
401 | |
402 | static int __live_lrc_state(struct intel_engine_cs *engine, |
403 | struct i915_vma *scratch) |
404 | { |
405 | struct intel_context *ce; |
406 | struct i915_request *rq; |
407 | struct i915_gem_ww_ctx ww; |
408 | enum { |
409 | RING_START_IDX = 0, |
410 | RING_TAIL_IDX, |
411 | MAX_IDX |
412 | }; |
413 | u32 expected[MAX_IDX]; |
414 | u32 *cs; |
415 | int err; |
416 | int n; |
417 | |
418 | ce = intel_context_create(engine); |
419 | if (IS_ERR(ptr: ce)) |
420 | return PTR_ERR(ptr: ce); |
421 | |
422 | i915_gem_ww_ctx_init(ctx: &ww, intr: false); |
423 | retry: |
424 | err = i915_gem_object_lock(obj: scratch->obj, ww: &ww); |
425 | if (!err) |
426 | err = intel_context_pin_ww(ce, ww: &ww); |
427 | if (err) |
428 | goto err_put; |
429 | |
430 | rq = i915_request_create(ce); |
431 | if (IS_ERR(ptr: rq)) { |
432 | err = PTR_ERR(ptr: rq); |
433 | goto err_unpin; |
434 | } |
435 | |
436 | cs = intel_ring_begin(rq, num_dwords: 4 * MAX_IDX); |
437 | if (IS_ERR(ptr: cs)) { |
438 | err = PTR_ERR(ptr: cs); |
439 | i915_request_add(rq); |
440 | goto err_unpin; |
441 | } |
442 | |
443 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; |
444 | *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); |
445 | *cs++ = i915_ggtt_offset(vma: scratch) + RING_START_IDX * sizeof(u32); |
446 | *cs++ = 0; |
447 | |
448 | expected[RING_START_IDX] = i915_ggtt_offset(vma: ce->ring->vma); |
449 | |
450 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; |
451 | *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); |
452 | *cs++ = i915_ggtt_offset(vma: scratch) + RING_TAIL_IDX * sizeof(u32); |
453 | *cs++ = 0; |
454 | |
455 | err = i915_vma_move_to_active(vma: scratch, rq, EXEC_OBJECT_WRITE); |
456 | |
457 | i915_request_get(rq); |
458 | i915_request_add(rq); |
459 | if (err) |
460 | goto err_rq; |
461 | |
462 | intel_engine_flush_submission(engine); |
463 | expected[RING_TAIL_IDX] = ce->ring->tail; |
464 | |
465 | if (i915_request_wait(rq, flags: 0, HZ / 5) < 0) { |
466 | err = -ETIME; |
467 | goto err_rq; |
468 | } |
469 | |
470 | cs = i915_gem_object_pin_map(obj: scratch->obj, type: I915_MAP_WB); |
471 | if (IS_ERR(ptr: cs)) { |
472 | err = PTR_ERR(ptr: cs); |
473 | goto err_rq; |
474 | } |
475 | |
476 | for (n = 0; n < MAX_IDX; n++) { |
477 | if (cs[n] != expected[n]) { |
478 | pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n" , |
479 | engine->name, n, cs[n], expected[n]); |
480 | err = -EINVAL; |
481 | break; |
482 | } |
483 | } |
484 | |
485 | i915_gem_object_unpin_map(obj: scratch->obj); |
486 | |
487 | err_rq: |
488 | i915_request_put(rq); |
489 | err_unpin: |
490 | intel_context_unpin(ce); |
491 | err_put: |
492 | if (err == -EDEADLK) { |
493 | err = i915_gem_ww_ctx_backoff(ctx: &ww); |
494 | if (!err) |
495 | goto retry; |
496 | } |
497 | i915_gem_ww_ctx_fini(ctx: &ww); |
498 | intel_context_put(ce); |
499 | return err; |
500 | } |
501 | |
502 | static int live_lrc_state(void *arg) |
503 | { |
504 | struct intel_gt *gt = arg; |
505 | struct intel_engine_cs *engine; |
506 | struct i915_vma *scratch; |
507 | enum intel_engine_id id; |
508 | int err = 0; |
509 | |
510 | /* |
511 | * Check the live register state matches what we expect for this |
512 | * intel_context. |
513 | */ |
514 | |
515 | scratch = create_scratch(gt); |
516 | if (IS_ERR(ptr: scratch)) |
517 | return PTR_ERR(ptr: scratch); |
518 | |
519 | for_each_engine(engine, gt, id) { |
520 | err = __live_lrc_state(engine, scratch); |
521 | if (err) |
522 | break; |
523 | } |
524 | |
525 | if (igt_flush_test(i915: gt->i915)) |
526 | err = -EIO; |
527 | |
528 | i915_vma_unpin_and_release(p_vma: &scratch, flags: 0); |
529 | return err; |
530 | } |
531 | |
532 | static int gpr_make_dirty(struct intel_context *ce) |
533 | { |
534 | struct i915_request *rq; |
535 | u32 *cs; |
536 | int n; |
537 | |
538 | rq = intel_context_create_request(ce); |
539 | if (IS_ERR(ptr: rq)) |
540 | return PTR_ERR(ptr: rq); |
541 | |
542 | cs = intel_ring_begin(rq, num_dwords: 2 * NUM_GPR_DW + 2); |
543 | if (IS_ERR(ptr: cs)) { |
544 | i915_request_add(rq); |
545 | return PTR_ERR(ptr: cs); |
546 | } |
547 | |
548 | *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); |
549 | for (n = 0; n < NUM_GPR_DW; n++) { |
550 | *cs++ = CS_GPR(ce->engine, n); |
551 | *cs++ = STACK_MAGIC; |
552 | } |
553 | *cs++ = MI_NOOP; |
554 | |
555 | intel_ring_advance(rq, cs); |
556 | |
557 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; |
558 | i915_request_add(rq); |
559 | |
560 | return 0; |
561 | } |
562 | |
563 | static struct i915_request * |
564 | __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) |
565 | { |
566 | const u32 offset = |
567 | i915_ggtt_offset(vma: ce->engine->status_page.vma) + |
568 | offset_in_page(slot); |
569 | struct i915_request *rq; |
570 | u32 *cs; |
571 | int err; |
572 | int n; |
573 | |
574 | rq = intel_context_create_request(ce); |
575 | if (IS_ERR(ptr: rq)) |
576 | return rq; |
577 | |
578 | cs = intel_ring_begin(rq, num_dwords: 6 + 4 * NUM_GPR_DW); |
579 | if (IS_ERR(ptr: cs)) { |
580 | i915_request_add(rq); |
581 | return ERR_CAST(ptr: cs); |
582 | } |
583 | |
584 | *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; |
585 | *cs++ = MI_NOOP; |
586 | |
587 | *cs++ = MI_SEMAPHORE_WAIT | |
588 | MI_SEMAPHORE_GLOBAL_GTT | |
589 | MI_SEMAPHORE_POLL | |
590 | MI_SEMAPHORE_SAD_NEQ_SDD; |
591 | *cs++ = 0; |
592 | *cs++ = offset; |
593 | *cs++ = 0; |
594 | |
595 | for (n = 0; n < NUM_GPR_DW; n++) { |
596 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; |
597 | *cs++ = CS_GPR(ce->engine, n); |
598 | *cs++ = i915_ggtt_offset(vma: scratch) + n * sizeof(u32); |
599 | *cs++ = 0; |
600 | } |
601 | |
602 | err = igt_vma_move_to_active_unlocked(vma: scratch, rq, EXEC_OBJECT_WRITE); |
603 | |
604 | i915_request_get(rq); |
605 | i915_request_add(rq); |
606 | if (err) { |
607 | i915_request_put(rq); |
608 | rq = ERR_PTR(error: err); |
609 | } |
610 | |
611 | return rq; |
612 | } |
613 | |
614 | static int __live_lrc_gpr(struct intel_engine_cs *engine, |
615 | struct i915_vma *scratch, |
616 | bool preempt) |
617 | { |
618 | u32 *slot = memset32(s: engine->status_page.addr + 1000, v: 0, n: 4); |
619 | struct intel_context *ce; |
620 | struct i915_request *rq; |
621 | u32 *cs; |
622 | int err; |
623 | int n; |
624 | |
625 | if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS) |
626 | return 0; /* GPR only on rcs0 for gen8 */ |
627 | |
628 | err = gpr_make_dirty(ce: engine->kernel_context); |
629 | if (err) |
630 | return err; |
631 | |
632 | ce = intel_context_create(engine); |
633 | if (IS_ERR(ptr: ce)) |
634 | return PTR_ERR(ptr: ce); |
635 | |
636 | rq = __gpr_read(ce, scratch, slot); |
637 | if (IS_ERR(ptr: rq)) { |
638 | err = PTR_ERR(ptr: rq); |
639 | goto err_put; |
640 | } |
641 | |
642 | err = wait_for_submit(engine, rq, HZ / 2); |
643 | if (err) |
644 | goto err_rq; |
645 | |
646 | if (preempt) { |
647 | err = gpr_make_dirty(ce: engine->kernel_context); |
648 | if (err) |
649 | goto err_rq; |
650 | |
651 | err = emit_semaphore_signal(ce: engine->kernel_context, slot); |
652 | if (err) |
653 | goto err_rq; |
654 | |
655 | err = wait_for_submit(engine, rq, HZ / 2); |
656 | if (err) |
657 | goto err_rq; |
658 | } else { |
659 | slot[0] = 1; |
660 | wmb(); |
661 | } |
662 | |
663 | if (i915_request_wait(rq, flags: 0, HZ / 5) < 0) { |
664 | err = -ETIME; |
665 | goto err_rq; |
666 | } |
667 | |
668 | cs = i915_gem_object_pin_map_unlocked(obj: scratch->obj, type: I915_MAP_WB); |
669 | if (IS_ERR(ptr: cs)) { |
670 | err = PTR_ERR(ptr: cs); |
671 | goto err_rq; |
672 | } |
673 | |
674 | for (n = 0; n < NUM_GPR_DW; n++) { |
675 | if (cs[n]) { |
676 | pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n" , |
677 | engine->name, |
678 | n / 2, n & 1 ? "udw" : "ldw" , |
679 | cs[n]); |
680 | err = -EINVAL; |
681 | break; |
682 | } |
683 | } |
684 | |
685 | i915_gem_object_unpin_map(obj: scratch->obj); |
686 | |
687 | err_rq: |
688 | memset32(s: &slot[0], v: -1, n: 4); |
689 | wmb(); |
690 | i915_request_put(rq); |
691 | err_put: |
692 | intel_context_put(ce); |
693 | return err; |
694 | } |
695 | |
696 | static int live_lrc_gpr(void *arg) |
697 | { |
698 | struct intel_gt *gt = arg; |
699 | struct intel_engine_cs *engine; |
700 | struct i915_vma *scratch; |
701 | enum intel_engine_id id; |
702 | int err = 0; |
703 | |
704 | /* |
705 | * Check that GPR registers are cleared in new contexts as we need |
706 | * to avoid leaking any information from previous contexts. |
707 | */ |
708 | |
709 | scratch = create_scratch(gt); |
710 | if (IS_ERR(ptr: scratch)) |
711 | return PTR_ERR(ptr: scratch); |
712 | |
713 | for_each_engine(engine, gt, id) { |
714 | st_engine_heartbeat_disable(engine); |
715 | |
716 | err = __live_lrc_gpr(engine, scratch, preempt: false); |
717 | if (err) |
718 | goto err; |
719 | |
720 | err = __live_lrc_gpr(engine, scratch, preempt: true); |
721 | if (err) |
722 | goto err; |
723 | |
724 | err: |
725 | st_engine_heartbeat_enable(engine); |
726 | if (igt_flush_test(i915: gt->i915)) |
727 | err = -EIO; |
728 | if (err) |
729 | break; |
730 | } |
731 | |
732 | i915_vma_unpin_and_release(p_vma: &scratch, flags: 0); |
733 | return err; |
734 | } |
735 | |
736 | static struct i915_request * |
737 | create_timestamp(struct intel_context *ce, void *slot, int idx) |
738 | { |
739 | const u32 offset = |
740 | i915_ggtt_offset(vma: ce->engine->status_page.vma) + |
741 | offset_in_page(slot); |
742 | struct i915_request *rq; |
743 | u32 *cs; |
744 | int err; |
745 | |
746 | rq = intel_context_create_request(ce); |
747 | if (IS_ERR(ptr: rq)) |
748 | return rq; |
749 | |
750 | cs = intel_ring_begin(rq, num_dwords: 10); |
751 | if (IS_ERR(ptr: cs)) { |
752 | err = PTR_ERR(ptr: cs); |
753 | goto err; |
754 | } |
755 | |
756 | *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; |
757 | *cs++ = MI_NOOP; |
758 | |
759 | *cs++ = MI_SEMAPHORE_WAIT | |
760 | MI_SEMAPHORE_GLOBAL_GTT | |
761 | MI_SEMAPHORE_POLL | |
762 | MI_SEMAPHORE_SAD_NEQ_SDD; |
763 | *cs++ = 0; |
764 | *cs++ = offset; |
765 | *cs++ = 0; |
766 | |
767 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; |
768 | *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); |
769 | *cs++ = offset + idx * sizeof(u32); |
770 | *cs++ = 0; |
771 | |
772 | intel_ring_advance(rq, cs); |
773 | |
774 | err = 0; |
775 | err: |
776 | i915_request_get(rq); |
777 | i915_request_add(rq); |
778 | if (err) { |
779 | i915_request_put(rq); |
780 | return ERR_PTR(error: err); |
781 | } |
782 | |
783 | return rq; |
784 | } |
785 | |
786 | struct lrc_timestamp { |
787 | struct intel_engine_cs *engine; |
788 | struct intel_context *ce[2]; |
789 | u32 poison; |
790 | }; |
791 | |
792 | static bool timestamp_advanced(u32 start, u32 end) |
793 | { |
794 | return (s32)(end - start) > 0; |
795 | } |
796 | |
797 | static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) |
798 | { |
799 | u32 *slot = memset32(s: arg->engine->status_page.addr + 1000, v: 0, n: 4); |
800 | struct i915_request *rq; |
801 | u32 timestamp; |
802 | int err = 0; |
803 | |
804 | arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; |
805 | rq = create_timestamp(ce: arg->ce[0], slot, idx: 1); |
806 | if (IS_ERR(ptr: rq)) |
807 | return PTR_ERR(ptr: rq); |
808 | |
809 | err = wait_for_submit(engine: rq->engine, rq, HZ / 2); |
810 | if (err) |
811 | goto err; |
812 | |
813 | if (preempt) { |
814 | arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; |
815 | err = emit_semaphore_signal(ce: arg->ce[1], slot); |
816 | if (err) |
817 | goto err; |
818 | } else { |
819 | slot[0] = 1; |
820 | wmb(); |
821 | } |
822 | |
823 | /* And wait for switch to kernel (to save our context to memory) */ |
824 | err = context_flush(ce: arg->ce[0], HZ / 2); |
825 | if (err) |
826 | goto err; |
827 | |
828 | if (!timestamp_advanced(start: arg->poison, end: slot[1])) { |
829 | pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n" , |
830 | arg->engine->name, preempt ? "preempt" : "simple" , |
831 | arg->poison, slot[1]); |
832 | err = -EINVAL; |
833 | } |
834 | |
835 | timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); |
836 | if (!timestamp_advanced(start: slot[1], end: timestamp)) { |
837 | pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n" , |
838 | arg->engine->name, preempt ? "preempt" : "simple" , |
839 | slot[1], timestamp); |
840 | err = -EINVAL; |
841 | } |
842 | |
843 | err: |
844 | memset32(s: slot, v: -1, n: 4); |
845 | i915_request_put(rq); |
846 | return err; |
847 | } |
848 | |
849 | static int live_lrc_timestamp(void *arg) |
850 | { |
851 | struct lrc_timestamp data = {}; |
852 | struct intel_gt *gt = arg; |
853 | enum intel_engine_id id; |
854 | const u32 poison[] = { |
855 | 0, |
856 | S32_MAX, |
857 | (u32)S32_MAX + 1, |
858 | U32_MAX, |
859 | }; |
860 | |
861 | /* |
862 | * We want to verify that the timestamp is saved and restore across |
863 | * context switches and is monotonic. |
864 | * |
865 | * So we do this with a little bit of LRC poisoning to check various |
866 | * boundary conditions, and see what happens if we preempt the context |
867 | * with a second request (carrying more poison into the timestamp). |
868 | */ |
869 | |
870 | for_each_engine(data.engine, gt, id) { |
871 | int i, err = 0; |
872 | |
873 | st_engine_heartbeat_disable(engine: data.engine); |
874 | |
875 | for (i = 0; i < ARRAY_SIZE(data.ce); i++) { |
876 | struct intel_context *tmp; |
877 | |
878 | tmp = intel_context_create(engine: data.engine); |
879 | if (IS_ERR(ptr: tmp)) { |
880 | err = PTR_ERR(ptr: tmp); |
881 | goto err; |
882 | } |
883 | |
884 | err = intel_context_pin(ce: tmp); |
885 | if (err) { |
886 | intel_context_put(ce: tmp); |
887 | goto err; |
888 | } |
889 | |
890 | data.ce[i] = tmp; |
891 | } |
892 | |
893 | for (i = 0; i < ARRAY_SIZE(poison); i++) { |
894 | data.poison = poison[i]; |
895 | |
896 | err = __lrc_timestamp(arg: &data, preempt: false); |
897 | if (err) |
898 | break; |
899 | |
900 | err = __lrc_timestamp(arg: &data, preempt: true); |
901 | if (err) |
902 | break; |
903 | } |
904 | |
905 | err: |
906 | st_engine_heartbeat_enable(engine: data.engine); |
907 | for (i = 0; i < ARRAY_SIZE(data.ce); i++) { |
908 | if (!data.ce[i]) |
909 | break; |
910 | |
911 | intel_context_unpin(ce: data.ce[i]); |
912 | intel_context_put(ce: data.ce[i]); |
913 | } |
914 | |
915 | if (igt_flush_test(i915: gt->i915)) |
916 | err = -EIO; |
917 | if (err) |
918 | return err; |
919 | } |
920 | |
921 | return 0; |
922 | } |
923 | |
924 | static struct i915_vma * |
925 | create_user_vma(struct i915_address_space *vm, unsigned long size) |
926 | { |
927 | struct drm_i915_gem_object *obj; |
928 | struct i915_vma *vma; |
929 | int err; |
930 | |
931 | obj = i915_gem_object_create_internal(i915: vm->i915, size); |
932 | if (IS_ERR(ptr: obj)) |
933 | return ERR_CAST(ptr: obj); |
934 | |
935 | vma = i915_vma_instance(obj, vm, NULL); |
936 | if (IS_ERR(ptr: vma)) { |
937 | i915_gem_object_put(obj); |
938 | return vma; |
939 | } |
940 | |
941 | err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_USER); |
942 | if (err) { |
943 | i915_gem_object_put(obj); |
944 | return ERR_PTR(error: err); |
945 | } |
946 | |
947 | return vma; |
948 | } |
949 | |
950 | static u32 safe_poison(u32 offset, u32 poison) |
951 | { |
952 | /* |
953 | * Do not enable predication as it will nop all subsequent commands, |
954 | * not only disabling the tests (by preventing all the other SRM) but |
955 | * also preventing the arbitration events at the end of the request. |
956 | */ |
957 | if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0))) |
958 | poison &= ~REG_BIT(0); |
959 | |
960 | return poison; |
961 | } |
962 | |
963 | static struct i915_vma * |
964 | store_context(struct intel_context *ce, struct i915_vma *scratch) |
965 | { |
966 | struct i915_vma *batch; |
967 | u32 dw, x, *cs, *hw; |
968 | u32 *defaults; |
969 | |
970 | batch = create_user_vma(vm: ce->vm, SZ_64K); |
971 | if (IS_ERR(ptr: batch)) |
972 | return batch; |
973 | |
974 | cs = i915_gem_object_pin_map_unlocked(obj: batch->obj, type: I915_MAP_WC); |
975 | if (IS_ERR(ptr: cs)) { |
976 | i915_vma_put(vma: batch); |
977 | return ERR_CAST(ptr: cs); |
978 | } |
979 | |
980 | defaults = shmem_pin_map(file: ce->engine->default_state); |
981 | if (!defaults) { |
982 | i915_gem_object_unpin_map(obj: batch->obj); |
983 | i915_vma_put(vma: batch); |
984 | return ERR_PTR(error: -ENOMEM); |
985 | } |
986 | |
987 | x = 0; |
988 | dw = 0; |
989 | hw = defaults; |
990 | hw += LRC_STATE_OFFSET / sizeof(*hw); |
991 | do { |
992 | u32 len = hw[dw] & LRI_LENGTH_MASK; |
993 | |
994 | /* |
995 | * Keep it simple, skip parsing complex commands |
996 | * |
997 | * At present, there are no more MI_LOAD_REGISTER_IMM |
998 | * commands after the first 3D state command. Rather |
999 | * than include a table (see i915_cmd_parser.c) of all |
1000 | * the possible commands and their instruction lengths |
1001 | * (or mask for variable length instructions), assume |
1002 | * we have gathered the complete list of registers and |
1003 | * bail out. |
1004 | */ |
1005 | if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT) |
1006 | break; |
1007 | |
1008 | if (hw[dw] == 0) { |
1009 | dw++; |
1010 | continue; |
1011 | } |
1012 | |
1013 | if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) { |
1014 | /* Assume all other MI commands match LRI length mask */ |
1015 | dw += len + 2; |
1016 | continue; |
1017 | } |
1018 | |
1019 | if (!len) { |
1020 | pr_err("%s: invalid LRI found in context image\n" , |
1021 | ce->engine->name); |
1022 | igt_hexdump(buf: defaults, PAGE_SIZE); |
1023 | break; |
1024 | } |
1025 | |
1026 | dw++; |
1027 | len = (len + 1) / 2; |
1028 | while (len--) { |
1029 | *cs++ = MI_STORE_REGISTER_MEM_GEN8; |
1030 | *cs++ = hw[dw]; |
1031 | *cs++ = lower_32_bits(i915_vma_offset(scratch) + x); |
1032 | *cs++ = upper_32_bits(i915_vma_offset(scratch) + x); |
1033 | |
1034 | dw += 2; |
1035 | x += 4; |
1036 | } |
1037 | } while (dw < PAGE_SIZE / sizeof(u32) && |
1038 | (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); |
1039 | |
1040 | *cs++ = MI_BATCH_BUFFER_END; |
1041 | |
1042 | shmem_unpin_map(file: ce->engine->default_state, ptr: defaults); |
1043 | |
1044 | i915_gem_object_flush_map(obj: batch->obj); |
1045 | i915_gem_object_unpin_map(obj: batch->obj); |
1046 | |
1047 | return batch; |
1048 | } |
1049 | |
1050 | static struct i915_request * |
1051 | record_registers(struct intel_context *ce, |
1052 | struct i915_vma *before, |
1053 | struct i915_vma *after, |
1054 | u32 *sema) |
1055 | { |
1056 | struct i915_vma *b_before, *b_after; |
1057 | struct i915_request *rq; |
1058 | u32 *cs; |
1059 | int err; |
1060 | |
1061 | b_before = store_context(ce, scratch: before); |
1062 | if (IS_ERR(ptr: b_before)) |
1063 | return ERR_CAST(ptr: b_before); |
1064 | |
1065 | b_after = store_context(ce, scratch: after); |
1066 | if (IS_ERR(ptr: b_after)) { |
1067 | rq = ERR_CAST(ptr: b_after); |
1068 | goto err_before; |
1069 | } |
1070 | |
1071 | rq = intel_context_create_request(ce); |
1072 | if (IS_ERR(ptr: rq)) |
1073 | goto err_after; |
1074 | |
1075 | err = igt_vma_move_to_active_unlocked(vma: before, rq, EXEC_OBJECT_WRITE); |
1076 | if (err) |
1077 | goto err_rq; |
1078 | |
1079 | err = igt_vma_move_to_active_unlocked(vma: b_before, rq, flags: 0); |
1080 | if (err) |
1081 | goto err_rq; |
1082 | |
1083 | err = igt_vma_move_to_active_unlocked(vma: after, rq, EXEC_OBJECT_WRITE); |
1084 | if (err) |
1085 | goto err_rq; |
1086 | |
1087 | err = igt_vma_move_to_active_unlocked(vma: b_after, rq, flags: 0); |
1088 | if (err) |
1089 | goto err_rq; |
1090 | |
1091 | cs = intel_ring_begin(rq, num_dwords: 14); |
1092 | if (IS_ERR(ptr: cs)) { |
1093 | err = PTR_ERR(ptr: cs); |
1094 | goto err_rq; |
1095 | } |
1096 | |
1097 | *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; |
1098 | *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); |
1099 | *cs++ = lower_32_bits(i915_vma_offset(b_before)); |
1100 | *cs++ = upper_32_bits(i915_vma_offset(b_before)); |
1101 | |
1102 | *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; |
1103 | *cs++ = MI_SEMAPHORE_WAIT | |
1104 | MI_SEMAPHORE_GLOBAL_GTT | |
1105 | MI_SEMAPHORE_POLL | |
1106 | MI_SEMAPHORE_SAD_NEQ_SDD; |
1107 | *cs++ = 0; |
1108 | *cs++ = i915_ggtt_offset(vma: ce->engine->status_page.vma) + |
1109 | offset_in_page(sema); |
1110 | *cs++ = 0; |
1111 | *cs++ = MI_NOOP; |
1112 | |
1113 | *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; |
1114 | *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); |
1115 | *cs++ = lower_32_bits(i915_vma_offset(b_after)); |
1116 | *cs++ = upper_32_bits(i915_vma_offset(b_after)); |
1117 | |
1118 | intel_ring_advance(rq, cs); |
1119 | |
1120 | WRITE_ONCE(*sema, 0); |
1121 | i915_request_get(rq); |
1122 | i915_request_add(rq); |
1123 | err_after: |
1124 | i915_vma_put(vma: b_after); |
1125 | err_before: |
1126 | i915_vma_put(vma: b_before); |
1127 | return rq; |
1128 | |
1129 | err_rq: |
1130 | i915_request_add(rq); |
1131 | rq = ERR_PTR(error: err); |
1132 | goto err_after; |
1133 | } |
1134 | |
1135 | static struct i915_vma *load_context(struct intel_context *ce, u32 poison) |
1136 | { |
1137 | struct i915_vma *batch; |
1138 | u32 dw, *cs, *hw; |
1139 | u32 *defaults; |
1140 | |
1141 | batch = create_user_vma(vm: ce->vm, SZ_64K); |
1142 | if (IS_ERR(ptr: batch)) |
1143 | return batch; |
1144 | |
1145 | cs = i915_gem_object_pin_map_unlocked(obj: batch->obj, type: I915_MAP_WC); |
1146 | if (IS_ERR(ptr: cs)) { |
1147 | i915_vma_put(vma: batch); |
1148 | return ERR_CAST(ptr: cs); |
1149 | } |
1150 | |
1151 | defaults = shmem_pin_map(file: ce->engine->default_state); |
1152 | if (!defaults) { |
1153 | i915_gem_object_unpin_map(obj: batch->obj); |
1154 | i915_vma_put(vma: batch); |
1155 | return ERR_PTR(error: -ENOMEM); |
1156 | } |
1157 | |
1158 | dw = 0; |
1159 | hw = defaults; |
1160 | hw += LRC_STATE_OFFSET / sizeof(*hw); |
1161 | do { |
1162 | u32 len = hw[dw] & LRI_LENGTH_MASK; |
1163 | |
1164 | /* For simplicity, break parsing at the first complex command */ |
1165 | if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT) |
1166 | break; |
1167 | |
1168 | if (hw[dw] == 0) { |
1169 | dw++; |
1170 | continue; |
1171 | } |
1172 | |
1173 | if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) { |
1174 | dw += len + 2; |
1175 | continue; |
1176 | } |
1177 | |
1178 | if (!len) { |
1179 | pr_err("%s: invalid LRI found in context image\n" , |
1180 | ce->engine->name); |
1181 | igt_hexdump(buf: defaults, PAGE_SIZE); |
1182 | break; |
1183 | } |
1184 | |
1185 | dw++; |
1186 | len = (len + 1) / 2; |
1187 | *cs++ = MI_LOAD_REGISTER_IMM(len); |
1188 | while (len--) { |
1189 | *cs++ = hw[dw]; |
1190 | *cs++ = safe_poison(offset: hw[dw] & get_lri_mask(engine: ce->engine, |
1191 | MI_LRI_LRM_CS_MMIO), |
1192 | poison); |
1193 | dw += 2; |
1194 | } |
1195 | } while (dw < PAGE_SIZE / sizeof(u32) && |
1196 | (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); |
1197 | |
1198 | *cs++ = MI_BATCH_BUFFER_END; |
1199 | |
1200 | shmem_unpin_map(file: ce->engine->default_state, ptr: defaults); |
1201 | |
1202 | i915_gem_object_flush_map(obj: batch->obj); |
1203 | i915_gem_object_unpin_map(obj: batch->obj); |
1204 | |
1205 | return batch; |
1206 | } |
1207 | |
1208 | static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) |
1209 | { |
1210 | struct i915_request *rq; |
1211 | struct i915_vma *batch; |
1212 | u32 *cs; |
1213 | int err; |
1214 | |
1215 | batch = load_context(ce, poison); |
1216 | if (IS_ERR(ptr: batch)) |
1217 | return PTR_ERR(ptr: batch); |
1218 | |
1219 | rq = intel_context_create_request(ce); |
1220 | if (IS_ERR(ptr: rq)) { |
1221 | err = PTR_ERR(ptr: rq); |
1222 | goto err_batch; |
1223 | } |
1224 | |
1225 | err = igt_vma_move_to_active_unlocked(vma: batch, rq, flags: 0); |
1226 | if (err) |
1227 | goto err_rq; |
1228 | |
1229 | cs = intel_ring_begin(rq, num_dwords: 8); |
1230 | if (IS_ERR(ptr: cs)) { |
1231 | err = PTR_ERR(ptr: cs); |
1232 | goto err_rq; |
1233 | } |
1234 | |
1235 | *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; |
1236 | *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); |
1237 | *cs++ = lower_32_bits(i915_vma_offset(batch)); |
1238 | *cs++ = upper_32_bits(i915_vma_offset(batch)); |
1239 | |
1240 | *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; |
1241 | *cs++ = i915_ggtt_offset(vma: ce->engine->status_page.vma) + |
1242 | offset_in_page(sema); |
1243 | *cs++ = 0; |
1244 | *cs++ = 1; |
1245 | |
1246 | intel_ring_advance(rq, cs); |
1247 | |
1248 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; |
1249 | err_rq: |
1250 | i915_request_add(rq); |
1251 | err_batch: |
1252 | i915_vma_put(vma: batch); |
1253 | return err; |
1254 | } |
1255 | |
1256 | static bool is_moving(u32 a, u32 b) |
1257 | { |
1258 | return a != b; |
1259 | } |
1260 | |
1261 | static int compare_isolation(struct intel_engine_cs *engine, |
1262 | struct i915_vma *ref[2], |
1263 | struct i915_vma *result[2], |
1264 | struct intel_context *ce, |
1265 | u32 poison) |
1266 | { |
1267 | u32 x, dw, *hw, *lrc; |
1268 | u32 *A[2], *B[2]; |
1269 | u32 *defaults; |
1270 | int err = 0; |
1271 | |
1272 | A[0] = i915_gem_object_pin_map_unlocked(obj: ref[0]->obj, type: I915_MAP_WC); |
1273 | if (IS_ERR(ptr: A[0])) |
1274 | return PTR_ERR(ptr: A[0]); |
1275 | |
1276 | A[1] = i915_gem_object_pin_map_unlocked(obj: ref[1]->obj, type: I915_MAP_WC); |
1277 | if (IS_ERR(ptr: A[1])) { |
1278 | err = PTR_ERR(ptr: A[1]); |
1279 | goto err_A0; |
1280 | } |
1281 | |
1282 | B[0] = i915_gem_object_pin_map_unlocked(obj: result[0]->obj, type: I915_MAP_WC); |
1283 | if (IS_ERR(ptr: B[0])) { |
1284 | err = PTR_ERR(ptr: B[0]); |
1285 | goto err_A1; |
1286 | } |
1287 | |
1288 | B[1] = i915_gem_object_pin_map_unlocked(obj: result[1]->obj, type: I915_MAP_WC); |
1289 | if (IS_ERR(ptr: B[1])) { |
1290 | err = PTR_ERR(ptr: B[1]); |
1291 | goto err_B0; |
1292 | } |
1293 | |
1294 | lrc = i915_gem_object_pin_map_unlocked(obj: ce->state->obj, |
1295 | type: intel_gt_coherent_map_type(gt: engine->gt, |
1296 | obj: ce->state->obj, |
1297 | always_coherent: false)); |
1298 | if (IS_ERR(ptr: lrc)) { |
1299 | err = PTR_ERR(ptr: lrc); |
1300 | goto err_B1; |
1301 | } |
1302 | lrc += LRC_STATE_OFFSET / sizeof(*hw); |
1303 | |
1304 | defaults = shmem_pin_map(file: ce->engine->default_state); |
1305 | if (!defaults) { |
1306 | err = -ENOMEM; |
1307 | goto err_lrc; |
1308 | } |
1309 | |
1310 | x = 0; |
1311 | dw = 0; |
1312 | hw = defaults; |
1313 | hw += LRC_STATE_OFFSET / sizeof(*hw); |
1314 | do { |
1315 | u32 len = hw[dw] & LRI_LENGTH_MASK; |
1316 | |
1317 | /* For simplicity, break parsing at the first complex command */ |
1318 | if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT) |
1319 | break; |
1320 | |
1321 | if (hw[dw] == 0) { |
1322 | dw++; |
1323 | continue; |
1324 | } |
1325 | |
1326 | if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) { |
1327 | dw += len + 2; |
1328 | continue; |
1329 | } |
1330 | |
1331 | if (!len) { |
1332 | pr_err("%s: invalid LRI found in context image\n" , |
1333 | engine->name); |
1334 | igt_hexdump(buf: defaults, PAGE_SIZE); |
1335 | break; |
1336 | } |
1337 | |
1338 | dw++; |
1339 | len = (len + 1) / 2; |
1340 | while (len--) { |
1341 | if (!is_moving(a: A[0][x], b: A[1][x]) && |
1342 | (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { |
1343 | switch (hw[dw] & 4095) { |
1344 | case 0x30: /* RING_HEAD */ |
1345 | case 0x34: /* RING_TAIL */ |
1346 | break; |
1347 | |
1348 | default: |
1349 | pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n" , |
1350 | engine->name, dw, |
1351 | hw[dw], hw[dw + 1], |
1352 | A[0][x], B[0][x], B[1][x], |
1353 | poison, lrc[dw + 1]); |
1354 | err = -EINVAL; |
1355 | } |
1356 | } |
1357 | dw += 2; |
1358 | x++; |
1359 | } |
1360 | } while (dw < PAGE_SIZE / sizeof(u32) && |
1361 | (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); |
1362 | |
1363 | shmem_unpin_map(file: ce->engine->default_state, ptr: defaults); |
1364 | err_lrc: |
1365 | i915_gem_object_unpin_map(obj: ce->state->obj); |
1366 | err_B1: |
1367 | i915_gem_object_unpin_map(obj: result[1]->obj); |
1368 | err_B0: |
1369 | i915_gem_object_unpin_map(obj: result[0]->obj); |
1370 | err_A1: |
1371 | i915_gem_object_unpin_map(obj: ref[1]->obj); |
1372 | err_A0: |
1373 | i915_gem_object_unpin_map(obj: ref[0]->obj); |
1374 | return err; |
1375 | } |
1376 | |
1377 | static struct i915_vma * |
1378 | create_result_vma(struct i915_address_space *vm, unsigned long sz) |
1379 | { |
1380 | struct i915_vma *vma; |
1381 | void *ptr; |
1382 | |
1383 | vma = create_user_vma(vm, size: sz); |
1384 | if (IS_ERR(ptr: vma)) |
1385 | return vma; |
1386 | |
1387 | /* Set the results to a known value distinct from the poison */ |
1388 | ptr = i915_gem_object_pin_map_unlocked(obj: vma->obj, type: I915_MAP_WC); |
1389 | if (IS_ERR(ptr)) { |
1390 | i915_vma_put(vma); |
1391 | return ERR_CAST(ptr); |
1392 | } |
1393 | |
1394 | memset(ptr, POISON_INUSE, vma->size); |
1395 | i915_gem_object_flush_map(obj: vma->obj); |
1396 | i915_gem_object_unpin_map(obj: vma->obj); |
1397 | |
1398 | return vma; |
1399 | } |
1400 | |
1401 | static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) |
1402 | { |
1403 | u32 *sema = memset32(s: engine->status_page.addr + 1000, v: 0, n: 1); |
1404 | struct i915_vma *ref[2], *result[2]; |
1405 | struct intel_context *A, *B; |
1406 | struct i915_request *rq; |
1407 | int err; |
1408 | |
1409 | A = intel_context_create(engine); |
1410 | if (IS_ERR(ptr: A)) |
1411 | return PTR_ERR(ptr: A); |
1412 | |
1413 | B = intel_context_create(engine); |
1414 | if (IS_ERR(ptr: B)) { |
1415 | err = PTR_ERR(ptr: B); |
1416 | goto err_A; |
1417 | } |
1418 | |
1419 | ref[0] = create_result_vma(vm: A->vm, SZ_64K); |
1420 | if (IS_ERR(ptr: ref[0])) { |
1421 | err = PTR_ERR(ptr: ref[0]); |
1422 | goto err_B; |
1423 | } |
1424 | |
1425 | ref[1] = create_result_vma(vm: A->vm, SZ_64K); |
1426 | if (IS_ERR(ptr: ref[1])) { |
1427 | err = PTR_ERR(ptr: ref[1]); |
1428 | goto err_ref0; |
1429 | } |
1430 | |
1431 | rq = record_registers(ce: A, before: ref[0], after: ref[1], sema); |
1432 | if (IS_ERR(ptr: rq)) { |
1433 | err = PTR_ERR(ptr: rq); |
1434 | goto err_ref1; |
1435 | } |
1436 | |
1437 | WRITE_ONCE(*sema, 1); |
1438 | wmb(); |
1439 | |
1440 | if (i915_request_wait(rq, flags: 0, HZ / 2) < 0) { |
1441 | i915_request_put(rq); |
1442 | err = -ETIME; |
1443 | goto err_ref1; |
1444 | } |
1445 | i915_request_put(rq); |
1446 | |
1447 | result[0] = create_result_vma(vm: A->vm, SZ_64K); |
1448 | if (IS_ERR(ptr: result[0])) { |
1449 | err = PTR_ERR(ptr: result[0]); |
1450 | goto err_ref1; |
1451 | } |
1452 | |
1453 | result[1] = create_result_vma(vm: A->vm, SZ_64K); |
1454 | if (IS_ERR(ptr: result[1])) { |
1455 | err = PTR_ERR(ptr: result[1]); |
1456 | goto err_result0; |
1457 | } |
1458 | |
1459 | rq = record_registers(ce: A, before: result[0], after: result[1], sema); |
1460 | if (IS_ERR(ptr: rq)) { |
1461 | err = PTR_ERR(ptr: rq); |
1462 | goto err_result1; |
1463 | } |
1464 | |
1465 | err = poison_registers(ce: B, poison, sema); |
1466 | if (err == 0 && i915_request_wait(rq, flags: 0, HZ / 2) < 0) { |
1467 | pr_err("%s(%s): wait for results timed out\n" , |
1468 | __func__, engine->name); |
1469 | err = -ETIME; |
1470 | } |
1471 | |
1472 | /* Always cancel the semaphore wait, just in case the GPU gets stuck */ |
1473 | WRITE_ONCE(*sema, -1); |
1474 | i915_request_put(rq); |
1475 | if (err) |
1476 | goto err_result1; |
1477 | |
1478 | err = compare_isolation(engine, ref, result, ce: A, poison); |
1479 | |
1480 | err_result1: |
1481 | i915_vma_put(vma: result[1]); |
1482 | err_result0: |
1483 | i915_vma_put(vma: result[0]); |
1484 | err_ref1: |
1485 | i915_vma_put(vma: ref[1]); |
1486 | err_ref0: |
1487 | i915_vma_put(vma: ref[0]); |
1488 | err_B: |
1489 | intel_context_put(ce: B); |
1490 | err_A: |
1491 | intel_context_put(ce: A); |
1492 | return err; |
1493 | } |
1494 | |
1495 | static bool skip_isolation(const struct intel_engine_cs *engine) |
1496 | { |
1497 | if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9) |
1498 | return true; |
1499 | |
1500 | if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11) |
1501 | return true; |
1502 | |
1503 | return false; |
1504 | } |
1505 | |
1506 | static int live_lrc_isolation(void *arg) |
1507 | { |
1508 | struct intel_gt *gt = arg; |
1509 | struct intel_engine_cs *engine; |
1510 | enum intel_engine_id id; |
1511 | const u32 poison[] = { |
1512 | STACK_MAGIC, |
1513 | 0x3a3a3a3a, |
1514 | 0x5c5c5c5c, |
1515 | 0xffffffff, |
1516 | 0xffff0000, |
1517 | }; |
1518 | int err = 0; |
1519 | |
1520 | /* |
1521 | * Our goal is try and verify that per-context state cannot be |
1522 | * tampered with by another non-privileged client. |
1523 | * |
1524 | * We take the list of context registers from the LRI in the default |
1525 | * context image and attempt to modify that list from a remote context. |
1526 | */ |
1527 | |
1528 | for_each_engine(engine, gt, id) { |
1529 | int i; |
1530 | |
1531 | /* Just don't even ask */ |
1532 | if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && |
1533 | skip_isolation(engine)) |
1534 | continue; |
1535 | |
1536 | intel_engine_pm_get(engine); |
1537 | for (i = 0; i < ARRAY_SIZE(poison); i++) { |
1538 | int result; |
1539 | |
1540 | result = __lrc_isolation(engine, poison: poison[i]); |
1541 | if (result && !err) |
1542 | err = result; |
1543 | |
1544 | result = __lrc_isolation(engine, poison: ~poison[i]); |
1545 | if (result && !err) |
1546 | err = result; |
1547 | } |
1548 | intel_engine_pm_put(engine); |
1549 | if (igt_flush_test(i915: gt->i915)) { |
1550 | err = -EIO; |
1551 | break; |
1552 | } |
1553 | } |
1554 | |
1555 | return err; |
1556 | } |
1557 | |
1558 | static int wabb_ctx_submit_req(struct intel_context *ce) |
1559 | { |
1560 | struct i915_request *rq; |
1561 | int err = 0; |
1562 | |
1563 | rq = intel_context_create_request(ce); |
1564 | if (IS_ERR(ptr: rq)) |
1565 | return PTR_ERR(ptr: rq); |
1566 | |
1567 | i915_request_get(rq); |
1568 | i915_request_add(rq); |
1569 | |
1570 | if (i915_request_wait(rq, flags: 0, HZ / 5) < 0) |
1571 | err = -ETIME; |
1572 | |
1573 | i915_request_put(rq); |
1574 | |
1575 | return err; |
1576 | } |
1577 | |
1578 | #define CTX_BB_CANARY_OFFSET (3 * 1024) |
1579 | #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) |
1580 | |
1581 | static u32 * |
1582 | emit_wabb_ctx_canary(const struct intel_context *ce, |
1583 | u32 *cs, bool per_ctx) |
1584 | { |
1585 | *cs++ = MI_STORE_REGISTER_MEM_GEN8 | |
1586 | MI_SRM_LRM_GLOBAL_GTT | |
1587 | MI_LRI_LRM_CS_MMIO; |
1588 | *cs++ = i915_mmio_reg_offset(RING_START(0)); |
1589 | *cs++ = i915_ggtt_offset(vma: ce->state) + |
1590 | context_wa_bb_offset(ce) + |
1591 | CTX_BB_CANARY_OFFSET + |
1592 | (per_ctx ? PAGE_SIZE : 0); |
1593 | *cs++ = 0; |
1594 | |
1595 | return cs; |
1596 | } |
1597 | |
1598 | static u32 * |
1599 | emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) |
1600 | { |
1601 | return emit_wabb_ctx_canary(ce, cs, per_ctx: false); |
1602 | } |
1603 | |
1604 | static u32 * |
1605 | emit_per_ctx_bb_canary(const struct intel_context *ce, u32 *cs) |
1606 | { |
1607 | return emit_wabb_ctx_canary(ce, cs, per_ctx: true); |
1608 | } |
1609 | |
1610 | static void |
1611 | wabb_ctx_setup(struct intel_context *ce, bool per_ctx) |
1612 | { |
1613 | u32 *cs = context_wabb(ce, per_ctx); |
1614 | |
1615 | cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; |
1616 | |
1617 | if (per_ctx) |
1618 | setup_per_ctx_bb(ce, engine: ce->engine, emit: emit_per_ctx_bb_canary); |
1619 | else |
1620 | setup_indirect_ctx_bb(ce, engine: ce->engine, emit: emit_indirect_ctx_bb_canary); |
1621 | } |
1622 | |
1623 | static bool check_ring_start(struct intel_context *ce, bool per_ctx) |
1624 | { |
1625 | const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - |
1626 | LRC_STATE_OFFSET + context_wa_bb_offset(ce) + |
1627 | (per_ctx ? PAGE_SIZE : 0); |
1628 | |
1629 | if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) |
1630 | return true; |
1631 | |
1632 | pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n" , |
1633 | ctx_bb[CTX_BB_CANARY_INDEX], |
1634 | ce->lrc_reg_state[CTX_RING_START]); |
1635 | |
1636 | return false; |
1637 | } |
1638 | |
1639 | static int wabb_ctx_check(struct intel_context *ce, bool per_ctx) |
1640 | { |
1641 | int err; |
1642 | |
1643 | err = wabb_ctx_submit_req(ce); |
1644 | if (err) |
1645 | return err; |
1646 | |
1647 | if (!check_ring_start(ce, per_ctx)) |
1648 | return -EINVAL; |
1649 | |
1650 | return 0; |
1651 | } |
1652 | |
1653 | static int __lrc_wabb_ctx(struct intel_engine_cs *engine, bool per_ctx) |
1654 | { |
1655 | struct intel_context *a, *b; |
1656 | int err; |
1657 | |
1658 | a = intel_context_create(engine); |
1659 | if (IS_ERR(ptr: a)) |
1660 | return PTR_ERR(ptr: a); |
1661 | err = intel_context_pin(ce: a); |
1662 | if (err) |
1663 | goto put_a; |
1664 | |
1665 | b = intel_context_create(engine); |
1666 | if (IS_ERR(ptr: b)) { |
1667 | err = PTR_ERR(ptr: b); |
1668 | goto unpin_a; |
1669 | } |
1670 | err = intel_context_pin(ce: b); |
1671 | if (err) |
1672 | goto put_b; |
1673 | |
1674 | /* We use the already reserved extra page in context state */ |
1675 | if (!a->wa_bb_page) { |
1676 | GEM_BUG_ON(b->wa_bb_page); |
1677 | GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12); |
1678 | goto unpin_b; |
1679 | } |
1680 | |
1681 | /* |
1682 | * In order to test that our per context bb is truly per context, |
1683 | * and executes at the intended spot on context restoring process, |
1684 | * make the batch store the ring start value to memory. |
1685 | * As ring start is restored apriori of starting the indirect ctx bb and |
1686 | * as it will be different for each context, it fits to this purpose. |
1687 | */ |
1688 | wabb_ctx_setup(ce: a, per_ctx); |
1689 | wabb_ctx_setup(ce: b, per_ctx); |
1690 | |
1691 | err = wabb_ctx_check(ce: a, per_ctx); |
1692 | if (err) |
1693 | goto unpin_b; |
1694 | |
1695 | err = wabb_ctx_check(ce: b, per_ctx); |
1696 | |
1697 | unpin_b: |
1698 | intel_context_unpin(ce: b); |
1699 | put_b: |
1700 | intel_context_put(ce: b); |
1701 | unpin_a: |
1702 | intel_context_unpin(ce: a); |
1703 | put_a: |
1704 | intel_context_put(ce: a); |
1705 | |
1706 | return err; |
1707 | } |
1708 | |
1709 | static int lrc_wabb_ctx(void *arg, bool per_ctx) |
1710 | { |
1711 | struct intel_gt *gt = arg; |
1712 | struct intel_engine_cs *engine; |
1713 | enum intel_engine_id id; |
1714 | int err = 0; |
1715 | |
1716 | for_each_engine(engine, gt, id) { |
1717 | intel_engine_pm_get(engine); |
1718 | err = __lrc_wabb_ctx(engine, per_ctx); |
1719 | intel_engine_pm_put(engine); |
1720 | |
1721 | if (igt_flush_test(i915: gt->i915)) |
1722 | err = -EIO; |
1723 | |
1724 | if (err) |
1725 | break; |
1726 | } |
1727 | |
1728 | return err; |
1729 | } |
1730 | |
1731 | static int live_lrc_indirect_ctx_bb(void *arg) |
1732 | { |
1733 | return lrc_wabb_ctx(arg, per_ctx: false); |
1734 | } |
1735 | |
1736 | static int live_lrc_per_ctx_bb(void *arg) |
1737 | { |
1738 | return lrc_wabb_ctx(arg, per_ctx: true); |
1739 | } |
1740 | |
1741 | static void garbage_reset(struct intel_engine_cs *engine, |
1742 | struct i915_request *rq) |
1743 | { |
1744 | const unsigned int bit = I915_RESET_ENGINE + engine->id; |
1745 | unsigned long *lock = &engine->gt->reset.flags; |
1746 | |
1747 | local_bh_disable(); |
1748 | if (!test_and_set_bit(nr: bit, addr: lock)) { |
1749 | tasklet_disable(t: &engine->sched_engine->tasklet); |
1750 | |
1751 | if (!rq->fence.error) |
1752 | __intel_engine_reset_bh(engine, NULL); |
1753 | |
1754 | tasklet_enable(t: &engine->sched_engine->tasklet); |
1755 | clear_and_wake_up_bit(bit, word: lock); |
1756 | } |
1757 | local_bh_enable(); |
1758 | } |
1759 | |
1760 | static struct i915_request *garbage(struct intel_context *ce, |
1761 | struct rnd_state *prng) |
1762 | { |
1763 | struct i915_request *rq; |
1764 | int err; |
1765 | |
1766 | err = intel_context_pin(ce); |
1767 | if (err) |
1768 | return ERR_PTR(error: err); |
1769 | |
1770 | prandom_bytes_state(state: prng, |
1771 | buf: ce->lrc_reg_state, |
1772 | nbytes: ce->engine->context_size - |
1773 | LRC_STATE_OFFSET); |
1774 | |
1775 | rq = intel_context_create_request(ce); |
1776 | if (IS_ERR(ptr: rq)) { |
1777 | err = PTR_ERR(ptr: rq); |
1778 | goto err_unpin; |
1779 | } |
1780 | |
1781 | i915_request_get(rq); |
1782 | i915_request_add(rq); |
1783 | return rq; |
1784 | |
1785 | err_unpin: |
1786 | intel_context_unpin(ce); |
1787 | return ERR_PTR(error: err); |
1788 | } |
1789 | |
1790 | static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) |
1791 | { |
1792 | struct intel_context *ce; |
1793 | struct i915_request *hang; |
1794 | int err = 0; |
1795 | |
1796 | ce = intel_context_create(engine); |
1797 | if (IS_ERR(ptr: ce)) |
1798 | return PTR_ERR(ptr: ce); |
1799 | |
1800 | hang = garbage(ce, prng); |
1801 | if (IS_ERR(ptr: hang)) { |
1802 | err = PTR_ERR(ptr: hang); |
1803 | goto err_ce; |
1804 | } |
1805 | |
1806 | if (wait_for_submit(engine, rq: hang, HZ / 2)) { |
1807 | i915_request_put(rq: hang); |
1808 | err = -ETIME; |
1809 | goto err_ce; |
1810 | } |
1811 | |
1812 | intel_context_set_banned(ce); |
1813 | garbage_reset(engine, rq: hang); |
1814 | |
1815 | intel_engine_flush_submission(engine); |
1816 | if (!hang->fence.error) { |
1817 | i915_request_put(rq: hang); |
1818 | pr_err("%s: corrupted context was not reset\n" , |
1819 | engine->name); |
1820 | err = -EINVAL; |
1821 | goto err_ce; |
1822 | } |
1823 | |
1824 | if (i915_request_wait(rq: hang, flags: 0, HZ / 2) < 0) { |
1825 | pr_err("%s: corrupted context did not recover\n" , |
1826 | engine->name); |
1827 | i915_request_put(rq: hang); |
1828 | err = -EIO; |
1829 | goto err_ce; |
1830 | } |
1831 | i915_request_put(rq: hang); |
1832 | |
1833 | err_ce: |
1834 | intel_context_put(ce); |
1835 | return err; |
1836 | } |
1837 | |
1838 | static int live_lrc_garbage(void *arg) |
1839 | { |
1840 | struct intel_gt *gt = arg; |
1841 | struct intel_engine_cs *engine; |
1842 | enum intel_engine_id id; |
1843 | |
1844 | /* |
1845 | * Verify that we can recover if one context state is completely |
1846 | * corrupted. |
1847 | */ |
1848 | |
1849 | if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) |
1850 | return 0; |
1851 | |
1852 | for_each_engine(engine, gt, id) { |
1853 | I915_RND_STATE(prng); |
1854 | int err = 0, i; |
1855 | |
1856 | if (!intel_has_reset_engine(gt: engine->gt)) |
1857 | continue; |
1858 | |
1859 | intel_engine_pm_get(engine); |
1860 | for (i = 0; i < 3; i++) { |
1861 | err = __lrc_garbage(engine, prng: &prng); |
1862 | if (err) |
1863 | break; |
1864 | } |
1865 | intel_engine_pm_put(engine); |
1866 | |
1867 | if (igt_flush_test(i915: gt->i915)) |
1868 | err = -EIO; |
1869 | if (err) |
1870 | return err; |
1871 | } |
1872 | |
1873 | return 0; |
1874 | } |
1875 | |
1876 | static int __live_pphwsp_runtime(struct intel_engine_cs *engine) |
1877 | { |
1878 | struct intel_context *ce; |
1879 | struct i915_request *rq; |
1880 | IGT_TIMEOUT(end_time); |
1881 | int err; |
1882 | |
1883 | ce = intel_context_create(engine); |
1884 | if (IS_ERR(ptr: ce)) |
1885 | return PTR_ERR(ptr: ce); |
1886 | |
1887 | ce->stats.runtime.num_underflow = 0; |
1888 | ce->stats.runtime.max_underflow = 0; |
1889 | |
1890 | do { |
1891 | unsigned int loop = 1024; |
1892 | |
1893 | while (loop) { |
1894 | rq = intel_context_create_request(ce); |
1895 | if (IS_ERR(ptr: rq)) { |
1896 | err = PTR_ERR(ptr: rq); |
1897 | goto err_rq; |
1898 | } |
1899 | |
1900 | if (--loop == 0) |
1901 | i915_request_get(rq); |
1902 | |
1903 | i915_request_add(rq); |
1904 | } |
1905 | |
1906 | if (__igt_timeout(timeout: end_time, NULL)) |
1907 | break; |
1908 | |
1909 | i915_request_put(rq); |
1910 | } while (1); |
1911 | |
1912 | err = i915_request_wait(rq, flags: 0, HZ / 5); |
1913 | if (err < 0) { |
1914 | pr_err("%s: request not completed!\n" , engine->name); |
1915 | goto err_wait; |
1916 | } |
1917 | |
1918 | igt_flush_test(i915: engine->i915); |
1919 | |
1920 | pr_info("%s: pphwsp runtime %lluns, average %lluns\n" , |
1921 | engine->name, |
1922 | intel_context_get_total_runtime_ns(ce), |
1923 | intel_context_get_avg_runtime_ns(ce)); |
1924 | |
1925 | err = 0; |
1926 | if (ce->stats.runtime.num_underflow) { |
1927 | pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n" , |
1928 | engine->name, |
1929 | ce->stats.runtime.num_underflow, |
1930 | ce->stats.runtime.max_underflow); |
1931 | GEM_TRACE_DUMP(); |
1932 | err = -EOVERFLOW; |
1933 | } |
1934 | |
1935 | err_wait: |
1936 | i915_request_put(rq); |
1937 | err_rq: |
1938 | intel_context_put(ce); |
1939 | return err; |
1940 | } |
1941 | |
1942 | static int live_pphwsp_runtime(void *arg) |
1943 | { |
1944 | struct intel_gt *gt = arg; |
1945 | struct intel_engine_cs *engine; |
1946 | enum intel_engine_id id; |
1947 | int err = 0; |
1948 | |
1949 | /* |
1950 | * Check that cumulative context runtime as stored in the pphwsp[16] |
1951 | * is monotonic. |
1952 | */ |
1953 | |
1954 | for_each_engine(engine, gt, id) { |
1955 | err = __live_pphwsp_runtime(engine); |
1956 | if (err) |
1957 | break; |
1958 | } |
1959 | |
1960 | if (igt_flush_test(i915: gt->i915)) |
1961 | err = -EIO; |
1962 | |
1963 | return err; |
1964 | } |
1965 | |
1966 | int intel_lrc_live_selftests(struct drm_i915_private *i915) |
1967 | { |
1968 | static const struct i915_subtest tests[] = { |
1969 | SUBTEST(live_lrc_layout), |
1970 | SUBTEST(live_lrc_fixed), |
1971 | SUBTEST(live_lrc_state), |
1972 | SUBTEST(live_lrc_gpr), |
1973 | SUBTEST(live_lrc_isolation), |
1974 | SUBTEST(live_lrc_timestamp), |
1975 | SUBTEST(live_lrc_garbage), |
1976 | SUBTEST(live_pphwsp_runtime), |
1977 | SUBTEST(live_lrc_indirect_ctx_bb), |
1978 | SUBTEST(live_lrc_per_ctx_bb), |
1979 | }; |
1980 | |
1981 | if (!HAS_LOGICAL_RING_CONTEXTS(i915)) |
1982 | return 0; |
1983 | |
1984 | return intel_gt_live_subtests(tests, to_gt(i915)); |
1985 | } |
1986 | |