1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2020-2021 Intel Corporation |
4 | */ |
5 | |
6 | #include "gt/intel_migrate.h" |
7 | #include "gt/intel_gpu_commands.h" |
8 | #include "gem/i915_gem_ttm_move.h" |
9 | |
10 | #include "i915_deps.h" |
11 | |
12 | #include "selftests/igt_reset.h" |
13 | #include "selftests/igt_spinner.h" |
14 | |
15 | static int igt_fill_check_buffer(struct drm_i915_gem_object *obj, |
16 | struct intel_gt *gt, |
17 | bool fill) |
18 | { |
19 | unsigned int i, count = obj->base.size / sizeof(u32); |
20 | enum i915_map_type map_type = |
21 | intel_gt_coherent_map_type(gt, obj, always_coherent: false); |
22 | u32 *cur; |
23 | int err = 0; |
24 | |
25 | assert_object_held(obj); |
26 | cur = i915_gem_object_pin_map(obj, type: map_type); |
27 | if (IS_ERR(ptr: cur)) |
28 | return PTR_ERR(ptr: cur); |
29 | |
30 | if (fill) |
31 | for (i = 0; i < count; ++i) |
32 | *cur++ = i; |
33 | else |
34 | for (i = 0; i < count; ++i) |
35 | if (*cur++ != i) { |
36 | pr_err("Object content mismatch at location %d of %d\n" , i, count); |
37 | err = -EINVAL; |
38 | break; |
39 | } |
40 | |
41 | i915_gem_object_unpin_map(obj); |
42 | |
43 | return err; |
44 | } |
45 | |
46 | static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src, |
47 | enum intel_region_id dst) |
48 | { |
49 | struct drm_i915_private *i915 = gt->i915; |
50 | struct intel_memory_region *src_mr = i915->mm.regions[src]; |
51 | struct intel_memory_region *dst_mr = i915->mm.regions[dst]; |
52 | struct drm_i915_gem_object *obj; |
53 | struct i915_gem_ww_ctx ww; |
54 | int err = 0; |
55 | |
56 | GEM_BUG_ON(!src_mr); |
57 | GEM_BUG_ON(!dst_mr); |
58 | |
59 | /* Switch object backing-store on create */ |
60 | obj = i915_gem_object_create_region(mem: src_mr, size: dst_mr->min_page_size, page_size: 0, flags: 0); |
61 | if (IS_ERR(ptr: obj)) |
62 | return PTR_ERR(ptr: obj); |
63 | |
64 | for_i915_gem_ww(&ww, err, true) { |
65 | err = i915_gem_object_lock(obj, ww: &ww); |
66 | if (err) |
67 | continue; |
68 | |
69 | err = igt_fill_check_buffer(obj, gt, fill: true); |
70 | if (err) |
71 | continue; |
72 | |
73 | err = i915_gem_object_migrate(obj, ww: &ww, id: dst); |
74 | if (err) |
75 | continue; |
76 | |
77 | err = i915_gem_object_pin_pages(obj); |
78 | if (err) |
79 | continue; |
80 | |
81 | if (i915_gem_object_can_migrate(obj, id: src)) |
82 | err = -EINVAL; |
83 | |
84 | i915_gem_object_unpin_pages(obj); |
85 | err = i915_gem_object_wait_migration(obj, flags: true); |
86 | if (err) |
87 | continue; |
88 | |
89 | err = igt_fill_check_buffer(obj, gt, fill: false); |
90 | } |
91 | i915_gem_object_put(obj); |
92 | |
93 | return err; |
94 | } |
95 | |
96 | static int igt_smem_create_migrate(void *arg) |
97 | { |
98 | return igt_create_migrate(gt: arg, src: INTEL_REGION_LMEM_0, dst: INTEL_REGION_SMEM); |
99 | } |
100 | |
101 | static int igt_lmem_create_migrate(void *arg) |
102 | { |
103 | return igt_create_migrate(gt: arg, src: INTEL_REGION_SMEM, dst: INTEL_REGION_LMEM_0); |
104 | } |
105 | |
106 | static int igt_same_create_migrate(void *arg) |
107 | { |
108 | return igt_create_migrate(gt: arg, src: INTEL_REGION_LMEM_0, dst: INTEL_REGION_LMEM_0); |
109 | } |
110 | |
111 | static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww, |
112 | struct drm_i915_gem_object *obj, |
113 | struct i915_vma *vma, |
114 | bool silent_migrate) |
115 | { |
116 | int err; |
117 | |
118 | err = i915_gem_object_lock(obj, ww); |
119 | if (err) |
120 | return err; |
121 | |
122 | if (vma) { |
123 | err = i915_vma_pin_ww(vma, ww, size: obj->base.size, alignment: 0, |
124 | flags: 0UL | PIN_OFFSET_FIXED | |
125 | PIN_USER); |
126 | if (err) { |
127 | if (err != -EINTR && err != ERESTARTSYS && |
128 | err != -EDEADLK) |
129 | pr_err("Failed to pin vma.\n" ); |
130 | return err; |
131 | } |
132 | |
133 | i915_vma_unpin(vma); |
134 | } |
135 | |
136 | /* |
137 | * Migration will implicitly unbind (asynchronously) any bound |
138 | * vmas. |
139 | */ |
140 | if (i915_gem_object_is_lmem(obj)) { |
141 | err = i915_gem_object_migrate(obj, ww, id: INTEL_REGION_SMEM); |
142 | if (err) { |
143 | if (!silent_migrate) |
144 | pr_err("Object failed migration to smem\n" ); |
145 | if (err) |
146 | return err; |
147 | } |
148 | |
149 | if (i915_gem_object_is_lmem(obj)) { |
150 | pr_err("object still backed by lmem\n" ); |
151 | err = -EINVAL; |
152 | } |
153 | |
154 | if (!i915_gem_object_has_struct_page(obj)) { |
155 | pr_err("object not backed by struct page\n" ); |
156 | err = -EINVAL; |
157 | } |
158 | |
159 | } else { |
160 | err = i915_gem_object_migrate(obj, ww, id: INTEL_REGION_LMEM_0); |
161 | if (err) { |
162 | if (!silent_migrate) |
163 | pr_err("Object failed migration to lmem\n" ); |
164 | if (err) |
165 | return err; |
166 | } |
167 | |
168 | if (i915_gem_object_has_struct_page(obj)) { |
169 | pr_err("object still backed by struct page\n" ); |
170 | err = -EINVAL; |
171 | } |
172 | |
173 | if (!i915_gem_object_is_lmem(obj)) { |
174 | pr_err("object not backed by lmem\n" ); |
175 | err = -EINVAL; |
176 | } |
177 | } |
178 | |
179 | return err; |
180 | } |
181 | |
182 | static int __igt_lmem_pages_migrate(struct intel_gt *gt, |
183 | struct i915_address_space *vm, |
184 | struct i915_deps *deps, |
185 | struct igt_spinner *spin, |
186 | struct dma_fence *spin_fence, |
187 | bool borked_migrate) |
188 | { |
189 | struct drm_i915_private *i915 = gt->i915; |
190 | struct drm_i915_gem_object *obj; |
191 | struct i915_vma *vma = NULL; |
192 | struct i915_gem_ww_ctx ww; |
193 | struct i915_request *rq; |
194 | int err; |
195 | int i; |
196 | |
197 | /* From LMEM to shmem and back again */ |
198 | |
199 | obj = i915_gem_object_create_lmem(i915, SZ_2M, flags: 0); |
200 | if (IS_ERR(ptr: obj)) |
201 | return PTR_ERR(ptr: obj); |
202 | |
203 | if (vm) { |
204 | vma = i915_vma_instance(obj, vm, NULL); |
205 | if (IS_ERR(ptr: vma)) { |
206 | err = PTR_ERR(ptr: vma); |
207 | goto out_put; |
208 | } |
209 | } |
210 | |
211 | /* Initial GPU fill, sync, CPU initialization. */ |
212 | for_i915_gem_ww(&ww, err, true) { |
213 | err = i915_gem_object_lock(obj, ww: &ww); |
214 | if (err) |
215 | continue; |
216 | |
217 | err = ____i915_gem_object_get_pages(obj); |
218 | if (err) |
219 | continue; |
220 | |
221 | err = intel_migrate_clear(m: >->migrate, ww: &ww, deps, |
222 | sg: obj->mm.pages->sgl, pat_index: obj->pat_index, |
223 | is_lmem: i915_gem_object_is_lmem(obj), |
224 | value: 0xdeadbeaf, out: &rq); |
225 | if (rq) { |
226 | err = dma_resv_reserve_fences(obj: obj->base.resv, num_fences: 1); |
227 | if (!err) |
228 | dma_resv_add_fence(obj: obj->base.resv, fence: &rq->fence, |
229 | usage: DMA_RESV_USAGE_KERNEL); |
230 | i915_request_put(rq); |
231 | } |
232 | if (err) |
233 | continue; |
234 | |
235 | if (!vma) { |
236 | err = igt_fill_check_buffer(obj, gt, fill: true); |
237 | if (err) |
238 | continue; |
239 | } |
240 | } |
241 | if (err) |
242 | goto out_put; |
243 | |
244 | /* |
245 | * Migrate to and from smem without explicitly syncing. |
246 | * Finalize with data in smem for fast readout. |
247 | */ |
248 | for (i = 1; i <= 5; ++i) { |
249 | for_i915_gem_ww(&ww, err, true) |
250 | err = lmem_pages_migrate_one(ww: &ww, obj, vma, |
251 | silent_migrate: borked_migrate); |
252 | if (err) |
253 | goto out_put; |
254 | } |
255 | |
256 | err = i915_gem_object_lock_interruptible(obj, NULL); |
257 | if (err) |
258 | goto out_put; |
259 | |
260 | if (spin) { |
261 | if (dma_fence_is_signaled(fence: spin_fence)) { |
262 | pr_err("Spinner was terminated by hangcheck.\n" ); |
263 | err = -EBUSY; |
264 | goto out_unlock; |
265 | } |
266 | igt_spinner_end(spin); |
267 | } |
268 | |
269 | /* Finally sync migration and check content. */ |
270 | err = i915_gem_object_wait_migration(obj, flags: true); |
271 | if (err) |
272 | goto out_unlock; |
273 | |
274 | if (vma) { |
275 | err = i915_vma_wait_for_bind(vma); |
276 | if (err) |
277 | goto out_unlock; |
278 | } else { |
279 | err = igt_fill_check_buffer(obj, gt, fill: false); |
280 | } |
281 | |
282 | out_unlock: |
283 | i915_gem_object_unlock(obj); |
284 | out_put: |
285 | i915_gem_object_put(obj); |
286 | |
287 | return err; |
288 | } |
289 | |
290 | static int igt_lmem_pages_failsafe_migrate(void *arg) |
291 | { |
292 | int fail_gpu, fail_alloc, ban_memcpy, ret; |
293 | struct intel_gt *gt = arg; |
294 | |
295 | for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) { |
296 | for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) { |
297 | for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) { |
298 | pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n" , |
299 | fail_gpu, fail_alloc, ban_memcpy); |
300 | i915_ttm_migrate_set_ban_memcpy(ban: ban_memcpy); |
301 | i915_ttm_migrate_set_failure_modes(gpu_migration: fail_gpu, |
302 | work_allocation: fail_alloc); |
303 | ret = __igt_lmem_pages_migrate(gt, NULL, NULL, |
304 | NULL, NULL, |
305 | borked_migrate: ban_memcpy && |
306 | fail_gpu); |
307 | |
308 | if (ban_memcpy && fail_gpu) { |
309 | struct intel_gt *__gt; |
310 | unsigned int id; |
311 | |
312 | if (ret != -EIO) { |
313 | pr_err("expected -EIO, got (%d)\n" , ret); |
314 | ret = -EINVAL; |
315 | } else { |
316 | ret = 0; |
317 | } |
318 | |
319 | for_each_gt(__gt, gt->i915, id) { |
320 | intel_wakeref_t wakeref; |
321 | bool wedged; |
322 | |
323 | mutex_lock(&__gt->reset.mutex); |
324 | wedged = test_bit(I915_WEDGED, &__gt->reset.flags); |
325 | mutex_unlock(lock: &__gt->reset.mutex); |
326 | |
327 | if (fail_gpu && !fail_alloc) { |
328 | if (!wedged) { |
329 | pr_err("gt(%u) not wedged\n" , id); |
330 | ret = -EINVAL; |
331 | continue; |
332 | } |
333 | } else if (wedged) { |
334 | pr_err("gt(%u) incorrectly wedged\n" , id); |
335 | ret = -EINVAL; |
336 | } else { |
337 | continue; |
338 | } |
339 | |
340 | wakeref = intel_runtime_pm_get(rpm: __gt->uncore->rpm); |
341 | igt_global_reset_lock(gt: __gt); |
342 | intel_gt_reset(gt: __gt, ALL_ENGINES, NULL); |
343 | igt_global_reset_unlock(gt: __gt); |
344 | intel_runtime_pm_put(rpm: __gt->uncore->rpm, wref: wakeref); |
345 | } |
346 | if (ret) |
347 | goto out_err; |
348 | } |
349 | } |
350 | } |
351 | } |
352 | |
353 | out_err: |
354 | i915_ttm_migrate_set_failure_modes(gpu_migration: false, work_allocation: false); |
355 | i915_ttm_migrate_set_ban_memcpy(ban: false); |
356 | return ret; |
357 | } |
358 | |
359 | /* |
360 | * This subtest tests that unbinding at migration is indeed performed |
361 | * async. We launch a spinner and a number of migrations depending on |
362 | * that spinner to have terminated. Before each migration we bind a |
363 | * vma, which should then be async unbound by the migration operation. |
364 | * If we are able to schedule migrations without blocking while the |
365 | * spinner is still running, those unbinds are indeed async and non- |
366 | * blocking. |
367 | * |
368 | * Note that each async bind operation is awaiting the previous migration |
369 | * due to the moving fence resulting from the migration. |
370 | */ |
371 | static int igt_async_migrate(struct intel_gt *gt) |
372 | { |
373 | struct intel_engine_cs *engine; |
374 | enum intel_engine_id id; |
375 | struct i915_ppgtt *ppgtt; |
376 | struct igt_spinner spin; |
377 | int err; |
378 | |
379 | ppgtt = i915_ppgtt_create(gt, lmem_pt_obj_flags: 0); |
380 | if (IS_ERR(ptr: ppgtt)) |
381 | return PTR_ERR(ptr: ppgtt); |
382 | |
383 | if (igt_spinner_init(spin: &spin, gt)) { |
384 | err = -ENOMEM; |
385 | goto out_spin; |
386 | } |
387 | |
388 | for_each_engine(engine, gt, id) { |
389 | struct ttm_operation_ctx ctx = { |
390 | .interruptible = true |
391 | }; |
392 | struct dma_fence *spin_fence; |
393 | struct intel_context *ce; |
394 | struct i915_request *rq; |
395 | struct i915_deps deps; |
396 | |
397 | ce = intel_context_create(engine); |
398 | if (IS_ERR(ptr: ce)) { |
399 | err = PTR_ERR(ptr: ce); |
400 | goto out_ce; |
401 | } |
402 | |
403 | /* |
404 | * Use MI_NOOP, making the spinner non-preemptible. If there |
405 | * is a code path where we fail async operation due to the |
406 | * running spinner, we will block and fail to end the |
407 | * spinner resulting in a deadlock. But with a non- |
408 | * preemptible spinner, hangcheck will terminate the spinner |
409 | * for us, and we will later detect that and fail the test. |
410 | */ |
411 | rq = igt_spinner_create_request(spin: &spin, ce, MI_NOOP); |
412 | intel_context_put(ce); |
413 | if (IS_ERR(ptr: rq)) { |
414 | err = PTR_ERR(ptr: rq); |
415 | goto out_ce; |
416 | } |
417 | |
418 | i915_deps_init(deps: &deps, GFP_KERNEL); |
419 | err = i915_deps_add_dependency(deps: &deps, fence: &rq->fence, ctx: &ctx); |
420 | spin_fence = dma_fence_get(fence: &rq->fence); |
421 | i915_request_add(rq); |
422 | if (err) |
423 | goto out_ce; |
424 | |
425 | err = __igt_lmem_pages_migrate(gt, vm: &ppgtt->vm, deps: &deps, spin: &spin, |
426 | spin_fence, borked_migrate: false); |
427 | i915_deps_fini(deps: &deps); |
428 | dma_fence_put(fence: spin_fence); |
429 | if (err) |
430 | goto out_ce; |
431 | } |
432 | |
433 | out_ce: |
434 | igt_spinner_fini(spin: &spin); |
435 | out_spin: |
436 | i915_vm_put(vm: &ppgtt->vm); |
437 | |
438 | return err; |
439 | } |
440 | |
441 | /* |
442 | * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while |
443 | * arming the migration error check and block async migration. This |
444 | * will cause us to deadlock and hangcheck will terminate the spinner |
445 | * causing the test to fail. |
446 | */ |
447 | #define ASYNC_FAIL_ALLOC 1 |
448 | static int igt_lmem_async_migrate(void *arg) |
449 | { |
450 | int fail_gpu, fail_alloc, ban_memcpy, ret; |
451 | struct intel_gt *gt = arg; |
452 | |
453 | for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) { |
454 | for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) { |
455 | for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) { |
456 | pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n" , |
457 | fail_gpu, fail_alloc, ban_memcpy); |
458 | i915_ttm_migrate_set_ban_memcpy(ban: ban_memcpy); |
459 | i915_ttm_migrate_set_failure_modes(gpu_migration: fail_gpu, |
460 | work_allocation: fail_alloc); |
461 | ret = igt_async_migrate(gt); |
462 | |
463 | if (fail_gpu && ban_memcpy) { |
464 | struct intel_gt *__gt; |
465 | unsigned int id; |
466 | |
467 | if (ret != -EIO) { |
468 | pr_err("expected -EIO, got (%d)\n" , ret); |
469 | ret = -EINVAL; |
470 | } else { |
471 | ret = 0; |
472 | } |
473 | |
474 | for_each_gt(__gt, gt->i915, id) { |
475 | intel_wakeref_t wakeref; |
476 | bool wedged; |
477 | |
478 | mutex_lock(&__gt->reset.mutex); |
479 | wedged = test_bit(I915_WEDGED, &__gt->reset.flags); |
480 | mutex_unlock(lock: &__gt->reset.mutex); |
481 | |
482 | if (fail_gpu && !fail_alloc) { |
483 | if (!wedged) { |
484 | pr_err("gt(%u) not wedged\n" , id); |
485 | ret = -EINVAL; |
486 | continue; |
487 | } |
488 | } else if (wedged) { |
489 | pr_err("gt(%u) incorrectly wedged\n" , id); |
490 | ret = -EINVAL; |
491 | } else { |
492 | continue; |
493 | } |
494 | |
495 | wakeref = intel_runtime_pm_get(rpm: __gt->uncore->rpm); |
496 | igt_global_reset_lock(gt: __gt); |
497 | intel_gt_reset(gt: __gt, ALL_ENGINES, NULL); |
498 | igt_global_reset_unlock(gt: __gt); |
499 | intel_runtime_pm_put(rpm: __gt->uncore->rpm, wref: wakeref); |
500 | } |
501 | } |
502 | if (ret) |
503 | goto out_err; |
504 | } |
505 | } |
506 | } |
507 | |
508 | out_err: |
509 | i915_ttm_migrate_set_failure_modes(gpu_migration: false, work_allocation: false); |
510 | i915_ttm_migrate_set_ban_memcpy(ban: false); |
511 | return ret; |
512 | } |
513 | |
514 | int i915_gem_migrate_live_selftests(struct drm_i915_private *i915) |
515 | { |
516 | static const struct i915_subtest tests[] = { |
517 | SUBTEST(igt_smem_create_migrate), |
518 | SUBTEST(igt_lmem_create_migrate), |
519 | SUBTEST(igt_same_create_migrate), |
520 | SUBTEST(igt_lmem_pages_failsafe_migrate), |
521 | SUBTEST(igt_lmem_async_migrate), |
522 | }; |
523 | |
524 | if (!HAS_LMEM(i915)) |
525 | return 0; |
526 | |
527 | return intel_gt_live_subtests(tests, to_gt(i915)); |
528 | } |
529 | |