1 | /* |
2 | * Copyright © 2016 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include "gem/i915_gem_internal.h" |
26 | #include "gem/i915_gem_pm.h" |
27 | #include "gem/selftests/igt_gem_utils.h" |
28 | #include "gem/selftests/mock_context.h" |
29 | #include "gt/intel_gt.h" |
30 | #include "gt/intel_gt_print.h" |
31 | |
32 | #include "i915_selftest.h" |
33 | |
34 | #include "igt_flush_test.h" |
35 | #include "lib_sw_fence.h" |
36 | #include "mock_drm.h" |
37 | #include "mock_gem_device.h" |
38 | |
39 | static void quirk_add(struct drm_i915_gem_object *obj, |
40 | struct list_head *objects) |
41 | { |
42 | /* quirk is only for live tiled objects, use it to declare ownership */ |
43 | GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); |
44 | i915_gem_object_set_tiling_quirk(obj); |
45 | list_add(new: &obj->st_link, head: objects); |
46 | } |
47 | |
48 | static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects) |
49 | { |
50 | struct drm_i915_gem_object *obj; |
51 | unsigned long count; |
52 | |
53 | count = 0; |
54 | do { |
55 | struct i915_vma *vma; |
56 | |
57 | obj = i915_gem_object_create_internal(i915: ggtt->vm.i915, |
58 | I915_GTT_PAGE_SIZE); |
59 | if (IS_ERR(ptr: obj)) |
60 | return PTR_ERR(ptr: obj); |
61 | |
62 | vma = i915_gem_object_ggtt_pin(obj, NULL, size: 0, alignment: 0, flags: 0); |
63 | if (IS_ERR(ptr: vma)) { |
64 | i915_gem_object_put(obj); |
65 | if (vma == ERR_PTR(error: -ENOSPC)) |
66 | break; |
67 | |
68 | return PTR_ERR(ptr: vma); |
69 | } |
70 | |
71 | quirk_add(obj, objects); |
72 | count++; |
73 | } while (1); |
74 | pr_debug("Filled GGTT with %lu pages [%llu total]\n" , |
75 | count, ggtt->vm.total / PAGE_SIZE); |
76 | |
77 | if (list_empty(head: &ggtt->vm.bound_list)) { |
78 | pr_err("No objects on the GGTT inactive list!\n" ); |
79 | return -EINVAL; |
80 | } |
81 | |
82 | return 0; |
83 | } |
84 | |
85 | static void unpin_ggtt(struct i915_ggtt *ggtt) |
86 | { |
87 | struct i915_vma *vma; |
88 | |
89 | list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) |
90 | if (i915_gem_object_has_tiling_quirk(obj: vma->obj)) |
91 | i915_vma_unpin(vma); |
92 | } |
93 | |
94 | static void cleanup_objects(struct i915_ggtt *ggtt, struct list_head *list) |
95 | { |
96 | struct drm_i915_gem_object *obj, *on; |
97 | |
98 | list_for_each_entry_safe(obj, on, list, st_link) { |
99 | GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj)); |
100 | i915_gem_object_set_tiling_quirk(obj); |
101 | i915_gem_object_put(obj); |
102 | } |
103 | |
104 | i915_gem_drain_freed_objects(i915: ggtt->vm.i915); |
105 | } |
106 | |
107 | static int igt_evict_something(void *arg) |
108 | { |
109 | struct intel_gt *gt = arg; |
110 | struct i915_ggtt *ggtt = gt->ggtt; |
111 | LIST_HEAD(objects); |
112 | int err; |
113 | |
114 | /* Fill the GGTT with pinned objects and try to evict one. */ |
115 | |
116 | err = populate_ggtt(ggtt, objects: &objects); |
117 | if (err) |
118 | goto cleanup; |
119 | |
120 | /* Everything is pinned, nothing should happen */ |
121 | mutex_lock(&ggtt->vm.mutex); |
122 | err = i915_gem_evict_something(vm: &ggtt->vm, NULL, |
123 | I915_GTT_PAGE_SIZE, alignment: 0, color: 0, |
124 | start: 0, U64_MAX, |
125 | flags: 0); |
126 | mutex_unlock(lock: &ggtt->vm.mutex); |
127 | if (err != -ENOSPC) { |
128 | pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n" , |
129 | err); |
130 | goto cleanup; |
131 | } |
132 | |
133 | unpin_ggtt(ggtt); |
134 | |
135 | /* Everything is unpinned, we should be able to evict something */ |
136 | mutex_lock(&ggtt->vm.mutex); |
137 | err = i915_gem_evict_something(vm: &ggtt->vm, NULL, |
138 | I915_GTT_PAGE_SIZE, alignment: 0, color: 0, |
139 | start: 0, U64_MAX, |
140 | flags: 0); |
141 | mutex_unlock(lock: &ggtt->vm.mutex); |
142 | if (err) { |
143 | pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n" , |
144 | err); |
145 | goto cleanup; |
146 | } |
147 | |
148 | cleanup: |
149 | cleanup_objects(ggtt, list: &objects); |
150 | return err; |
151 | } |
152 | |
153 | static int igt_overcommit(void *arg) |
154 | { |
155 | struct intel_gt *gt = arg; |
156 | struct i915_ggtt *ggtt = gt->ggtt; |
157 | struct drm_i915_gem_object *obj; |
158 | struct i915_vma *vma; |
159 | LIST_HEAD(objects); |
160 | int err; |
161 | |
162 | /* Fill the GGTT with pinned objects and then try to pin one more. |
163 | * We expect it to fail. |
164 | */ |
165 | |
166 | err = populate_ggtt(ggtt, objects: &objects); |
167 | if (err) |
168 | goto cleanup; |
169 | |
170 | obj = i915_gem_object_create_internal(i915: gt->i915, I915_GTT_PAGE_SIZE); |
171 | if (IS_ERR(ptr: obj)) { |
172 | err = PTR_ERR(ptr: obj); |
173 | goto cleanup; |
174 | } |
175 | |
176 | quirk_add(obj, objects: &objects); |
177 | |
178 | vma = i915_gem_object_ggtt_pin(obj, NULL, size: 0, alignment: 0, flags: 0); |
179 | if (vma != ERR_PTR(error: -ENOSPC)) { |
180 | pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n" , (int)PTR_ERR_OR_ZERO(vma)); |
181 | err = -EINVAL; |
182 | goto cleanup; |
183 | } |
184 | |
185 | cleanup: |
186 | cleanup_objects(ggtt, list: &objects); |
187 | return err; |
188 | } |
189 | |
190 | static int igt_evict_for_vma(void *arg) |
191 | { |
192 | struct intel_gt *gt = arg; |
193 | struct i915_ggtt *ggtt = gt->ggtt; |
194 | struct drm_mm_node target = { |
195 | .start = 0, |
196 | .size = 4096, |
197 | }; |
198 | LIST_HEAD(objects); |
199 | int err; |
200 | |
201 | /* Fill the GGTT with pinned objects and try to evict a range. */ |
202 | |
203 | err = populate_ggtt(ggtt, objects: &objects); |
204 | if (err) |
205 | goto cleanup; |
206 | |
207 | /* Everything is pinned, nothing should happen */ |
208 | mutex_lock(&ggtt->vm.mutex); |
209 | err = i915_gem_evict_for_node(vm: &ggtt->vm, NULL, target: &target, flags: 0); |
210 | mutex_unlock(lock: &ggtt->vm.mutex); |
211 | if (err != -ENOSPC) { |
212 | pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n" , |
213 | err); |
214 | goto cleanup; |
215 | } |
216 | |
217 | unpin_ggtt(ggtt); |
218 | |
219 | /* Everything is unpinned, we should be able to evict the node */ |
220 | mutex_lock(&ggtt->vm.mutex); |
221 | err = i915_gem_evict_for_node(vm: &ggtt->vm, NULL, target: &target, flags: 0); |
222 | mutex_unlock(lock: &ggtt->vm.mutex); |
223 | if (err) { |
224 | pr_err("i915_gem_evict_for_node returned err=%d\n" , |
225 | err); |
226 | goto cleanup; |
227 | } |
228 | |
229 | cleanup: |
230 | cleanup_objects(ggtt, list: &objects); |
231 | return err; |
232 | } |
233 | |
234 | static void mock_color_adjust(const struct drm_mm_node *node, |
235 | unsigned long color, |
236 | u64 *start, |
237 | u64 *end) |
238 | { |
239 | } |
240 | |
241 | static int igt_evict_for_cache_color(void *arg) |
242 | { |
243 | struct intel_gt *gt = arg; |
244 | struct i915_ggtt *ggtt = gt->ggtt; |
245 | const unsigned long flags = PIN_OFFSET_FIXED; |
246 | struct drm_mm_node target = { |
247 | .start = I915_GTT_PAGE_SIZE * 2, |
248 | .size = I915_GTT_PAGE_SIZE, |
249 | .color = i915_gem_get_pat_index(i915: gt->i915, level: I915_CACHE_LLC), |
250 | }; |
251 | struct drm_i915_gem_object *obj; |
252 | struct i915_vma *vma; |
253 | LIST_HEAD(objects); |
254 | int err; |
255 | |
256 | /* |
257 | * Currently the use of color_adjust for the GGTT is limited to cache |
258 | * coloring and guard pages, and so the presence of mm.color_adjust for |
259 | * the GGTT is assumed to be i915_ggtt_color_adjust, hence using a mock |
260 | * color adjust will work just fine for our purposes. |
261 | */ |
262 | ggtt->vm.mm.color_adjust = mock_color_adjust; |
263 | GEM_BUG_ON(!i915_vm_has_cache_coloring(&ggtt->vm)); |
264 | |
265 | obj = i915_gem_object_create_internal(i915: gt->i915, I915_GTT_PAGE_SIZE); |
266 | if (IS_ERR(ptr: obj)) { |
267 | err = PTR_ERR(ptr: obj); |
268 | goto cleanup; |
269 | } |
270 | i915_gem_object_set_cache_coherency(obj, cache_level: I915_CACHE_LLC); |
271 | quirk_add(obj, objects: &objects); |
272 | |
273 | vma = i915_gem_object_ggtt_pin(obj, NULL, size: 0, alignment: 0, |
274 | I915_GTT_PAGE_SIZE | flags); |
275 | if (IS_ERR(ptr: vma)) { |
276 | pr_err("[0]i915_gem_object_ggtt_pin failed\n" ); |
277 | err = PTR_ERR(ptr: vma); |
278 | goto cleanup; |
279 | } |
280 | |
281 | obj = i915_gem_object_create_internal(i915: gt->i915, I915_GTT_PAGE_SIZE); |
282 | if (IS_ERR(ptr: obj)) { |
283 | err = PTR_ERR(ptr: obj); |
284 | goto cleanup; |
285 | } |
286 | i915_gem_object_set_cache_coherency(obj, cache_level: I915_CACHE_LLC); |
287 | quirk_add(obj, objects: &objects); |
288 | |
289 | /* Neighbouring; same colour - should fit */ |
290 | vma = i915_gem_object_ggtt_pin(obj, NULL, size: 0, alignment: 0, |
291 | flags: (I915_GTT_PAGE_SIZE * 2) | flags); |
292 | if (IS_ERR(ptr: vma)) { |
293 | pr_err("[1]i915_gem_object_ggtt_pin failed\n" ); |
294 | err = PTR_ERR(ptr: vma); |
295 | goto cleanup; |
296 | } |
297 | |
298 | i915_vma_unpin(vma); |
299 | |
300 | /* Remove just the second vma */ |
301 | mutex_lock(&ggtt->vm.mutex); |
302 | err = i915_gem_evict_for_node(vm: &ggtt->vm, NULL, target: &target, flags: 0); |
303 | mutex_unlock(lock: &ggtt->vm.mutex); |
304 | if (err) { |
305 | pr_err("[0]i915_gem_evict_for_node returned err=%d\n" , err); |
306 | goto cleanup; |
307 | } |
308 | |
309 | /* Attempt to remove the first *pinned* vma, by removing the (empty) |
310 | * neighbour -- this should fail. |
311 | */ |
312 | target.color = i915_gem_get_pat_index(i915: gt->i915, level: I915_CACHE_L3_LLC); |
313 | |
314 | mutex_lock(&ggtt->vm.mutex); |
315 | err = i915_gem_evict_for_node(vm: &ggtt->vm, NULL, target: &target, flags: 0); |
316 | mutex_unlock(lock: &ggtt->vm.mutex); |
317 | if (!err) { |
318 | pr_err("[1]i915_gem_evict_for_node returned err=%d\n" , err); |
319 | err = -EINVAL; |
320 | goto cleanup; |
321 | } |
322 | |
323 | err = 0; |
324 | |
325 | cleanup: |
326 | unpin_ggtt(ggtt); |
327 | cleanup_objects(ggtt, list: &objects); |
328 | ggtt->vm.mm.color_adjust = NULL; |
329 | return err; |
330 | } |
331 | |
332 | static int igt_evict_vm(void *arg) |
333 | { |
334 | struct intel_gt *gt = arg; |
335 | struct i915_ggtt *ggtt = gt->ggtt; |
336 | struct i915_gem_ww_ctx ww; |
337 | LIST_HEAD(objects); |
338 | int err; |
339 | |
340 | /* Fill the GGTT with pinned objects and try to evict everything. */ |
341 | |
342 | err = populate_ggtt(ggtt, objects: &objects); |
343 | if (err) |
344 | goto cleanup; |
345 | |
346 | /* Everything is pinned, nothing should happen */ |
347 | mutex_lock(&ggtt->vm.mutex); |
348 | err = i915_gem_evict_vm(vm: &ggtt->vm, NULL, NULL); |
349 | mutex_unlock(lock: &ggtt->vm.mutex); |
350 | if (err) { |
351 | pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n" , |
352 | err); |
353 | goto cleanup; |
354 | } |
355 | |
356 | unpin_ggtt(ggtt); |
357 | |
358 | for_i915_gem_ww(&ww, err, false) { |
359 | mutex_lock(&ggtt->vm.mutex); |
360 | err = i915_gem_evict_vm(vm: &ggtt->vm, ww: &ww, NULL); |
361 | mutex_unlock(lock: &ggtt->vm.mutex); |
362 | } |
363 | |
364 | if (err) { |
365 | pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n" , |
366 | err); |
367 | goto cleanup; |
368 | } |
369 | |
370 | cleanup: |
371 | cleanup_objects(ggtt, list: &objects); |
372 | return err; |
373 | } |
374 | |
375 | static int igt_evict_contexts(void *arg) |
376 | { |
377 | const u64 PRETEND_GGTT_SIZE = 16ull << 20; |
378 | struct intel_gt *gt = arg; |
379 | struct i915_ggtt *ggtt = gt->ggtt; |
380 | struct drm_i915_private *i915 = gt->i915; |
381 | struct intel_engine_cs *engine; |
382 | enum intel_engine_id id; |
383 | struct reserved { |
384 | struct drm_mm_node node; |
385 | struct reserved *next; |
386 | } *reserved = NULL; |
387 | intel_wakeref_t wakeref; |
388 | struct drm_mm_node hole; |
389 | unsigned long count; |
390 | int err; |
391 | |
392 | /* |
393 | * The purpose of this test is to verify that we will trigger an |
394 | * eviction in the GGTT when constructing a request that requires |
395 | * additional space in the GGTT for pinning the context. This space |
396 | * is not directly tied to the request so reclaiming it requires |
397 | * extra work. |
398 | * |
399 | * As such this test is only meaningful for full-ppgtt environments |
400 | * where the GTT space of the request is separate from the GGTT |
401 | * allocation required to build the request. |
402 | */ |
403 | if (!HAS_FULL_PPGTT(i915)) |
404 | return 0; |
405 | |
406 | wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm); |
407 | |
408 | /* Reserve a block so that we know we have enough to fit a few rq */ |
409 | memset(&hole, 0, sizeof(hole)); |
410 | mutex_lock(&ggtt->vm.mutex); |
411 | err = i915_gem_gtt_insert(vm: &ggtt->vm, NULL, node: &hole, |
412 | size: PRETEND_GGTT_SIZE, alignment: 0, I915_COLOR_UNEVICTABLE, |
413 | start: 0, end: ggtt->vm.total, |
414 | PIN_NOEVICT); |
415 | if (err) |
416 | goto out_locked; |
417 | |
418 | /* Make the GGTT appear small by filling it with unevictable nodes */ |
419 | count = 0; |
420 | do { |
421 | struct reserved *r; |
422 | |
423 | mutex_unlock(lock: &ggtt->vm.mutex); |
424 | r = kcalloc(n: 1, size: sizeof(*r), GFP_KERNEL); |
425 | mutex_lock(&ggtt->vm.mutex); |
426 | if (!r) { |
427 | err = -ENOMEM; |
428 | goto out_locked; |
429 | } |
430 | |
431 | if (i915_gem_gtt_insert(vm: &ggtt->vm, NULL, node: &r->node, |
432 | size: 1ul << 20, alignment: 0, I915_COLOR_UNEVICTABLE, |
433 | start: 0, end: ggtt->vm.total, |
434 | PIN_NOEVICT)) { |
435 | kfree(objp: r); |
436 | break; |
437 | } |
438 | |
439 | r->next = reserved; |
440 | reserved = r; |
441 | |
442 | count++; |
443 | } while (1); |
444 | drm_mm_remove_node(node: &hole); |
445 | mutex_unlock(lock: &ggtt->vm.mutex); |
446 | pr_info("Filled GGTT with %lu 1MiB nodes\n" , count); |
447 | |
448 | /* Overfill the GGTT with context objects and so try to evict one. */ |
449 | for_each_engine(engine, gt, id) { |
450 | struct i915_sw_fence fence; |
451 | struct i915_request *last = NULL; |
452 | |
453 | count = 0; |
454 | onstack_fence_init(&fence); |
455 | do { |
456 | struct intel_context *ce; |
457 | struct i915_request *rq; |
458 | |
459 | ce = intel_context_create(engine); |
460 | if (IS_ERR(ptr: ce)) |
461 | break; |
462 | |
463 | /* We will need some GGTT space for the rq's context */ |
464 | igt_evict_ctl.fail_if_busy = true; |
465 | rq = intel_context_create_request(ce); |
466 | igt_evict_ctl.fail_if_busy = false; |
467 | intel_context_put(ce); |
468 | |
469 | if (IS_ERR(ptr: rq)) { |
470 | /* When full, fail_if_busy will trigger EBUSY */ |
471 | if (PTR_ERR(ptr: rq) != -EBUSY) { |
472 | pr_err("Unexpected error from request alloc (on %s): %d\n" , |
473 | engine->name, |
474 | (int)PTR_ERR(rq)); |
475 | err = PTR_ERR(ptr: rq); |
476 | } |
477 | break; |
478 | } |
479 | |
480 | /* Keep every request/ctx pinned until we are full */ |
481 | err = i915_sw_fence_await_sw_fence_gfp(fence: &rq->submit, |
482 | after: &fence, |
483 | GFP_KERNEL); |
484 | if (err < 0) |
485 | break; |
486 | |
487 | i915_request_add(rq); |
488 | count++; |
489 | if (last) |
490 | i915_request_put(rq: last); |
491 | last = i915_request_get(rq); |
492 | err = 0; |
493 | } while(1); |
494 | onstack_fence_fini(fence: &fence); |
495 | pr_info("Submitted %lu contexts/requests on %s\n" , |
496 | count, engine->name); |
497 | if (err) |
498 | break; |
499 | if (last) { |
500 | if (i915_request_wait(rq: last, flags: 0, HZ) < 0) { |
501 | err = -EIO; |
502 | i915_request_put(rq: last); |
503 | pr_err("Failed waiting for last request (on %s)" , |
504 | engine->name); |
505 | break; |
506 | } |
507 | i915_request_put(rq: last); |
508 | } |
509 | err = intel_gt_wait_for_idle(gt: engine->gt, HZ * 3); |
510 | if (err) { |
511 | gt_err(engine->gt, "Failed to idle GT (on %s)" , |
512 | engine->name); |
513 | break; |
514 | } |
515 | } |
516 | |
517 | mutex_lock(&ggtt->vm.mutex); |
518 | out_locked: |
519 | if (igt_flush_test(i915)) |
520 | err = -EIO; |
521 | while (reserved) { |
522 | struct reserved *next = reserved->next; |
523 | |
524 | drm_mm_remove_node(node: &reserved->node); |
525 | kfree(objp: reserved); |
526 | |
527 | reserved = next; |
528 | } |
529 | if (drm_mm_node_allocated(node: &hole)) |
530 | drm_mm_remove_node(node: &hole); |
531 | mutex_unlock(lock: &ggtt->vm.mutex); |
532 | intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref); |
533 | |
534 | return err; |
535 | } |
536 | |
537 | int i915_gem_evict_mock_selftests(void) |
538 | { |
539 | static const struct i915_subtest tests[] = { |
540 | SUBTEST(igt_evict_something), |
541 | SUBTEST(igt_evict_for_vma), |
542 | SUBTEST(igt_evict_for_cache_color), |
543 | SUBTEST(igt_evict_vm), |
544 | SUBTEST(igt_overcommit), |
545 | }; |
546 | struct drm_i915_private *i915; |
547 | intel_wakeref_t wakeref; |
548 | int err = 0; |
549 | |
550 | i915 = mock_gem_device(); |
551 | if (!i915) |
552 | return -ENOMEM; |
553 | |
554 | with_intel_runtime_pm(&i915->runtime_pm, wakeref) |
555 | err = i915_subtests(tests, to_gt(i915)); |
556 | |
557 | mock_destroy_device(i915); |
558 | return err; |
559 | } |
560 | |
561 | int i915_gem_evict_live_selftests(struct drm_i915_private *i915) |
562 | { |
563 | static const struct i915_subtest tests[] = { |
564 | SUBTEST(igt_evict_contexts), |
565 | }; |
566 | |
567 | if (intel_gt_is_wedged(gt: to_gt(i915))) |
568 | return 0; |
569 | |
570 | return intel_gt_live_subtests(tests, to_gt(i915)); |
571 | } |
572 | |