1 | /* |
2 | * Copyright © 2016 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include <linux/prime_numbers.h> |
26 | |
27 | #include "gem/i915_gem_context.h" |
28 | #include "gem/i915_gem_internal.h" |
29 | #include "gem/selftests/mock_context.h" |
30 | |
31 | #include "i915_scatterlist.h" |
32 | #include "i915_selftest.h" |
33 | |
34 | #include "mock_gem_device.h" |
35 | #include "mock_gtt.h" |
36 | |
37 | static bool assert_vma(struct i915_vma *vma, |
38 | struct drm_i915_gem_object *obj, |
39 | struct i915_gem_context *ctx) |
40 | { |
41 | bool ok = true; |
42 | |
43 | if (vma->vm != ctx->vm) { |
44 | pr_err("VMA created with wrong VM\n" ); |
45 | ok = false; |
46 | } |
47 | |
48 | if (vma->size != obj->base.size) { |
49 | pr_err("VMA created with wrong size, found %llu, expected %zu\n" , |
50 | vma->size, obj->base.size); |
51 | ok = false; |
52 | } |
53 | |
54 | if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) { |
55 | pr_err("VMA created with wrong type [%d]\n" , |
56 | vma->gtt_view.type); |
57 | ok = false; |
58 | } |
59 | |
60 | return ok; |
61 | } |
62 | |
63 | static struct i915_vma * |
64 | checked_vma_instance(struct drm_i915_gem_object *obj, |
65 | struct i915_address_space *vm, |
66 | const struct i915_gtt_view *view) |
67 | { |
68 | struct i915_vma *vma; |
69 | bool ok = true; |
70 | |
71 | vma = i915_vma_instance(obj, vm, view); |
72 | if (IS_ERR(ptr: vma)) |
73 | return vma; |
74 | |
75 | /* Manual checks, will be reinforced by i915_vma_compare! */ |
76 | if (vma->vm != vm) { |
77 | pr_err("VMA's vm [%p] does not match request [%p]\n" , |
78 | vma->vm, vm); |
79 | ok = false; |
80 | } |
81 | |
82 | if (i915_is_ggtt(vm) != i915_vma_is_ggtt(vma)) { |
83 | pr_err("VMA ggtt status [%d] does not match parent [%d]\n" , |
84 | i915_vma_is_ggtt(vma), i915_is_ggtt(vm)); |
85 | ok = false; |
86 | } |
87 | |
88 | if (i915_vma_compare(vma, vm, view)) { |
89 | pr_err("i915_vma_compare failed with create parameters!\n" ); |
90 | return ERR_PTR(error: -EINVAL); |
91 | } |
92 | |
93 | if (i915_vma_compare(vma, vm: vma->vm, |
94 | view: i915_vma_is_ggtt(vma) ? &vma->gtt_view : NULL)) { |
95 | pr_err("i915_vma_compare failed with itself\n" ); |
96 | return ERR_PTR(error: -EINVAL); |
97 | } |
98 | |
99 | if (!ok) { |
100 | pr_err("i915_vma_compare failed to detect the difference!\n" ); |
101 | return ERR_PTR(error: -EINVAL); |
102 | } |
103 | |
104 | return vma; |
105 | } |
106 | |
107 | static int create_vmas(struct drm_i915_private *i915, |
108 | struct list_head *objects, |
109 | struct list_head *contexts) |
110 | { |
111 | struct drm_i915_gem_object *obj; |
112 | struct i915_gem_context *ctx; |
113 | int pinned; |
114 | |
115 | list_for_each_entry(obj, objects, st_link) { |
116 | for (pinned = 0; pinned <= 1; pinned++) { |
117 | list_for_each_entry(ctx, contexts, link) { |
118 | struct i915_address_space *vm; |
119 | struct i915_vma *vma; |
120 | int err; |
121 | |
122 | vm = i915_gem_context_get_eb_vm(ctx); |
123 | vma = checked_vma_instance(obj, vm, NULL); |
124 | i915_vm_put(vm); |
125 | if (IS_ERR(ptr: vma)) |
126 | return PTR_ERR(ptr: vma); |
127 | |
128 | if (!assert_vma(vma, obj, ctx)) { |
129 | pr_err("VMA lookup/create failed\n" ); |
130 | return -EINVAL; |
131 | } |
132 | |
133 | if (!pinned) { |
134 | err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_USER); |
135 | if (err) { |
136 | pr_err("Failed to pin VMA\n" ); |
137 | return err; |
138 | } |
139 | } else { |
140 | i915_vma_unpin(vma); |
141 | } |
142 | } |
143 | } |
144 | } |
145 | |
146 | return 0; |
147 | } |
148 | |
149 | static int igt_vma_create(void *arg) |
150 | { |
151 | struct i915_ggtt *ggtt = arg; |
152 | struct drm_i915_private *i915 = ggtt->vm.i915; |
153 | struct drm_i915_gem_object *obj, *on; |
154 | struct i915_gem_context *ctx, *cn; |
155 | unsigned long num_obj, num_ctx; |
156 | unsigned long no, nc; |
157 | IGT_TIMEOUT(end_time); |
158 | LIST_HEAD(contexts); |
159 | LIST_HEAD(objects); |
160 | int err = -ENOMEM; |
161 | |
162 | /* Exercise creating many vma amonst many objections, checking the |
163 | * vma creation and lookup routines. |
164 | */ |
165 | |
166 | no = 0; |
167 | for_each_prime_number(num_obj, ULONG_MAX - 1) { |
168 | for (; no < num_obj; no++) { |
169 | obj = i915_gem_object_create_internal(i915, PAGE_SIZE); |
170 | if (IS_ERR(ptr: obj)) |
171 | goto out; |
172 | |
173 | list_add(new: &obj->st_link, head: &objects); |
174 | } |
175 | |
176 | nc = 0; |
177 | for_each_prime_number(num_ctx, 2 * BITS_PER_LONG) { |
178 | for (; nc < num_ctx; nc++) { |
179 | ctx = mock_context(i915, name: "mock" ); |
180 | if (!ctx) |
181 | goto out; |
182 | |
183 | list_move(list: &ctx->link, head: &contexts); |
184 | } |
185 | |
186 | err = create_vmas(i915, objects: &objects, contexts: &contexts); |
187 | if (err) |
188 | goto out; |
189 | |
190 | if (igt_timeout(end_time, |
191 | "%s timed out: after %lu objects in %lu contexts\n" , |
192 | __func__, no, nc)) |
193 | goto end; |
194 | } |
195 | |
196 | list_for_each_entry_safe(ctx, cn, &contexts, link) { |
197 | list_del_init(entry: &ctx->link); |
198 | mock_context_close(ctx); |
199 | } |
200 | |
201 | cond_resched(); |
202 | } |
203 | |
204 | end: |
205 | /* Final pass to lookup all created contexts */ |
206 | err = create_vmas(i915, objects: &objects, contexts: &contexts); |
207 | out: |
208 | list_for_each_entry_safe(ctx, cn, &contexts, link) { |
209 | list_del_init(entry: &ctx->link); |
210 | mock_context_close(ctx); |
211 | } |
212 | |
213 | list_for_each_entry_safe(obj, on, &objects, st_link) |
214 | i915_gem_object_put(obj); |
215 | return err; |
216 | } |
217 | |
218 | struct pin_mode { |
219 | u64 size; |
220 | u64 flags; |
221 | bool (*assert)(const struct i915_vma *, |
222 | const struct pin_mode *mode, |
223 | int result); |
224 | const char *string; |
225 | }; |
226 | |
227 | static bool assert_pin_valid(const struct i915_vma *vma, |
228 | const struct pin_mode *mode, |
229 | int result) |
230 | { |
231 | if (result) |
232 | return false; |
233 | |
234 | if (i915_vma_misplaced(vma, size: mode->size, alignment: 0, flags: mode->flags)) |
235 | return false; |
236 | |
237 | return true; |
238 | } |
239 | |
240 | __maybe_unused |
241 | static bool assert_pin_enospc(const struct i915_vma *vma, |
242 | const struct pin_mode *mode, |
243 | int result) |
244 | { |
245 | return result == -ENOSPC; |
246 | } |
247 | |
248 | __maybe_unused |
249 | static bool assert_pin_einval(const struct i915_vma *vma, |
250 | const struct pin_mode *mode, |
251 | int result) |
252 | { |
253 | return result == -EINVAL; |
254 | } |
255 | |
256 | static int igt_vma_pin1(void *arg) |
257 | { |
258 | struct i915_ggtt *ggtt = arg; |
259 | const struct pin_mode modes[] = { |
260 | #define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " } |
261 | #define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" } |
262 | #define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL) |
263 | #define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC) |
264 | VALID(0, PIN_GLOBAL), |
265 | VALID(0, PIN_GLOBAL | PIN_MAPPABLE), |
266 | |
267 | VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 4096), |
268 | VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 8192), |
269 | VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), |
270 | VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), |
271 | VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)), |
272 | |
273 | VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)), |
274 | INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | ggtt->mappable_end), |
275 | VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)), |
276 | INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | ggtt->vm.total), |
277 | INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | round_down(U64_MAX, PAGE_SIZE)), |
278 | |
279 | VALID(4096, PIN_GLOBAL), |
280 | VALID(8192, PIN_GLOBAL), |
281 | VALID(ggtt->mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE), |
282 | VALID(ggtt->mappable_end, PIN_GLOBAL | PIN_MAPPABLE), |
283 | NOSPACE(ggtt->mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE), |
284 | VALID(ggtt->vm.total - 4096, PIN_GLOBAL), |
285 | VALID(ggtt->vm.total, PIN_GLOBAL), |
286 | NOSPACE(ggtt->vm.total + 4096, PIN_GLOBAL), |
287 | NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL), |
288 | INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)), |
289 | INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)), |
290 | INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)), |
291 | |
292 | VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), |
293 | |
294 | #if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) |
295 | /* Misusing BIAS is a programming error (it is not controllable |
296 | * from userspace) so when debugging is enabled, it explodes. |
297 | * However, the tests are still quite interesting for checking |
298 | * variable start, end and size. |
299 | */ |
300 | NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | ggtt->mappable_end), |
301 | NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | ggtt->vm.total), |
302 | NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)), |
303 | NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)), |
304 | #endif |
305 | { }, |
306 | #undef NOSPACE |
307 | #undef INVALID |
308 | #undef __INVALID |
309 | #undef VALID |
310 | }, *m; |
311 | struct drm_i915_gem_object *obj; |
312 | struct i915_vma *vma; |
313 | int err = -EINVAL; |
314 | |
315 | /* Exercise all the weird and wonderful i915_vma_pin requests, |
316 | * focusing on error handling of boundary conditions. |
317 | */ |
318 | |
319 | GEM_BUG_ON(!drm_mm_clean(&ggtt->vm.mm)); |
320 | |
321 | obj = i915_gem_object_create_internal(i915: ggtt->vm.i915, PAGE_SIZE); |
322 | if (IS_ERR(ptr: obj)) |
323 | return PTR_ERR(ptr: obj); |
324 | |
325 | vma = checked_vma_instance(obj, vm: &ggtt->vm, NULL); |
326 | if (IS_ERR(ptr: vma)) |
327 | goto out; |
328 | |
329 | for (m = modes; m->assert; m++) { |
330 | err = i915_vma_pin(vma, size: m->size, alignment: 0, flags: m->flags); |
331 | if (!m->assert(vma, m, err)) { |
332 | pr_err("%s to pin single page into GGTT with mode[%d:%s]: size=%llx flags=%llx, err=%d\n" , |
333 | m->assert == assert_pin_valid ? "Failed" : "Unexpectedly succeeded" , |
334 | (int)(m - modes), m->string, m->size, m->flags, |
335 | err); |
336 | if (!err) |
337 | i915_vma_unpin(vma); |
338 | err = -EINVAL; |
339 | goto out; |
340 | } |
341 | |
342 | if (!err) { |
343 | i915_vma_unpin(vma); |
344 | err = i915_vma_unbind_unlocked(vma); |
345 | if (err) { |
346 | pr_err("Failed to unbind single page from GGTT, err=%d\n" , err); |
347 | goto out; |
348 | } |
349 | } |
350 | |
351 | cond_resched(); |
352 | } |
353 | |
354 | err = 0; |
355 | out: |
356 | i915_gem_object_put(obj); |
357 | return err; |
358 | } |
359 | |
360 | static unsigned long rotated_index(const struct intel_rotation_info *r, |
361 | unsigned int n, |
362 | unsigned int x, |
363 | unsigned int y) |
364 | { |
365 | return (r->plane[n].src_stride * (r->plane[n].height - y - 1) + |
366 | r->plane[n].offset + x); |
367 | } |
368 | |
369 | static struct scatterlist * |
370 | assert_rotated(struct drm_i915_gem_object *obj, |
371 | const struct intel_rotation_info *r, unsigned int n, |
372 | struct scatterlist *sg) |
373 | { |
374 | unsigned int x, y; |
375 | |
376 | for (x = 0; x < r->plane[n].width; x++) { |
377 | unsigned int left; |
378 | |
379 | for (y = 0; y < r->plane[n].height; y++) { |
380 | unsigned long src_idx; |
381 | dma_addr_t src; |
382 | |
383 | if (!sg) { |
384 | pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n" , |
385 | n, x, y); |
386 | return ERR_PTR(error: -EINVAL); |
387 | } |
388 | |
389 | src_idx = rotated_index(r, n, x, y); |
390 | src = i915_gem_object_get_dma_address(obj, src_idx); |
391 | |
392 | if (sg_dma_len(sg) != PAGE_SIZE) { |
393 | pr_err("Invalid sg.length, found %d, expected %lu for rotated page (%d, %d) [src index %lu]\n" , |
394 | sg_dma_len(sg), PAGE_SIZE, |
395 | x, y, src_idx); |
396 | return ERR_PTR(error: -EINVAL); |
397 | } |
398 | |
399 | if (sg_dma_address(sg) != src) { |
400 | pr_err("Invalid address for rotated page (%d, %d) [src index %lu]\n" , |
401 | x, y, src_idx); |
402 | return ERR_PTR(error: -EINVAL); |
403 | } |
404 | |
405 | sg = sg_next(sg); |
406 | } |
407 | |
408 | left = (r->plane[n].dst_stride - y) * PAGE_SIZE; |
409 | |
410 | if (!left) |
411 | continue; |
412 | |
413 | if (!sg) { |
414 | pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n" , |
415 | n, x, y); |
416 | return ERR_PTR(error: -EINVAL); |
417 | } |
418 | |
419 | if (sg_dma_len(sg) != left) { |
420 | pr_err("Invalid sg.length, found %d, expected %u for rotated page (%d, %d)\n" , |
421 | sg_dma_len(sg), left, x, y); |
422 | return ERR_PTR(error: -EINVAL); |
423 | } |
424 | |
425 | if (sg_dma_address(sg) != 0) { |
426 | pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n" , |
427 | &sg_dma_address(sg), x, y); |
428 | return ERR_PTR(error: -EINVAL); |
429 | } |
430 | |
431 | sg = sg_next(sg); |
432 | } |
433 | |
434 | return sg; |
435 | } |
436 | |
437 | static unsigned long remapped_index(const struct intel_remapped_info *r, |
438 | unsigned int n, |
439 | unsigned int x, |
440 | unsigned int y) |
441 | { |
442 | return (r->plane[n].src_stride * y + |
443 | r->plane[n].offset + x); |
444 | } |
445 | |
446 | static struct scatterlist * |
447 | assert_remapped(struct drm_i915_gem_object *obj, |
448 | const struct intel_remapped_info *r, unsigned int n, |
449 | struct scatterlist *sg) |
450 | { |
451 | unsigned int x, y; |
452 | unsigned int left = 0; |
453 | unsigned int offset; |
454 | |
455 | for (y = 0; y < r->plane[n].height; y++) { |
456 | for (x = 0; x < r->plane[n].width; x++) { |
457 | unsigned long src_idx; |
458 | dma_addr_t src; |
459 | |
460 | if (!sg) { |
461 | pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n" , |
462 | n, x, y); |
463 | return ERR_PTR(error: -EINVAL); |
464 | } |
465 | if (!left) { |
466 | offset = 0; |
467 | left = sg_dma_len(sg); |
468 | } |
469 | |
470 | src_idx = remapped_index(r, n, x, y); |
471 | src = i915_gem_object_get_dma_address(obj, src_idx); |
472 | |
473 | if (left < PAGE_SIZE || left & (PAGE_SIZE-1)) { |
474 | pr_err("Invalid sg.length, found %d, expected %lu for remapped page (%d, %d) [src index %lu]\n" , |
475 | sg_dma_len(sg), PAGE_SIZE, |
476 | x, y, src_idx); |
477 | return ERR_PTR(error: -EINVAL); |
478 | } |
479 | |
480 | if (sg_dma_address(sg) + offset != src) { |
481 | pr_err("Invalid address for remapped page (%d, %d) [src index %lu]\n" , |
482 | x, y, src_idx); |
483 | return ERR_PTR(error: -EINVAL); |
484 | } |
485 | |
486 | left -= PAGE_SIZE; |
487 | offset += PAGE_SIZE; |
488 | |
489 | |
490 | if (!left) |
491 | sg = sg_next(sg); |
492 | } |
493 | |
494 | if (left) { |
495 | pr_err("Unexpected sg tail with %d size for remapped page (%d, %d)\n" , |
496 | left, |
497 | x, y); |
498 | return ERR_PTR(error: -EINVAL); |
499 | } |
500 | |
501 | left = (r->plane[n].dst_stride - r->plane[n].width) * PAGE_SIZE; |
502 | |
503 | if (!left) |
504 | continue; |
505 | |
506 | if (!sg) { |
507 | pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n" , |
508 | n, x, y); |
509 | return ERR_PTR(error: -EINVAL); |
510 | } |
511 | |
512 | if (sg_dma_len(sg) != left) { |
513 | pr_err("Invalid sg.length, found %u, expected %u for remapped page (%d, %d)\n" , |
514 | sg_dma_len(sg), left, |
515 | x, y); |
516 | return ERR_PTR(error: -EINVAL); |
517 | } |
518 | |
519 | if (sg_dma_address(sg) != 0) { |
520 | pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n" , |
521 | &sg_dma_address(sg), |
522 | x, y); |
523 | return ERR_PTR(error: -EINVAL); |
524 | } |
525 | |
526 | sg = sg_next(sg); |
527 | left = 0; |
528 | } |
529 | |
530 | return sg; |
531 | } |
532 | |
533 | static unsigned int remapped_size(enum i915_gtt_view_type view_type, |
534 | const struct intel_remapped_plane_info *a, |
535 | const struct intel_remapped_plane_info *b) |
536 | { |
537 | |
538 | if (view_type == I915_GTT_VIEW_ROTATED) |
539 | return a->dst_stride * a->width + b->dst_stride * b->width; |
540 | else |
541 | return a->dst_stride * a->height + b->dst_stride * b->height; |
542 | } |
543 | |
544 | static int igt_vma_rotate_remap(void *arg) |
545 | { |
546 | struct i915_ggtt *ggtt = arg; |
547 | struct i915_address_space *vm = &ggtt->vm; |
548 | struct drm_i915_gem_object *obj; |
549 | const struct intel_remapped_plane_info planes[] = { |
550 | { .width = 1, .height = 1, .src_stride = 1 }, |
551 | { .width = 2, .height = 2, .src_stride = 2 }, |
552 | { .width = 4, .height = 4, .src_stride = 4 }, |
553 | { .width = 8, .height = 8, .src_stride = 8 }, |
554 | |
555 | { .width = 3, .height = 5, .src_stride = 3 }, |
556 | { .width = 3, .height = 5, .src_stride = 4 }, |
557 | { .width = 3, .height = 5, .src_stride = 5 }, |
558 | |
559 | { .width = 5, .height = 3, .src_stride = 5 }, |
560 | { .width = 5, .height = 3, .src_stride = 7 }, |
561 | { .width = 5, .height = 3, .src_stride = 9 }, |
562 | |
563 | { .width = 4, .height = 6, .src_stride = 6 }, |
564 | { .width = 6, .height = 4, .src_stride = 6 }, |
565 | |
566 | { .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 }, |
567 | { .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 }, |
568 | { .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 }, |
569 | |
570 | { } |
571 | }, *a, *b; |
572 | enum i915_gtt_view_type types[] = { |
573 | I915_GTT_VIEW_ROTATED, |
574 | I915_GTT_VIEW_REMAPPED, |
575 | 0, |
576 | }, *t; |
577 | const unsigned int max_pages = 64; |
578 | int err = -ENOMEM; |
579 | |
580 | /* Create VMA for many different combinations of planes and check |
581 | * that the page layout within the rotated VMA match our expectations. |
582 | */ |
583 | |
584 | obj = i915_gem_object_create_internal(i915: vm->i915, size: max_pages * PAGE_SIZE); |
585 | if (IS_ERR(ptr: obj)) |
586 | goto out; |
587 | |
588 | for (t = types; *t; t++) { |
589 | for (a = planes; a->width; a++) { |
590 | for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) { |
591 | struct i915_gtt_view view = { |
592 | .type = *t, |
593 | .remapped.plane[0] = *a, |
594 | .remapped.plane[1] = *b, |
595 | }; |
596 | struct intel_remapped_plane_info *plane_info = view.remapped.plane; |
597 | unsigned int n, max_offset; |
598 | |
599 | max_offset = max(plane_info[0].src_stride * plane_info[0].height, |
600 | plane_info[1].src_stride * plane_info[1].height); |
601 | GEM_BUG_ON(max_offset > max_pages); |
602 | max_offset = max_pages - max_offset; |
603 | |
604 | if (!plane_info[0].dst_stride) |
605 | plane_info[0].dst_stride = view.type == I915_GTT_VIEW_ROTATED ? |
606 | plane_info[0].height : |
607 | plane_info[0].width; |
608 | if (!plane_info[1].dst_stride) |
609 | plane_info[1].dst_stride = view.type == I915_GTT_VIEW_ROTATED ? |
610 | plane_info[1].height : |
611 | plane_info[1].width; |
612 | |
613 | for_each_prime_number_from(plane_info[0].offset, 0, max_offset) { |
614 | for_each_prime_number_from(plane_info[1].offset, 0, max_offset) { |
615 | struct scatterlist *sg; |
616 | struct i915_vma *vma; |
617 | unsigned int expected_pages; |
618 | |
619 | vma = checked_vma_instance(obj, vm, view: &view); |
620 | if (IS_ERR(ptr: vma)) { |
621 | err = PTR_ERR(ptr: vma); |
622 | goto out_object; |
623 | } |
624 | |
625 | err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL); |
626 | if (err) { |
627 | pr_err("Failed to pin VMA, err=%d\n" , err); |
628 | goto out_object; |
629 | } |
630 | |
631 | expected_pages = remapped_size(view_type: view.type, a: &plane_info[0], b: &plane_info[1]); |
632 | |
633 | if (view.type == I915_GTT_VIEW_ROTATED && |
634 | vma->size != expected_pages * PAGE_SIZE) { |
635 | pr_err("VMA is wrong size, expected %lu, found %llu\n" , |
636 | PAGE_SIZE * expected_pages, vma->size); |
637 | err = -EINVAL; |
638 | goto out_object; |
639 | } |
640 | |
641 | if (view.type == I915_GTT_VIEW_REMAPPED && |
642 | vma->size > expected_pages * PAGE_SIZE) { |
643 | pr_err("VMA is wrong size, expected %lu, found %llu\n" , |
644 | PAGE_SIZE * expected_pages, vma->size); |
645 | err = -EINVAL; |
646 | goto out_object; |
647 | } |
648 | |
649 | if (vma->pages->nents > expected_pages) { |
650 | pr_err("sg table is wrong sizeo, expected %u, found %u nents\n" , |
651 | expected_pages, vma->pages->nents); |
652 | err = -EINVAL; |
653 | goto out_object; |
654 | } |
655 | |
656 | if (vma->node.size < vma->size) { |
657 | pr_err("VMA binding too small, expected %llu, found %llu\n" , |
658 | vma->size, vma->node.size); |
659 | err = -EINVAL; |
660 | goto out_object; |
661 | } |
662 | |
663 | if (vma->pages == obj->mm.pages) { |
664 | pr_err("VMA using unrotated object pages!\n" ); |
665 | err = -EINVAL; |
666 | goto out_object; |
667 | } |
668 | |
669 | sg = vma->pages->sgl; |
670 | for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) { |
671 | if (view.type == I915_GTT_VIEW_ROTATED) |
672 | sg = assert_rotated(obj, r: &view.rotated, n, sg); |
673 | else |
674 | sg = assert_remapped(obj, r: &view.remapped, n, sg); |
675 | if (IS_ERR(ptr: sg)) { |
676 | pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n" , |
677 | view.type == I915_GTT_VIEW_ROTATED ? |
678 | "rotated" : "remapped" , n, |
679 | plane_info[0].width, |
680 | plane_info[0].height, |
681 | plane_info[0].src_stride, |
682 | plane_info[0].dst_stride, |
683 | plane_info[0].offset, |
684 | plane_info[1].width, |
685 | plane_info[1].height, |
686 | plane_info[1].src_stride, |
687 | plane_info[1].dst_stride, |
688 | plane_info[1].offset); |
689 | err = -EINVAL; |
690 | goto out_object; |
691 | } |
692 | } |
693 | |
694 | i915_vma_unpin(vma); |
695 | err = i915_vma_unbind_unlocked(vma); |
696 | if (err) { |
697 | pr_err("Unbinding returned %i\n" , err); |
698 | goto out_object; |
699 | } |
700 | cond_resched(); |
701 | } |
702 | } |
703 | } |
704 | } |
705 | } |
706 | |
707 | out_object: |
708 | i915_gem_object_put(obj); |
709 | out: |
710 | return err; |
711 | } |
712 | |
713 | static bool assert_partial(struct drm_i915_gem_object *obj, |
714 | struct i915_vma *vma, |
715 | unsigned long offset, |
716 | unsigned long size) |
717 | { |
718 | struct sgt_iter sgt; |
719 | dma_addr_t dma; |
720 | |
721 | for_each_sgt_daddr(dma, sgt, vma->pages) { |
722 | dma_addr_t src; |
723 | |
724 | if (!size) { |
725 | pr_err("Partial scattergather list too long\n" ); |
726 | return false; |
727 | } |
728 | |
729 | src = i915_gem_object_get_dma_address(obj, offset); |
730 | if (src != dma) { |
731 | pr_err("DMA mismatch for partial page offset %lu\n" , |
732 | offset); |
733 | return false; |
734 | } |
735 | |
736 | offset++; |
737 | size--; |
738 | } |
739 | |
740 | return true; |
741 | } |
742 | |
743 | static bool assert_pin(struct i915_vma *vma, |
744 | struct i915_gtt_view *view, |
745 | u64 size, |
746 | const char *name) |
747 | { |
748 | bool ok = true; |
749 | |
750 | if (vma->size != size) { |
751 | pr_err("(%s) VMA is wrong size, expected %llu, found %llu\n" , |
752 | name, size, vma->size); |
753 | ok = false; |
754 | } |
755 | |
756 | if (vma->node.size < vma->size) { |
757 | pr_err("(%s) VMA binding too small, expected %llu, found %llu\n" , |
758 | name, vma->size, vma->node.size); |
759 | ok = false; |
760 | } |
761 | |
762 | if (view && view->type != I915_GTT_VIEW_NORMAL) { |
763 | if (memcmp(p: &vma->gtt_view, q: view, size: sizeof(*view))) { |
764 | pr_err("(%s) VMA mismatch upon creation!\n" , |
765 | name); |
766 | ok = false; |
767 | } |
768 | |
769 | if (vma->pages == vma->obj->mm.pages) { |
770 | pr_err("(%s) VMA using original object pages!\n" , |
771 | name); |
772 | ok = false; |
773 | } |
774 | } else { |
775 | if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) { |
776 | pr_err("Not the normal ggtt view! Found %d\n" , |
777 | vma->gtt_view.type); |
778 | ok = false; |
779 | } |
780 | |
781 | if (vma->pages != vma->obj->mm.pages) { |
782 | pr_err("VMA not using object pages!\n" ); |
783 | ok = false; |
784 | } |
785 | } |
786 | |
787 | return ok; |
788 | } |
789 | |
790 | static int igt_vma_partial(void *arg) |
791 | { |
792 | struct i915_ggtt *ggtt = arg; |
793 | struct i915_address_space *vm = &ggtt->vm; |
794 | const unsigned int npages = 1021; /* prime! */ |
795 | struct drm_i915_gem_object *obj; |
796 | const struct phase { |
797 | const char *name; |
798 | } phases[] = { |
799 | { "create" }, |
800 | { "lookup" }, |
801 | { }, |
802 | }, *p; |
803 | unsigned int sz, offset; |
804 | struct i915_vma *vma; |
805 | int err = -ENOMEM; |
806 | |
807 | /* Create lots of different VMA for the object and check that |
808 | * we are returned the same VMA when we later request the same range. |
809 | */ |
810 | |
811 | obj = i915_gem_object_create_internal(i915: vm->i915, size: npages * PAGE_SIZE); |
812 | if (IS_ERR(ptr: obj)) |
813 | goto out; |
814 | |
815 | for (p = phases; p->name; p++) { /* exercise both create/lookup */ |
816 | unsigned int count, nvma; |
817 | |
818 | nvma = 0; |
819 | for_each_prime_number_from(sz, 1, npages) { |
820 | for_each_prime_number_from(offset, 0, npages - sz) { |
821 | struct i915_gtt_view view; |
822 | |
823 | view.type = I915_GTT_VIEW_PARTIAL; |
824 | view.partial.offset = offset; |
825 | view.partial.size = sz; |
826 | |
827 | if (sz == npages) |
828 | view.type = I915_GTT_VIEW_NORMAL; |
829 | |
830 | vma = checked_vma_instance(obj, vm, view: &view); |
831 | if (IS_ERR(ptr: vma)) { |
832 | err = PTR_ERR(ptr: vma); |
833 | goto out_object; |
834 | } |
835 | |
836 | err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL); |
837 | if (err) |
838 | goto out_object; |
839 | |
840 | if (!assert_pin(vma, view: &view, size: sz*PAGE_SIZE, name: p->name)) { |
841 | pr_err("(%s) Inconsistent partial pinning for (offset=%d, size=%d)\n" , |
842 | p->name, offset, sz); |
843 | err = -EINVAL; |
844 | goto out_object; |
845 | } |
846 | |
847 | if (!assert_partial(obj, vma, offset, size: sz)) { |
848 | pr_err("(%s) Inconsistent partial pages for (offset=%d, size=%d)\n" , |
849 | p->name, offset, sz); |
850 | err = -EINVAL; |
851 | goto out_object; |
852 | } |
853 | |
854 | i915_vma_unpin(vma); |
855 | nvma++; |
856 | err = i915_vma_unbind_unlocked(vma); |
857 | if (err) { |
858 | pr_err("Unbinding returned %i\n" , err); |
859 | goto out_object; |
860 | } |
861 | |
862 | cond_resched(); |
863 | } |
864 | } |
865 | |
866 | count = 0; |
867 | list_for_each_entry(vma, &obj->vma.list, obj_link) |
868 | count++; |
869 | if (count != nvma) { |
870 | pr_err("(%s) All partial vma were not recorded on the obj->vma_list: found %u, expected %u\n" , |
871 | p->name, count, nvma); |
872 | err = -EINVAL; |
873 | goto out_object; |
874 | } |
875 | |
876 | /* Check that we did create the whole object mapping */ |
877 | vma = checked_vma_instance(obj, vm, NULL); |
878 | if (IS_ERR(ptr: vma)) { |
879 | err = PTR_ERR(ptr: vma); |
880 | goto out_object; |
881 | } |
882 | |
883 | err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL); |
884 | if (err) |
885 | goto out_object; |
886 | |
887 | if (!assert_pin(vma, NULL, size: obj->base.size, name: p->name)) { |
888 | pr_err("(%s) inconsistent full pin\n" , p->name); |
889 | err = -EINVAL; |
890 | goto out_object; |
891 | } |
892 | |
893 | i915_vma_unpin(vma); |
894 | |
895 | err = i915_vma_unbind_unlocked(vma); |
896 | if (err) { |
897 | pr_err("Unbinding returned %i\n" , err); |
898 | goto out_object; |
899 | } |
900 | |
901 | count = 0; |
902 | list_for_each_entry(vma, &obj->vma.list, obj_link) |
903 | count++; |
904 | if (count != nvma) { |
905 | pr_err("(%s) allocated an extra full vma!\n" , p->name); |
906 | err = -EINVAL; |
907 | goto out_object; |
908 | } |
909 | } |
910 | |
911 | out_object: |
912 | i915_gem_object_put(obj); |
913 | out: |
914 | return err; |
915 | } |
916 | |
917 | int i915_vma_mock_selftests(void) |
918 | { |
919 | static const struct i915_subtest tests[] = { |
920 | SUBTEST(igt_vma_create), |
921 | SUBTEST(igt_vma_pin1), |
922 | SUBTEST(igt_vma_rotate_remap), |
923 | SUBTEST(igt_vma_partial), |
924 | }; |
925 | struct drm_i915_private *i915; |
926 | struct intel_gt *gt; |
927 | int err; |
928 | |
929 | i915 = mock_gem_device(); |
930 | if (!i915) |
931 | return -ENOMEM; |
932 | |
933 | /* allocate the ggtt */ |
934 | err = intel_gt_assign_ggtt(gt: to_gt(i915)); |
935 | if (err) |
936 | goto out_put; |
937 | |
938 | gt = to_gt(i915); |
939 | |
940 | mock_init_ggtt(gt); |
941 | |
942 | err = i915_subtests(tests, gt->ggtt); |
943 | |
944 | mock_device_flush(i915); |
945 | i915_gem_drain_freed_objects(i915); |
946 | mock_fini_ggtt(ggtt: gt->ggtt); |
947 | |
948 | out_put: |
949 | mock_destroy_device(i915); |
950 | return err; |
951 | } |
952 | |
953 | static int igt_vma_remapped_gtt(void *arg) |
954 | { |
955 | struct drm_i915_private *i915 = arg; |
956 | const struct intel_remapped_plane_info planes[] = { |
957 | { .width = 1, .height = 1, .src_stride = 1 }, |
958 | { .width = 2, .height = 2, .src_stride = 2 }, |
959 | { .width = 4, .height = 4, .src_stride = 4 }, |
960 | { .width = 8, .height = 8, .src_stride = 8 }, |
961 | |
962 | { .width = 3, .height = 5, .src_stride = 3 }, |
963 | { .width = 3, .height = 5, .src_stride = 4 }, |
964 | { .width = 3, .height = 5, .src_stride = 5 }, |
965 | |
966 | { .width = 5, .height = 3, .src_stride = 5 }, |
967 | { .width = 5, .height = 3, .src_stride = 7 }, |
968 | { .width = 5, .height = 3, .src_stride = 9 }, |
969 | |
970 | { .width = 4, .height = 6, .src_stride = 6 }, |
971 | { .width = 6, .height = 4, .src_stride = 6 }, |
972 | |
973 | { .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 }, |
974 | { .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 }, |
975 | { .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 }, |
976 | |
977 | { } |
978 | }, *p; |
979 | enum i915_gtt_view_type types[] = { |
980 | I915_GTT_VIEW_ROTATED, |
981 | I915_GTT_VIEW_REMAPPED, |
982 | 0, |
983 | }, *t; |
984 | struct drm_i915_gem_object *obj; |
985 | intel_wakeref_t wakeref; |
986 | int err = 0; |
987 | |
988 | if (!i915_ggtt_has_aperture(ggtt: to_gt(i915)->ggtt)) |
989 | return 0; |
990 | |
991 | obj = i915_gem_object_create_internal(i915, size: 10 * 10 * PAGE_SIZE); |
992 | if (IS_ERR(ptr: obj)) |
993 | return PTR_ERR(ptr: obj); |
994 | |
995 | wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm); |
996 | |
997 | for (t = types; *t; t++) { |
998 | for (p = planes; p->width; p++) { |
999 | struct i915_gtt_view view = { |
1000 | .type = *t, |
1001 | .rotated.plane[0] = *p, |
1002 | }; |
1003 | struct intel_remapped_plane_info *plane_info = view.rotated.plane; |
1004 | struct i915_vma *vma; |
1005 | u32 __iomem *map; |
1006 | unsigned int x, y; |
1007 | |
1008 | i915_gem_object_lock(obj, NULL); |
1009 | err = i915_gem_object_set_to_gtt_domain(obj, write: true); |
1010 | i915_gem_object_unlock(obj); |
1011 | if (err) |
1012 | goto out; |
1013 | |
1014 | if (!plane_info[0].dst_stride) |
1015 | plane_info[0].dst_stride = *t == I915_GTT_VIEW_ROTATED ? |
1016 | p->height : p->width; |
1017 | |
1018 | vma = i915_gem_object_ggtt_pin(obj, view: &view, size: 0, alignment: 0, PIN_MAPPABLE); |
1019 | if (IS_ERR(ptr: vma)) { |
1020 | err = PTR_ERR(ptr: vma); |
1021 | goto out; |
1022 | } |
1023 | |
1024 | GEM_BUG_ON(vma->gtt_view.type != *t); |
1025 | |
1026 | map = i915_vma_pin_iomap(vma); |
1027 | i915_vma_unpin(vma); |
1028 | if (IS_ERR(ptr: map)) { |
1029 | err = PTR_ERR(ptr: map); |
1030 | goto out; |
1031 | } |
1032 | |
1033 | for (y = 0 ; y < plane_info[0].height; y++) { |
1034 | for (x = 0 ; x < plane_info[0].width; x++) { |
1035 | unsigned int offset; |
1036 | u32 val = y << 16 | x; |
1037 | |
1038 | if (*t == I915_GTT_VIEW_ROTATED) |
1039 | offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE; |
1040 | else |
1041 | offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE; |
1042 | |
1043 | iowrite32(val, &map[offset / sizeof(*map)]); |
1044 | } |
1045 | } |
1046 | |
1047 | i915_vma_unpin_iomap(vma); |
1048 | |
1049 | vma = i915_gem_object_ggtt_pin(obj, NULL, size: 0, alignment: 0, PIN_MAPPABLE); |
1050 | if (IS_ERR(ptr: vma)) { |
1051 | err = PTR_ERR(ptr: vma); |
1052 | goto out; |
1053 | } |
1054 | |
1055 | GEM_BUG_ON(vma->gtt_view.type != I915_GTT_VIEW_NORMAL); |
1056 | |
1057 | map = i915_vma_pin_iomap(vma); |
1058 | i915_vma_unpin(vma); |
1059 | if (IS_ERR(ptr: map)) { |
1060 | err = PTR_ERR(ptr: map); |
1061 | goto out; |
1062 | } |
1063 | |
1064 | for (y = 0 ; y < plane_info[0].height; y++) { |
1065 | for (x = 0 ; x < plane_info[0].width; x++) { |
1066 | unsigned int offset, src_idx; |
1067 | u32 exp = y << 16 | x; |
1068 | u32 val; |
1069 | |
1070 | if (*t == I915_GTT_VIEW_ROTATED) |
1071 | src_idx = rotated_index(r: &view.rotated, n: 0, x, y); |
1072 | else |
1073 | src_idx = remapped_index(r: &view.remapped, n: 0, x, y); |
1074 | offset = src_idx * PAGE_SIZE; |
1075 | |
1076 | val = ioread32(&map[offset / sizeof(*map)]); |
1077 | if (val != exp) { |
1078 | pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n" , |
1079 | *t == I915_GTT_VIEW_ROTATED ? "Rotated" : "Remapped" , |
1080 | exp, val); |
1081 | i915_vma_unpin_iomap(vma); |
1082 | err = -EINVAL; |
1083 | goto out; |
1084 | } |
1085 | } |
1086 | } |
1087 | i915_vma_unpin_iomap(vma); |
1088 | |
1089 | cond_resched(); |
1090 | } |
1091 | } |
1092 | |
1093 | out: |
1094 | intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref); |
1095 | i915_gem_object_put(obj); |
1096 | |
1097 | return err; |
1098 | } |
1099 | |
1100 | int i915_vma_live_selftests(struct drm_i915_private *i915) |
1101 | { |
1102 | static const struct i915_subtest tests[] = { |
1103 | SUBTEST(igt_vma_remapped_gtt), |
1104 | }; |
1105 | |
1106 | return i915_live_subtests(tests, i915); |
1107 | } |
1108 | |