1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * Copyright © 2018 Intel Corporation |
5 | */ |
6 | #include "gt/intel_gpu_commands.h" |
7 | #include "gt/intel_gt.h" |
8 | |
9 | #include "gem/i915_gem_internal.h" |
10 | #include "gem/selftests/igt_gem_utils.h" |
11 | |
12 | #include "igt_spinner.h" |
13 | |
14 | int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt) |
15 | { |
16 | int err; |
17 | |
18 | memset(spin, 0, sizeof(*spin)); |
19 | spin->gt = gt; |
20 | |
21 | spin->hws = i915_gem_object_create_internal(i915: gt->i915, PAGE_SIZE); |
22 | if (IS_ERR(ptr: spin->hws)) { |
23 | err = PTR_ERR(ptr: spin->hws); |
24 | goto err; |
25 | } |
26 | i915_gem_object_set_cache_coherency(obj: spin->hws, cache_level: I915_CACHE_LLC); |
27 | |
28 | spin->obj = i915_gem_object_create_internal(i915: gt->i915, PAGE_SIZE); |
29 | if (IS_ERR(ptr: spin->obj)) { |
30 | err = PTR_ERR(ptr: spin->obj); |
31 | goto err_hws; |
32 | } |
33 | |
34 | return 0; |
35 | |
36 | err_hws: |
37 | i915_gem_object_put(obj: spin->hws); |
38 | err: |
39 | return err; |
40 | } |
41 | |
42 | static void *igt_spinner_pin_obj(struct intel_context *ce, |
43 | struct i915_gem_ww_ctx *ww, |
44 | struct drm_i915_gem_object *obj, |
45 | unsigned int mode, struct i915_vma **vma) |
46 | { |
47 | void *vaddr; |
48 | int ret; |
49 | |
50 | *vma = i915_vma_instance(obj, vm: ce->vm, NULL); |
51 | if (IS_ERR(ptr: *vma)) |
52 | return ERR_CAST(ptr: *vma); |
53 | |
54 | ret = i915_gem_object_lock(obj, ww); |
55 | if (ret) |
56 | return ERR_PTR(error: ret); |
57 | |
58 | vaddr = i915_gem_object_pin_map(obj, type: mode); |
59 | |
60 | if (!ww) |
61 | i915_gem_object_unlock(obj); |
62 | |
63 | if (IS_ERR(ptr: vaddr)) |
64 | return vaddr; |
65 | |
66 | if (ww) |
67 | ret = i915_vma_pin_ww(vma: *vma, ww, size: 0, alignment: 0, PIN_USER); |
68 | else |
69 | ret = i915_vma_pin(vma: *vma, size: 0, alignment: 0, PIN_USER); |
70 | |
71 | if (ret) { |
72 | i915_gem_object_unpin_map(obj); |
73 | return ERR_PTR(error: ret); |
74 | } |
75 | |
76 | return vaddr; |
77 | } |
78 | |
79 | int igt_spinner_pin(struct igt_spinner *spin, |
80 | struct intel_context *ce, |
81 | struct i915_gem_ww_ctx *ww) |
82 | { |
83 | void *vaddr; |
84 | |
85 | if (spin->ce && WARN_ON(spin->ce != ce)) |
86 | return -ENODEV; |
87 | spin->ce = ce; |
88 | |
89 | if (!spin->seqno) { |
90 | vaddr = igt_spinner_pin_obj(ce, ww, obj: spin->hws, mode: I915_MAP_WB, vma: &spin->hws_vma); |
91 | if (IS_ERR(ptr: vaddr)) |
92 | return PTR_ERR(ptr: vaddr); |
93 | |
94 | spin->seqno = memset(vaddr, 0xff, PAGE_SIZE); |
95 | } |
96 | |
97 | if (!spin->batch) { |
98 | unsigned int mode; |
99 | |
100 | mode = intel_gt_coherent_map_type(gt: spin->gt, obj: spin->obj, always_coherent: false); |
101 | vaddr = igt_spinner_pin_obj(ce, ww, obj: spin->obj, mode, vma: &spin->batch_vma); |
102 | if (IS_ERR(ptr: vaddr)) |
103 | return PTR_ERR(ptr: vaddr); |
104 | |
105 | spin->batch = vaddr; |
106 | } |
107 | |
108 | return 0; |
109 | } |
110 | |
111 | static unsigned int seqno_offset(u64 fence) |
112 | { |
113 | return offset_in_page(sizeof(u32) * fence); |
114 | } |
115 | |
116 | static u64 hws_address(const struct i915_vma *hws, |
117 | const struct i915_request *rq) |
118 | { |
119 | return i915_vma_offset(vma: hws) + seqno_offset(fence: rq->fence.context); |
120 | } |
121 | |
122 | struct i915_request * |
123 | igt_spinner_create_request(struct igt_spinner *spin, |
124 | struct intel_context *ce, |
125 | u32 arbitration_command) |
126 | { |
127 | struct intel_engine_cs *engine = ce->engine; |
128 | struct i915_request *rq = NULL; |
129 | struct i915_vma *hws, *vma; |
130 | unsigned int flags; |
131 | u32 *batch; |
132 | int err; |
133 | |
134 | GEM_BUG_ON(spin->gt != ce->vm->gt); |
135 | |
136 | if (!intel_engine_can_store_dword(engine: ce->engine)) |
137 | return ERR_PTR(error: -ENODEV); |
138 | |
139 | if (!spin->batch) { |
140 | err = igt_spinner_pin(spin, ce, NULL); |
141 | if (err) |
142 | return ERR_PTR(error: err); |
143 | } |
144 | |
145 | hws = spin->hws_vma; |
146 | vma = spin->batch_vma; |
147 | |
148 | rq = intel_context_create_request(ce); |
149 | if (IS_ERR(ptr: rq)) |
150 | return ERR_CAST(ptr: rq); |
151 | |
152 | err = igt_vma_move_to_active_unlocked(vma, rq, flags: 0); |
153 | if (err) |
154 | goto cancel_rq; |
155 | |
156 | err = igt_vma_move_to_active_unlocked(vma: hws, rq, flags: 0); |
157 | if (err) |
158 | goto cancel_rq; |
159 | |
160 | batch = spin->batch; |
161 | |
162 | if (GRAPHICS_VER(rq->i915) >= 8) { |
163 | *batch++ = MI_STORE_DWORD_IMM_GEN4; |
164 | *batch++ = lower_32_bits(hws_address(hws, rq)); |
165 | *batch++ = upper_32_bits(hws_address(hws, rq)); |
166 | } else if (GRAPHICS_VER(rq->i915) >= 6) { |
167 | *batch++ = MI_STORE_DWORD_IMM_GEN4; |
168 | *batch++ = 0; |
169 | *batch++ = hws_address(hws, rq); |
170 | } else if (GRAPHICS_VER(rq->i915) >= 4) { |
171 | *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; |
172 | *batch++ = 0; |
173 | *batch++ = hws_address(hws, rq); |
174 | } else { |
175 | *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; |
176 | *batch++ = hws_address(hws, rq); |
177 | } |
178 | *batch++ = rq->fence.seqno; |
179 | |
180 | *batch++ = arbitration_command; |
181 | |
182 | memset32(s: batch, MI_NOOP, n: 128); |
183 | batch += 128; |
184 | |
185 | if (GRAPHICS_VER(rq->i915) >= 8) |
186 | *batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1; |
187 | else if (IS_HASWELL(rq->i915)) |
188 | *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW; |
189 | else if (GRAPHICS_VER(rq->i915) >= 6) |
190 | *batch++ = MI_BATCH_BUFFER_START; |
191 | else |
192 | *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; |
193 | *batch++ = lower_32_bits(i915_vma_offset(vma)); |
194 | *batch++ = upper_32_bits(i915_vma_offset(vma)); |
195 | |
196 | *batch++ = MI_BATCH_BUFFER_END; /* not reached */ |
197 | |
198 | intel_gt_chipset_flush(gt: engine->gt); |
199 | |
200 | if (engine->emit_init_breadcrumb) { |
201 | err = engine->emit_init_breadcrumb(rq); |
202 | if (err) |
203 | goto cancel_rq; |
204 | } |
205 | |
206 | flags = 0; |
207 | if (GRAPHICS_VER(rq->i915) <= 5) |
208 | flags |= I915_DISPATCH_SECURE; |
209 | err = engine->emit_bb_start(rq, i915_vma_offset(vma), PAGE_SIZE, flags); |
210 | |
211 | cancel_rq: |
212 | if (err) { |
213 | i915_request_set_error_once(rq, error: err); |
214 | i915_request_add(rq); |
215 | } |
216 | return err ? ERR_PTR(error: err) : rq; |
217 | } |
218 | |
219 | static u32 |
220 | hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq) |
221 | { |
222 | u32 *seqno = spin->seqno + seqno_offset(fence: rq->fence.context); |
223 | |
224 | return READ_ONCE(*seqno); |
225 | } |
226 | |
227 | void igt_spinner_end(struct igt_spinner *spin) |
228 | { |
229 | if (!spin->batch) |
230 | return; |
231 | |
232 | *spin->batch = MI_BATCH_BUFFER_END; |
233 | intel_gt_chipset_flush(gt: spin->gt); |
234 | } |
235 | |
236 | void igt_spinner_fini(struct igt_spinner *spin) |
237 | { |
238 | igt_spinner_end(spin); |
239 | |
240 | if (spin->batch) { |
241 | i915_vma_unpin(vma: spin->batch_vma); |
242 | i915_gem_object_unpin_map(obj: spin->obj); |
243 | } |
244 | i915_gem_object_put(obj: spin->obj); |
245 | |
246 | if (spin->seqno) { |
247 | i915_vma_unpin(vma: spin->hws_vma); |
248 | i915_gem_object_unpin_map(obj: spin->hws); |
249 | } |
250 | i915_gem_object_put(obj: spin->hws); |
251 | } |
252 | |
253 | bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq) |
254 | { |
255 | if (i915_request_is_ready(rq)) |
256 | intel_engine_flush_submission(engine: rq->engine); |
257 | |
258 | return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq), |
259 | rq->fence.seqno), |
260 | 100) && |
261 | wait_for(i915_seqno_passed(hws_seqno(spin, rq), |
262 | rq->fence.seqno), |
263 | 50)); |
264 | } |
265 | |