1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6#include "gt/intel_gpu_commands.h"
7#include "gt/intel_gt.h"
8
9#include "gem/i915_gem_internal.h"
10#include "gem/selftests/igt_gem_utils.h"
11
12#include "igt_spinner.h"
13
14int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt)
15{
16 int err;
17
18 memset(spin, 0, sizeof(*spin));
19 spin->gt = gt;
20
21 spin->hws = i915_gem_object_create_internal(i915: gt->i915, PAGE_SIZE);
22 if (IS_ERR(ptr: spin->hws)) {
23 err = PTR_ERR(ptr: spin->hws);
24 goto err;
25 }
26 i915_gem_object_set_cache_coherency(obj: spin->hws, cache_level: I915_CACHE_LLC);
27
28 spin->obj = i915_gem_object_create_internal(i915: gt->i915, PAGE_SIZE);
29 if (IS_ERR(ptr: spin->obj)) {
30 err = PTR_ERR(ptr: spin->obj);
31 goto err_hws;
32 }
33
34 return 0;
35
36err_hws:
37 i915_gem_object_put(obj: spin->hws);
38err:
39 return err;
40}
41
42static void *igt_spinner_pin_obj(struct intel_context *ce,
43 struct i915_gem_ww_ctx *ww,
44 struct drm_i915_gem_object *obj,
45 unsigned int mode, struct i915_vma **vma)
46{
47 void *vaddr;
48 int ret;
49
50 *vma = i915_vma_instance(obj, vm: ce->vm, NULL);
51 if (IS_ERR(ptr: *vma))
52 return ERR_CAST(ptr: *vma);
53
54 ret = i915_gem_object_lock(obj, ww);
55 if (ret)
56 return ERR_PTR(error: ret);
57
58 vaddr = i915_gem_object_pin_map(obj, type: mode);
59
60 if (!ww)
61 i915_gem_object_unlock(obj);
62
63 if (IS_ERR(ptr: vaddr))
64 return vaddr;
65
66 if (ww)
67 ret = i915_vma_pin_ww(vma: *vma, ww, size: 0, alignment: 0, PIN_USER);
68 else
69 ret = i915_vma_pin(vma: *vma, size: 0, alignment: 0, PIN_USER);
70
71 if (ret) {
72 i915_gem_object_unpin_map(obj);
73 return ERR_PTR(error: ret);
74 }
75
76 return vaddr;
77}
78
79int igt_spinner_pin(struct igt_spinner *spin,
80 struct intel_context *ce,
81 struct i915_gem_ww_ctx *ww)
82{
83 void *vaddr;
84
85 if (spin->ce && WARN_ON(spin->ce != ce))
86 return -ENODEV;
87 spin->ce = ce;
88
89 if (!spin->seqno) {
90 vaddr = igt_spinner_pin_obj(ce, ww, obj: spin->hws, mode: I915_MAP_WB, vma: &spin->hws_vma);
91 if (IS_ERR(ptr: vaddr))
92 return PTR_ERR(ptr: vaddr);
93
94 spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
95 }
96
97 if (!spin->batch) {
98 unsigned int mode;
99
100 mode = intel_gt_coherent_map_type(gt: spin->gt, obj: spin->obj, always_coherent: false);
101 vaddr = igt_spinner_pin_obj(ce, ww, obj: spin->obj, mode, vma: &spin->batch_vma);
102 if (IS_ERR(ptr: vaddr))
103 return PTR_ERR(ptr: vaddr);
104
105 spin->batch = vaddr;
106 }
107
108 return 0;
109}
110
111static unsigned int seqno_offset(u64 fence)
112{
113 return offset_in_page(sizeof(u32) * fence);
114}
115
116static u64 hws_address(const struct i915_vma *hws,
117 const struct i915_request *rq)
118{
119 return i915_vma_offset(vma: hws) + seqno_offset(fence: rq->fence.context);
120}
121
122struct i915_request *
123igt_spinner_create_request(struct igt_spinner *spin,
124 struct intel_context *ce,
125 u32 arbitration_command)
126{
127 struct intel_engine_cs *engine = ce->engine;
128 struct i915_request *rq = NULL;
129 struct i915_vma *hws, *vma;
130 unsigned int flags;
131 u32 *batch;
132 int err;
133
134 GEM_BUG_ON(spin->gt != ce->vm->gt);
135
136 if (!intel_engine_can_store_dword(engine: ce->engine))
137 return ERR_PTR(error: -ENODEV);
138
139 if (!spin->batch) {
140 err = igt_spinner_pin(spin, ce, NULL);
141 if (err)
142 return ERR_PTR(error: err);
143 }
144
145 hws = spin->hws_vma;
146 vma = spin->batch_vma;
147
148 rq = intel_context_create_request(ce);
149 if (IS_ERR(ptr: rq))
150 return ERR_CAST(ptr: rq);
151
152 err = igt_vma_move_to_active_unlocked(vma, rq, flags: 0);
153 if (err)
154 goto cancel_rq;
155
156 err = igt_vma_move_to_active_unlocked(vma: hws, rq, flags: 0);
157 if (err)
158 goto cancel_rq;
159
160 batch = spin->batch;
161
162 if (GRAPHICS_VER(rq->i915) >= 8) {
163 *batch++ = MI_STORE_DWORD_IMM_GEN4;
164 *batch++ = lower_32_bits(hws_address(hws, rq));
165 *batch++ = upper_32_bits(hws_address(hws, rq));
166 } else if (GRAPHICS_VER(rq->i915) >= 6) {
167 *batch++ = MI_STORE_DWORD_IMM_GEN4;
168 *batch++ = 0;
169 *batch++ = hws_address(hws, rq);
170 } else if (GRAPHICS_VER(rq->i915) >= 4) {
171 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
172 *batch++ = 0;
173 *batch++ = hws_address(hws, rq);
174 } else {
175 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
176 *batch++ = hws_address(hws, rq);
177 }
178 *batch++ = rq->fence.seqno;
179
180 *batch++ = arbitration_command;
181
182 memset32(s: batch, MI_NOOP, n: 128);
183 batch += 128;
184
185 if (GRAPHICS_VER(rq->i915) >= 8)
186 *batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
187 else if (IS_HASWELL(rq->i915))
188 *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW;
189 else if (GRAPHICS_VER(rq->i915) >= 6)
190 *batch++ = MI_BATCH_BUFFER_START;
191 else
192 *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
193 *batch++ = lower_32_bits(i915_vma_offset(vma));
194 *batch++ = upper_32_bits(i915_vma_offset(vma));
195
196 *batch++ = MI_BATCH_BUFFER_END; /* not reached */
197
198 intel_gt_chipset_flush(gt: engine->gt);
199
200 if (engine->emit_init_breadcrumb) {
201 err = engine->emit_init_breadcrumb(rq);
202 if (err)
203 goto cancel_rq;
204 }
205
206 flags = 0;
207 if (GRAPHICS_VER(rq->i915) <= 5)
208 flags |= I915_DISPATCH_SECURE;
209 err = engine->emit_bb_start(rq, i915_vma_offset(vma), PAGE_SIZE, flags);
210
211cancel_rq:
212 if (err) {
213 i915_request_set_error_once(rq, error: err);
214 i915_request_add(rq);
215 }
216 return err ? ERR_PTR(error: err) : rq;
217}
218
219static u32
220hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq)
221{
222 u32 *seqno = spin->seqno + seqno_offset(fence: rq->fence.context);
223
224 return READ_ONCE(*seqno);
225}
226
227void igt_spinner_end(struct igt_spinner *spin)
228{
229 if (!spin->batch)
230 return;
231
232 *spin->batch = MI_BATCH_BUFFER_END;
233 intel_gt_chipset_flush(gt: spin->gt);
234}
235
236void igt_spinner_fini(struct igt_spinner *spin)
237{
238 igt_spinner_end(spin);
239
240 if (spin->batch) {
241 i915_vma_unpin(vma: spin->batch_vma);
242 i915_gem_object_unpin_map(obj: spin->obj);
243 }
244 i915_gem_object_put(obj: spin->obj);
245
246 if (spin->seqno) {
247 i915_vma_unpin(vma: spin->hws_vma);
248 i915_gem_object_unpin_map(obj: spin->hws);
249 }
250 i915_gem_object_put(obj: spin->hws);
251}
252
253bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq)
254{
255 if (i915_request_is_ready(rq))
256 intel_engine_flush_submission(engine: rq->engine);
257
258 return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq),
259 rq->fence.seqno),
260 100) &&
261 wait_for(i915_seqno_passed(hws_seqno(spin, rq),
262 rq->fence.seqno),
263 50));
264}
265

source code of linux/drivers/gpu/drm/i915/selftests/igt_spinner.c