1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2021 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/dma-fence.h> |
7 | #include <linux/slab.h> |
8 | |
9 | #include <drm/ttm/ttm_bo.h> |
10 | |
11 | #include "i915_deps.h" |
12 | |
13 | /** |
14 | * DOC: Set of utilities to dynamically collect dependencies into a |
15 | * structure which is fed into the GT migration code. |
16 | * |
17 | * Once we can do async unbinding, this is also needed to coalesce |
18 | * the migration fence with the unbind fences if these are coalesced |
19 | * post-migration. |
20 | * |
21 | * While collecting the individual dependencies, we store the refcounted |
22 | * struct dma_fence pointers in a realloc-managed pointer array, since |
23 | * that can be easily fed into a dma_fence_array. Other options are |
24 | * available, like for example an xarray for similarity with drm/sched. |
25 | * Can be changed easily if needed. |
26 | * |
27 | * A struct i915_deps need to be initialized using i915_deps_init(). |
28 | * If i915_deps_add_dependency() or i915_deps_add_resv() return an |
29 | * error code they will internally call i915_deps_fini(), which frees |
30 | * all internal references and allocations. |
31 | */ |
32 | |
33 | /* Min number of fence pointers in the array when an allocation occurs. */ |
34 | #define I915_DEPS_MIN_ALLOC_CHUNK 8U |
35 | |
36 | static void i915_deps_reset_fences(struct i915_deps *deps) |
37 | { |
38 | if (deps->fences != &deps->single) |
39 | kfree(objp: deps->fences); |
40 | deps->num_deps = 0; |
41 | deps->fences_size = 1; |
42 | deps->fences = &deps->single; |
43 | } |
44 | |
45 | /** |
46 | * i915_deps_init - Initialize an i915_deps structure |
47 | * @deps: Pointer to the i915_deps structure to initialize. |
48 | * @gfp: The allocation mode for subsequenst allocations. |
49 | */ |
50 | void i915_deps_init(struct i915_deps *deps, gfp_t gfp) |
51 | { |
52 | deps->fences = NULL; |
53 | deps->gfp = gfp; |
54 | i915_deps_reset_fences(deps); |
55 | } |
56 | |
57 | /** |
58 | * i915_deps_fini - Finalize an i915_deps structure |
59 | * @deps: Pointer to the i915_deps structure to finalize. |
60 | * |
61 | * This function drops all fence references taken, conditionally frees and |
62 | * then resets the fences array. |
63 | */ |
64 | void i915_deps_fini(struct i915_deps *deps) |
65 | { |
66 | unsigned int i; |
67 | |
68 | for (i = 0; i < deps->num_deps; ++i) |
69 | dma_fence_put(fence: deps->fences[i]); |
70 | |
71 | if (deps->fences != &deps->single) |
72 | kfree(objp: deps->fences); |
73 | } |
74 | |
75 | static int i915_deps_grow(struct i915_deps *deps, struct dma_fence *fence, |
76 | const struct ttm_operation_ctx *ctx) |
77 | { |
78 | int ret; |
79 | |
80 | if (deps->num_deps >= deps->fences_size) { |
81 | unsigned int new_size = 2 * deps->fences_size; |
82 | struct dma_fence **new_fences; |
83 | |
84 | new_size = max(new_size, I915_DEPS_MIN_ALLOC_CHUNK); |
85 | new_fences = kmalloc_array(n: new_size, size: sizeof(*new_fences), flags: deps->gfp); |
86 | if (!new_fences) |
87 | goto sync; |
88 | |
89 | memcpy(new_fences, deps->fences, |
90 | deps->fences_size * sizeof(*new_fences)); |
91 | swap(new_fences, deps->fences); |
92 | if (new_fences != &deps->single) |
93 | kfree(objp: new_fences); |
94 | deps->fences_size = new_size; |
95 | } |
96 | deps->fences[deps->num_deps++] = dma_fence_get(fence); |
97 | return 0; |
98 | |
99 | sync: |
100 | if (ctx->no_wait_gpu && !dma_fence_is_signaled(fence)) { |
101 | ret = -EBUSY; |
102 | goto unref; |
103 | } |
104 | |
105 | ret = dma_fence_wait(fence, intr: ctx->interruptible); |
106 | if (ret) |
107 | goto unref; |
108 | |
109 | ret = fence->error; |
110 | if (ret) |
111 | goto unref; |
112 | |
113 | return 0; |
114 | |
115 | unref: |
116 | i915_deps_fini(deps); |
117 | return ret; |
118 | } |
119 | |
120 | /** |
121 | * i915_deps_sync - Wait for all the fences in the dependency collection |
122 | * @deps: Pointer to the i915_deps structure the fences of which to wait for. |
123 | * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits |
124 | * should be performed. |
125 | * |
126 | * This function waits for fences in the dependency collection. If it |
127 | * encounters an error during the wait or a fence error, the wait for |
128 | * further fences is aborted and the error returned. |
129 | * |
130 | * Return: Zero if successful, Negative error code on error. |
131 | */ |
132 | int i915_deps_sync(const struct i915_deps *deps, const struct ttm_operation_ctx *ctx) |
133 | { |
134 | struct dma_fence **fences = deps->fences; |
135 | unsigned int i; |
136 | int ret = 0; |
137 | |
138 | for (i = 0; i < deps->num_deps; ++i, ++fences) { |
139 | if (ctx->no_wait_gpu && !dma_fence_is_signaled(fence: *fences)) { |
140 | ret = -EBUSY; |
141 | break; |
142 | } |
143 | |
144 | ret = dma_fence_wait(fence: *fences, intr: ctx->interruptible); |
145 | if (!ret) |
146 | ret = (*fences)->error; |
147 | if (ret) |
148 | break; |
149 | } |
150 | |
151 | return ret; |
152 | } |
153 | |
154 | /** |
155 | * i915_deps_add_dependency - Add a fence to the dependency collection |
156 | * @deps: Pointer to the i915_deps structure a fence is to be added to. |
157 | * @fence: The fence to add. |
158 | * @ctx: Pointer to a struct ttm_operation_ctx indicating how waits are to |
159 | * be performed if waiting. |
160 | * |
161 | * Adds a fence to the dependency collection, and takes a reference on it. |
162 | * If the fence context is not zero and there was a later fence from the |
163 | * same fence context already added, then the fence is not added to the |
164 | * dependency collection. If the fence context is not zero and there was |
165 | * an earlier fence already added, then the fence will replace the older |
166 | * fence from the same context and the reference on the earlier fence will |
167 | * be dropped. |
168 | * If there is a failure to allocate memory to accommodate the new fence to |
169 | * be added, the new fence will instead be waited for and an error may |
170 | * be returned; depending on the value of @ctx, or if there was a fence |
171 | * error. If an error was returned, the dependency collection will be |
172 | * finalized and all fence reference dropped. |
173 | * |
174 | * Return: 0 if success. Negative error code on error. |
175 | */ |
176 | int i915_deps_add_dependency(struct i915_deps *deps, |
177 | struct dma_fence *fence, |
178 | const struct ttm_operation_ctx *ctx) |
179 | { |
180 | unsigned int i; |
181 | int ret; |
182 | |
183 | if (!fence) |
184 | return 0; |
185 | |
186 | if (dma_fence_is_signaled(fence)) { |
187 | ret = fence->error; |
188 | if (ret) |
189 | i915_deps_fini(deps); |
190 | return ret; |
191 | } |
192 | |
193 | for (i = 0; i < deps->num_deps; ++i) { |
194 | struct dma_fence *entry = deps->fences[i]; |
195 | |
196 | if (!entry->context || entry->context != fence->context) |
197 | continue; |
198 | |
199 | if (dma_fence_is_later(f1: fence, f2: entry)) { |
200 | dma_fence_put(fence: entry); |
201 | deps->fences[i] = dma_fence_get(fence); |
202 | } |
203 | |
204 | return 0; |
205 | } |
206 | |
207 | return i915_deps_grow(deps, fence, ctx); |
208 | } |
209 | |
210 | /** |
211 | * i915_deps_add_resv - Add the fences of a reservation object to a dependency |
212 | * collection. |
213 | * @deps: Pointer to the i915_deps structure a fence is to be added to. |
214 | * @resv: The reservation object, then fences of which to add. |
215 | * @ctx: Pointer to a struct ttm_operation_ctx indicating how waits are to |
216 | * be performed if waiting. |
217 | * |
218 | * Calls i915_deps_add_depencency() on the indicated fences of @resv. |
219 | * |
220 | * Return: Zero on success. Negative error code on error. |
221 | */ |
222 | int i915_deps_add_resv(struct i915_deps *deps, struct dma_resv *resv, |
223 | const struct ttm_operation_ctx *ctx) |
224 | { |
225 | struct dma_resv_iter iter; |
226 | struct dma_fence *fence; |
227 | |
228 | dma_resv_assert_held(resv); |
229 | dma_resv_for_each_fence(&iter, resv, dma_resv_usage_rw(true), fence) { |
230 | int ret = i915_deps_add_dependency(deps, fence, ctx); |
231 | |
232 | if (ret) |
233 | return ret; |
234 | } |
235 | |
236 | return 0; |
237 | } |
238 | |