1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright © 2019 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/workqueue.h> |
7 | |
8 | #include "i915_drv.h" /* for_each_engine() */ |
9 | #include "i915_request.h" |
10 | #include "intel_engine_heartbeat.h" |
11 | #include "intel_execlists_submission.h" |
12 | #include "intel_gt.h" |
13 | #include "intel_gt_pm.h" |
14 | #include "intel_gt_requests.h" |
15 | #include "intel_timeline.h" |
16 | |
17 | static bool retire_requests(struct intel_timeline *tl) |
18 | { |
19 | struct i915_request *rq, *rn; |
20 | |
21 | list_for_each_entry_safe(rq, rn, &tl->requests, link) |
22 | if (!i915_request_retire(rq)) |
23 | return false; |
24 | |
25 | /* And check nothing new was submitted */ |
26 | return !i915_active_fence_isset(active: &tl->last_request); |
27 | } |
28 | |
29 | static bool engine_active(const struct intel_engine_cs *engine) |
30 | { |
31 | return !list_empty(head: &engine->kernel_context->timeline->requests); |
32 | } |
33 | |
34 | static bool flush_submission(struct intel_gt *gt, long timeout) |
35 | { |
36 | struct intel_engine_cs *engine; |
37 | enum intel_engine_id id; |
38 | bool active = false; |
39 | |
40 | if (!timeout) |
41 | return false; |
42 | |
43 | if (!intel_gt_pm_is_awake(gt)) |
44 | return false; |
45 | |
46 | for_each_engine(engine, gt, id) { |
47 | intel_engine_flush_submission(engine); |
48 | |
49 | /* Flush the background retirement and idle barriers */ |
50 | flush_work(work: &engine->retire_work); |
51 | flush_delayed_work(dwork: &engine->wakeref.work); |
52 | |
53 | /* Is the idle barrier still outstanding? */ |
54 | active |= engine_active(engine); |
55 | } |
56 | |
57 | return active; |
58 | } |
59 | |
60 | static void engine_retire(struct work_struct *work) |
61 | { |
62 | struct intel_engine_cs *engine = |
63 | container_of(work, typeof(*engine), retire_work); |
64 | struct intel_timeline *tl = xchg(&engine->retire, NULL); |
65 | |
66 | do { |
67 | struct intel_timeline *next = xchg(&tl->retire, NULL); |
68 | |
69 | /* |
70 | * Our goal here is to retire _idle_ timelines as soon as |
71 | * possible (as they are idle, we do not expect userspace |
72 | * to be cleaning up anytime soon). |
73 | * |
74 | * If the timeline is currently locked, either it is being |
75 | * retired elsewhere or about to be! |
76 | */ |
77 | if (mutex_trylock(lock: &tl->mutex)) { |
78 | retire_requests(tl); |
79 | mutex_unlock(lock: &tl->mutex); |
80 | } |
81 | intel_timeline_put(timeline: tl); |
82 | |
83 | GEM_BUG_ON(!next); |
84 | tl = ptr_mask_bits(next, 1); |
85 | } while (tl); |
86 | } |
87 | |
88 | static bool add_retire(struct intel_engine_cs *engine, |
89 | struct intel_timeline *tl) |
90 | { |
91 | #define STUB ((struct intel_timeline *)1) |
92 | struct intel_timeline *first; |
93 | |
94 | /* |
95 | * We open-code a llist here to include the additional tag [BIT(0)] |
96 | * so that we know when the timeline is already on a |
97 | * retirement queue: either this engine or another. |
98 | */ |
99 | |
100 | if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */ |
101 | return false; |
102 | |
103 | intel_timeline_get(timeline: tl); |
104 | first = READ_ONCE(engine->retire); |
105 | do |
106 | tl->retire = ptr_pack_bits(first, 1, 1); |
107 | while (!try_cmpxchg(&engine->retire, &first, tl)); |
108 | |
109 | return !first; |
110 | } |
111 | |
112 | void intel_engine_add_retire(struct intel_engine_cs *engine, |
113 | struct intel_timeline *tl) |
114 | { |
115 | /* We don't deal well with the engine disappearing beneath us */ |
116 | GEM_BUG_ON(intel_engine_is_virtual(engine)); |
117 | |
118 | if (add_retire(engine, tl)) |
119 | queue_work(wq: engine->i915->unordered_wq, work: &engine->retire_work); |
120 | } |
121 | |
122 | void intel_engine_init_retire(struct intel_engine_cs *engine) |
123 | { |
124 | INIT_WORK(&engine->retire_work, engine_retire); |
125 | } |
126 | |
127 | void intel_engine_fini_retire(struct intel_engine_cs *engine) |
128 | { |
129 | flush_work(work: &engine->retire_work); |
130 | GEM_BUG_ON(engine->retire); |
131 | } |
132 | |
133 | long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout, |
134 | long *remaining_timeout) |
135 | { |
136 | struct intel_gt_timelines *timelines = >->timelines; |
137 | struct intel_timeline *tl, *tn; |
138 | unsigned long active_count = 0; |
139 | LIST_HEAD(free); |
140 | |
141 | flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */ |
142 | spin_lock(lock: &timelines->lock); |
143 | list_for_each_entry_safe(tl, tn, &timelines->active_list, link) { |
144 | if (!mutex_trylock(lock: &tl->mutex)) { |
145 | active_count++; /* report busy to caller, try again? */ |
146 | continue; |
147 | } |
148 | |
149 | intel_timeline_get(timeline: tl); |
150 | GEM_BUG_ON(!atomic_read(&tl->active_count)); |
151 | atomic_inc(v: &tl->active_count); /* pin the list element */ |
152 | spin_unlock(lock: &timelines->lock); |
153 | |
154 | if (timeout > 0) { |
155 | struct dma_fence *fence; |
156 | |
157 | fence = i915_active_fence_get(active: &tl->last_request); |
158 | if (fence) { |
159 | mutex_unlock(lock: &tl->mutex); |
160 | |
161 | timeout = dma_fence_wait_timeout(fence, |
162 | intr: true, |
163 | timeout); |
164 | dma_fence_put(fence); |
165 | |
166 | /* Retirement is best effort */ |
167 | if (!mutex_trylock(lock: &tl->mutex)) { |
168 | active_count++; |
169 | goto out_active; |
170 | } |
171 | } |
172 | } |
173 | |
174 | if (!retire_requests(tl)) |
175 | active_count++; |
176 | mutex_unlock(lock: &tl->mutex); |
177 | |
178 | out_active: spin_lock(lock: &timelines->lock); |
179 | |
180 | /* Resume list iteration after reacquiring spinlock */ |
181 | list_safe_reset_next(tl, tn, link); |
182 | if (atomic_dec_and_test(v: &tl->active_count)) |
183 | list_del(entry: &tl->link); |
184 | |
185 | /* Defer the final release to after the spinlock */ |
186 | if (refcount_dec_and_test(r: &tl->kref.refcount)) { |
187 | GEM_BUG_ON(atomic_read(&tl->active_count)); |
188 | list_add(new: &tl->link, head: &free); |
189 | } |
190 | } |
191 | spin_unlock(lock: &timelines->lock); |
192 | |
193 | list_for_each_entry_safe(tl, tn, &free, link) |
194 | __intel_timeline_free(kref: &tl->kref); |
195 | |
196 | if (flush_submission(gt, timeout)) /* Wait, there's more! */ |
197 | active_count++; |
198 | |
199 | if (remaining_timeout) |
200 | *remaining_timeout = timeout; |
201 | |
202 | return active_count ? timeout ?: -ETIME : 0; |
203 | } |
204 | |
205 | static void retire_work_handler(struct work_struct *work) |
206 | { |
207 | struct intel_gt *gt = |
208 | container_of(work, typeof(*gt), requests.retire_work.work); |
209 | |
210 | queue_delayed_work(wq: gt->i915->unordered_wq, dwork: >->requests.retire_work, |
211 | delay: round_jiffies_up_relative(HZ)); |
212 | intel_gt_retire_requests(gt); |
213 | } |
214 | |
215 | void intel_gt_init_requests(struct intel_gt *gt) |
216 | { |
217 | INIT_DELAYED_WORK(>->requests.retire_work, retire_work_handler); |
218 | } |
219 | |
220 | void intel_gt_park_requests(struct intel_gt *gt) |
221 | { |
222 | cancel_delayed_work(dwork: >->requests.retire_work); |
223 | } |
224 | |
225 | void intel_gt_unpark_requests(struct intel_gt *gt) |
226 | { |
227 | queue_delayed_work(wq: gt->i915->unordered_wq, dwork: >->requests.retire_work, |
228 | delay: round_jiffies_up_relative(HZ)); |
229 | } |
230 | |
231 | void intel_gt_fini_requests(struct intel_gt *gt) |
232 | { |
233 | /* Wait until the work is marked as finished before unloading! */ |
234 | cancel_delayed_work_sync(dwork: >->requests.retire_work); |
235 | |
236 | flush_work(work: >->watchdog.work); |
237 | } |
238 | |
239 | void intel_gt_watchdog_work(struct work_struct *work) |
240 | { |
241 | struct intel_gt *gt = |
242 | container_of(work, typeof(*gt), watchdog.work); |
243 | struct i915_request *rq, *rn; |
244 | struct llist_node *first; |
245 | |
246 | first = llist_del_all(head: >->watchdog.list); |
247 | if (!first) |
248 | return; |
249 | |
250 | llist_for_each_entry_safe(rq, rn, first, watchdog.link) { |
251 | if (!i915_request_completed(rq)) { |
252 | struct dma_fence *f = &rq->fence; |
253 | |
254 | pr_notice("Fence expiration time out i915-%s:%s:%llx!\n" , |
255 | f->ops->get_driver_name(f), |
256 | f->ops->get_timeline_name(f), |
257 | f->seqno); |
258 | i915_request_cancel(rq, error: -EINTR); |
259 | } |
260 | i915_request_put(rq); |
261 | } |
262 | } |
263 | |