1 | /* |
2 | * Copyright © 2014 Intel Corporation |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
21 | * DEALINGS IN THE SOFTWARE. |
22 | * |
23 | * Authors: |
24 | * Daniel Vetter <daniel.vetter@ffwll.ch> |
25 | */ |
26 | |
27 | /** |
28 | * DOC: frontbuffer tracking |
29 | * |
30 | * Many features require us to track changes to the currently active |
31 | * frontbuffer, especially rendering targeted at the frontbuffer. |
32 | * |
33 | * To be able to do so we track frontbuffers using a bitmask for all possible |
34 | * frontbuffer slots through intel_frontbuffer_track(). The functions in this |
35 | * file are then called when the contents of the frontbuffer are invalidated, |
36 | * when frontbuffer rendering has stopped again to flush out all the changes |
37 | * and when the frontbuffer is exchanged with a flip. Subsystems interested in |
38 | * frontbuffer changes (e.g. PSR, FBC, DRRS) should directly put their callbacks |
39 | * into the relevant places and filter for the frontbuffer slots that they are |
40 | * interested int. |
41 | * |
42 | * On a high level there are two types of powersaving features. The first one |
43 | * work like a special cache (FBC and PSR) and are interested when they should |
44 | * stop caching and when to restart caching. This is done by placing callbacks |
45 | * into the invalidate and the flush functions: At invalidate the caching must |
46 | * be stopped and at flush time it can be restarted. And maybe they need to know |
47 | * when the frontbuffer changes (e.g. when the hw doesn't initiate an invalidate |
48 | * and flush on its own) which can be achieved with placing callbacks into the |
49 | * flip functions. |
50 | * |
51 | * The other type of display power saving feature only cares about busyness |
52 | * (e.g. DRRS). In that case all three (invalidate, flush and flip) indicate |
53 | * busyness. There is no direct way to detect idleness. Instead an idle timer |
54 | * work delayed work should be started from the flush and flip functions and |
55 | * cancelled as soon as busyness is detected. |
56 | */ |
57 | |
58 | #include "gem/i915_gem_object_frontbuffer.h" |
59 | #include "i915_active.h" |
60 | #include "i915_drv.h" |
61 | #include "intel_display_trace.h" |
62 | #include "intel_display_types.h" |
63 | #include "intel_dp.h" |
64 | #include "intel_drrs.h" |
65 | #include "intel_fbc.h" |
66 | #include "intel_frontbuffer.h" |
67 | #include "intel_psr.h" |
68 | |
69 | /** |
70 | * frontbuffer_flush - flush frontbuffer |
71 | * @i915: i915 device |
72 | * @frontbuffer_bits: frontbuffer plane tracking bits |
73 | * @origin: which operation caused the flush |
74 | * |
75 | * This function gets called every time rendering on the given planes has |
76 | * completed and frontbuffer caching can be started again. Flushes will get |
77 | * delayed if they're blocked by some outstanding asynchronous rendering. |
78 | * |
79 | * Can be called without any locks held. |
80 | */ |
81 | static void frontbuffer_flush(struct drm_i915_private *i915, |
82 | unsigned int frontbuffer_bits, |
83 | enum fb_op_origin origin) |
84 | { |
85 | /* Delay flushing when rings are still busy.*/ |
86 | spin_lock(lock: &i915->display.fb_tracking.lock); |
87 | frontbuffer_bits &= ~i915->display.fb_tracking.busy_bits; |
88 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
89 | |
90 | if (!frontbuffer_bits) |
91 | return; |
92 | |
93 | trace_intel_frontbuffer_flush(i915, frontbuffer_bits, origin); |
94 | |
95 | might_sleep(); |
96 | intel_drrs_flush(dev_priv: i915, frontbuffer_bits); |
97 | intel_psr_flush(dev_priv: i915, frontbuffer_bits, origin); |
98 | intel_fbc_flush(dev_priv: i915, frontbuffer_bits, origin); |
99 | } |
100 | |
101 | /** |
102 | * intel_frontbuffer_flip_prepare - prepare asynchronous frontbuffer flip |
103 | * @i915: i915 device |
104 | * @frontbuffer_bits: frontbuffer plane tracking bits |
105 | * |
106 | * This function gets called after scheduling a flip on @obj. The actual |
107 | * frontbuffer flushing will be delayed until completion is signalled with |
108 | * intel_frontbuffer_flip_complete. If an invalidate happens in between this |
109 | * flush will be cancelled. |
110 | * |
111 | * Can be called without any locks held. |
112 | */ |
113 | void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915, |
114 | unsigned frontbuffer_bits) |
115 | { |
116 | spin_lock(lock: &i915->display.fb_tracking.lock); |
117 | i915->display.fb_tracking.flip_bits |= frontbuffer_bits; |
118 | /* Remove stale busy bits due to the old buffer. */ |
119 | i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits; |
120 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
121 | } |
122 | |
123 | /** |
124 | * intel_frontbuffer_flip_complete - complete asynchronous frontbuffer flip |
125 | * @i915: i915 device |
126 | * @frontbuffer_bits: frontbuffer plane tracking bits |
127 | * |
128 | * This function gets called after the flip has been latched and will complete |
129 | * on the next vblank. It will execute the flush if it hasn't been cancelled yet. |
130 | * |
131 | * Can be called without any locks held. |
132 | */ |
133 | void intel_frontbuffer_flip_complete(struct drm_i915_private *i915, |
134 | unsigned frontbuffer_bits) |
135 | { |
136 | spin_lock(lock: &i915->display.fb_tracking.lock); |
137 | /* Mask any cancelled flips. */ |
138 | frontbuffer_bits &= i915->display.fb_tracking.flip_bits; |
139 | i915->display.fb_tracking.flip_bits &= ~frontbuffer_bits; |
140 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
141 | |
142 | if (frontbuffer_bits) |
143 | frontbuffer_flush(i915, frontbuffer_bits, origin: ORIGIN_FLIP); |
144 | } |
145 | |
146 | /** |
147 | * intel_frontbuffer_flip - synchronous frontbuffer flip |
148 | * @i915: i915 device |
149 | * @frontbuffer_bits: frontbuffer plane tracking bits |
150 | * |
151 | * This function gets called after scheduling a flip on @obj. This is for |
152 | * synchronous plane updates which will happen on the next vblank and which will |
153 | * not get delayed by pending gpu rendering. |
154 | * |
155 | * Can be called without any locks held. |
156 | */ |
157 | void intel_frontbuffer_flip(struct drm_i915_private *i915, |
158 | unsigned frontbuffer_bits) |
159 | { |
160 | spin_lock(lock: &i915->display.fb_tracking.lock); |
161 | /* Remove stale busy bits due to the old buffer. */ |
162 | i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits; |
163 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
164 | |
165 | frontbuffer_flush(i915, frontbuffer_bits, origin: ORIGIN_FLIP); |
166 | } |
167 | |
168 | void __intel_fb_invalidate(struct intel_frontbuffer *front, |
169 | enum fb_op_origin origin, |
170 | unsigned int frontbuffer_bits) |
171 | { |
172 | struct drm_i915_private *i915 = intel_bo_to_i915(front->obj); |
173 | |
174 | if (origin == ORIGIN_CS) { |
175 | spin_lock(lock: &i915->display.fb_tracking.lock); |
176 | i915->display.fb_tracking.busy_bits |= frontbuffer_bits; |
177 | i915->display.fb_tracking.flip_bits &= ~frontbuffer_bits; |
178 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
179 | } |
180 | |
181 | trace_intel_frontbuffer_invalidate(i915, frontbuffer_bits, origin); |
182 | |
183 | might_sleep(); |
184 | intel_psr_invalidate(dev_priv: i915, frontbuffer_bits, origin); |
185 | intel_drrs_invalidate(dev_priv: i915, frontbuffer_bits); |
186 | intel_fbc_invalidate(dev_priv: i915, frontbuffer_bits, origin); |
187 | } |
188 | |
189 | void __intel_fb_flush(struct intel_frontbuffer *front, |
190 | enum fb_op_origin origin, |
191 | unsigned int frontbuffer_bits) |
192 | { |
193 | struct drm_i915_private *i915 = intel_bo_to_i915(front->obj); |
194 | |
195 | if (origin == ORIGIN_CS) { |
196 | spin_lock(lock: &i915->display.fb_tracking.lock); |
197 | /* Filter out new bits since rendering started. */ |
198 | frontbuffer_bits &= i915->display.fb_tracking.busy_bits; |
199 | i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits; |
200 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
201 | } |
202 | |
203 | if (frontbuffer_bits) |
204 | frontbuffer_flush(i915, frontbuffer_bits, origin); |
205 | } |
206 | |
207 | static void intel_frontbuffer_flush_work(struct work_struct *work) |
208 | { |
209 | struct intel_frontbuffer *front = |
210 | container_of(work, struct intel_frontbuffer, flush_work); |
211 | |
212 | i915_gem_object_flush_if_display(obj: front->obj); |
213 | intel_frontbuffer_flush(front, origin: ORIGIN_DIRTYFB); |
214 | intel_frontbuffer_put(front); |
215 | } |
216 | |
217 | /** |
218 | * intel_frontbuffer_queue_flush - queue flushing frontbuffer object |
219 | * @front: GEM object to flush |
220 | * |
221 | * This function is targeted for our dirty callback for queueing flush when |
222 | * dma fence is signales |
223 | */ |
224 | void intel_frontbuffer_queue_flush(struct intel_frontbuffer *front) |
225 | { |
226 | if (!front) |
227 | return; |
228 | |
229 | kref_get(kref: &front->ref); |
230 | if (!schedule_work(work: &front->flush_work)) |
231 | intel_frontbuffer_put(front); |
232 | } |
233 | |
234 | static int frontbuffer_active(struct i915_active *ref) |
235 | { |
236 | struct intel_frontbuffer *front = |
237 | container_of(ref, typeof(*front), write); |
238 | |
239 | kref_get(kref: &front->ref); |
240 | return 0; |
241 | } |
242 | |
243 | static void frontbuffer_retire(struct i915_active *ref) |
244 | { |
245 | struct intel_frontbuffer *front = |
246 | container_of(ref, typeof(*front), write); |
247 | |
248 | intel_frontbuffer_flush(front, origin: ORIGIN_CS); |
249 | intel_frontbuffer_put(front); |
250 | } |
251 | |
252 | static void frontbuffer_release(struct kref *ref) |
253 | __releases(&intel_bo_to_i915(front->obj)->display.fb_tracking.lock) |
254 | { |
255 | struct intel_frontbuffer *ret, *front = |
256 | container_of(ref, typeof(*front), ref); |
257 | struct drm_i915_gem_object *obj = front->obj; |
258 | |
259 | drm_WARN_ON(&intel_bo_to_i915(obj)->drm, atomic_read(&front->bits)); |
260 | |
261 | i915_ggtt_clear_scanout(obj); |
262 | |
263 | ret = i915_gem_object_set_frontbuffer(obj, NULL); |
264 | drm_WARN_ON(&intel_bo_to_i915(obj)->drm, ret); |
265 | spin_unlock(lock: &intel_bo_to_i915(obj)->display.fb_tracking.lock); |
266 | |
267 | i915_active_fini(ref: &front->write); |
268 | kfree_rcu(front, rcu); |
269 | } |
270 | |
271 | struct intel_frontbuffer * |
272 | intel_frontbuffer_get(struct drm_i915_gem_object *obj) |
273 | { |
274 | struct drm_i915_private *i915 = intel_bo_to_i915(obj); |
275 | struct intel_frontbuffer *front, *cur; |
276 | |
277 | front = i915_gem_object_get_frontbuffer(obj); |
278 | if (front) |
279 | return front; |
280 | |
281 | front = kmalloc(size: sizeof(*front), GFP_KERNEL); |
282 | if (!front) |
283 | return NULL; |
284 | |
285 | front->obj = obj; |
286 | kref_init(kref: &front->ref); |
287 | atomic_set(v: &front->bits, i: 0); |
288 | i915_active_init(&front->write, |
289 | frontbuffer_active, |
290 | frontbuffer_retire, |
291 | I915_ACTIVE_RETIRE_SLEEPS); |
292 | INIT_WORK(&front->flush_work, intel_frontbuffer_flush_work); |
293 | |
294 | spin_lock(lock: &i915->display.fb_tracking.lock); |
295 | cur = i915_gem_object_set_frontbuffer(obj, front); |
296 | spin_unlock(lock: &i915->display.fb_tracking.lock); |
297 | if (cur != front) |
298 | kfree(objp: front); |
299 | return cur; |
300 | } |
301 | |
302 | void intel_frontbuffer_put(struct intel_frontbuffer *front) |
303 | { |
304 | kref_put_lock(kref: &front->ref, |
305 | release: frontbuffer_release, |
306 | lock: &intel_bo_to_i915(front->obj)->display.fb_tracking.lock); |
307 | } |
308 | |
309 | /** |
310 | * intel_frontbuffer_track - update frontbuffer tracking |
311 | * @old: current buffer for the frontbuffer slots |
312 | * @new: new buffer for the frontbuffer slots |
313 | * @frontbuffer_bits: bitmask of frontbuffer slots |
314 | * |
315 | * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them |
316 | * from @old and setting them in @new. Both @old and @new can be NULL. |
317 | */ |
318 | void intel_frontbuffer_track(struct intel_frontbuffer *old, |
319 | struct intel_frontbuffer *new, |
320 | unsigned int frontbuffer_bits) |
321 | { |
322 | /* |
323 | * Control of individual bits within the mask are guarded by |
324 | * the owning plane->mutex, i.e. we can never see concurrent |
325 | * manipulation of individual bits. But since the bitfield as a whole |
326 | * is updated using RMW, we need to use atomics in order to update |
327 | * the bits. |
328 | */ |
329 | BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > |
330 | BITS_PER_TYPE(atomic_t)); |
331 | BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 32); |
332 | BUILD_BUG_ON(I915_MAX_PLANES > INTEL_FRONTBUFFER_BITS_PER_PIPE); |
333 | |
334 | if (old) { |
335 | drm_WARN_ON(&intel_bo_to_i915(old->obj)->drm, |
336 | !(atomic_read(&old->bits) & frontbuffer_bits)); |
337 | atomic_andnot(i: frontbuffer_bits, v: &old->bits); |
338 | } |
339 | |
340 | if (new) { |
341 | drm_WARN_ON(&intel_bo_to_i915(new->obj)->drm, |
342 | atomic_read(&new->bits) & frontbuffer_bits); |
343 | atomic_or(i: frontbuffer_bits, v: &new->bits); |
344 | } |
345 | } |
346 | |