1 | /* |
2 | * SPDX-License-Identifier: MIT |
3 | * |
4 | * Copyright © 2019 Intel Corporation |
5 | */ |
6 | |
7 | #include <linux/wait_bit.h> |
8 | |
9 | #include "intel_runtime_pm.h" |
10 | #include "intel_wakeref.h" |
11 | #include "i915_drv.h" |
12 | |
13 | int __intel_wakeref_get_first(struct intel_wakeref *wf) |
14 | { |
15 | intel_wakeref_t wakeref; |
16 | int ret = 0; |
17 | |
18 | wakeref = intel_runtime_pm_get(rpm: &wf->i915->runtime_pm); |
19 | /* |
20 | * Treat get/put as different subclasses, as we may need to run |
21 | * the put callback from under the shrinker and do not want to |
22 | * cross-contanimate that callback with any extra work performed |
23 | * upon acquiring the wakeref. |
24 | */ |
25 | mutex_lock_nested(lock: &wf->mutex, SINGLE_DEPTH_NESTING); |
26 | |
27 | if (!atomic_read(v: &wf->count)) { |
28 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
29 | wf->wakeref = wakeref; |
30 | wakeref = 0; |
31 | |
32 | ret = wf->ops->get(wf); |
33 | if (ret) { |
34 | wakeref = xchg(&wf->wakeref, 0); |
35 | wake_up_var(var: &wf->wakeref); |
36 | goto unlock; |
37 | } |
38 | |
39 | smp_mb__before_atomic(); /* release wf->count */ |
40 | } |
41 | |
42 | atomic_inc(v: &wf->count); |
43 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
44 | |
45 | unlock: |
46 | mutex_unlock(lock: &wf->mutex); |
47 | if (unlikely(wakeref)) |
48 | intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref); |
49 | |
50 | return ret; |
51 | } |
52 | |
53 | static void ____intel_wakeref_put_last(struct intel_wakeref *wf) |
54 | { |
55 | intel_wakeref_t wakeref = 0; |
56 | |
57 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
58 | if (unlikely(!atomic_dec_and_test(&wf->count))) |
59 | goto unlock; |
60 | |
61 | /* ops->put() must reschedule its own release on error/deferral */ |
62 | if (likely(!wf->ops->put(wf))) { |
63 | INTEL_WAKEREF_BUG_ON(!wf->wakeref); |
64 | wakeref = xchg(&wf->wakeref, 0); |
65 | wake_up_var(var: &wf->wakeref); |
66 | } |
67 | |
68 | unlock: |
69 | mutex_unlock(lock: &wf->mutex); |
70 | if (wakeref) |
71 | intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref); |
72 | } |
73 | |
74 | void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) |
75 | { |
76 | INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work)); |
77 | |
78 | /* Assume we are not in process context and so cannot sleep. */ |
79 | if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(lock: &wf->mutex)) { |
80 | mod_delayed_work(wq: wf->i915->unordered_wq, dwork: &wf->work, |
81 | FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); |
82 | return; |
83 | } |
84 | |
85 | ____intel_wakeref_put_last(wf); |
86 | } |
87 | |
88 | static void __intel_wakeref_put_work(struct work_struct *wrk) |
89 | { |
90 | struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work); |
91 | |
92 | if (atomic_add_unless(v: &wf->count, a: -1, u: 1)) |
93 | return; |
94 | |
95 | mutex_lock(&wf->mutex); |
96 | ____intel_wakeref_put_last(wf); |
97 | } |
98 | |
99 | void __intel_wakeref_init(struct intel_wakeref *wf, |
100 | struct drm_i915_private *i915, |
101 | const struct intel_wakeref_ops *ops, |
102 | struct intel_wakeref_lockclass *key) |
103 | { |
104 | wf->i915 = i915; |
105 | wf->ops = ops; |
106 | |
107 | __mutex_init(lock: &wf->mutex, name: "wakeref.mutex" , key: &key->mutex); |
108 | atomic_set(v: &wf->count, i: 0); |
109 | wf->wakeref = 0; |
110 | |
111 | INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work); |
112 | lockdep_init_map(lock: &wf->work.work.lockdep_map, |
113 | name: "wakeref.work" , key: &key->work, subclass: 0); |
114 | } |
115 | |
116 | int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) |
117 | { |
118 | int err; |
119 | |
120 | might_sleep(); |
121 | |
122 | err = wait_var_event_killable(&wf->wakeref, |
123 | !intel_wakeref_is_active(wf)); |
124 | if (err) |
125 | return err; |
126 | |
127 | intel_wakeref_unlock_wait(wf); |
128 | return 0; |
129 | } |
130 | |
131 | static void wakeref_auto_timeout(struct timer_list *t) |
132 | { |
133 | struct intel_wakeref_auto *wf = from_timer(wf, t, timer); |
134 | intel_wakeref_t wakeref; |
135 | unsigned long flags; |
136 | |
137 | if (!refcount_dec_and_lock_irqsave(r: &wf->count, lock: &wf->lock, flags: &flags)) |
138 | return; |
139 | |
140 | wakeref = fetch_and_zero(&wf->wakeref); |
141 | spin_unlock_irqrestore(lock: &wf->lock, flags); |
142 | |
143 | intel_runtime_pm_put(rpm: &wf->i915->runtime_pm, wref: wakeref); |
144 | } |
145 | |
146 | void intel_wakeref_auto_init(struct intel_wakeref_auto *wf, |
147 | struct drm_i915_private *i915) |
148 | { |
149 | spin_lock_init(&wf->lock); |
150 | timer_setup(&wf->timer, wakeref_auto_timeout, 0); |
151 | refcount_set(r: &wf->count, n: 0); |
152 | wf->wakeref = 0; |
153 | wf->i915 = i915; |
154 | } |
155 | |
156 | void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout) |
157 | { |
158 | unsigned long flags; |
159 | |
160 | if (!timeout) { |
161 | if (del_timer_sync(timer: &wf->timer)) |
162 | wakeref_auto_timeout(t: &wf->timer); |
163 | return; |
164 | } |
165 | |
166 | /* Our mission is that we only extend an already active wakeref */ |
167 | assert_rpm_wakelock_held(rpm: &wf->i915->runtime_pm); |
168 | |
169 | if (!refcount_inc_not_zero(r: &wf->count)) { |
170 | spin_lock_irqsave(&wf->lock, flags); |
171 | if (!refcount_inc_not_zero(r: &wf->count)) { |
172 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
173 | wf->wakeref = |
174 | intel_runtime_pm_get_if_in_use(rpm: &wf->i915->runtime_pm); |
175 | refcount_set(r: &wf->count, n: 1); |
176 | } |
177 | spin_unlock_irqrestore(lock: &wf->lock, flags); |
178 | } |
179 | |
180 | /* |
181 | * If we extend a pending timer, we will only get a single timer |
182 | * callback and so need to cancel the local inc by running the |
183 | * elided callback to keep the wf->count balanced. |
184 | */ |
185 | if (mod_timer(timer: &wf->timer, expires: jiffies + timeout)) |
186 | wakeref_auto_timeout(t: &wf->timer); |
187 | } |
188 | |
189 | void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf) |
190 | { |
191 | intel_wakeref_auto(wf, timeout: 0); |
192 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
193 | } |
194 | |