1 | #![cfg (any( |
2 | target_os = "linux" , |
3 | target_os = "android" , |
4 | all(target_os = "emscripten" , target_feature = "atomics" ), |
5 | target_os = "freebsd" , |
6 | target_os = "openbsd" , |
7 | target_os = "dragonfly" , |
8 | target_os = "fuchsia" , |
9 | ))] |
10 | |
11 | use crate::sync::atomic::AtomicU32; |
12 | use crate::time::Duration; |
13 | |
14 | /// Wait for a futex_wake operation to wake us. |
15 | /// |
16 | /// Returns directly if the futex doesn't hold the expected value. |
17 | /// |
18 | /// Returns false on timeout, and true in all other cases. |
19 | #[cfg (any(target_os = "linux" , target_os = "android" , target_os = "freebsd" ))] |
20 | pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool { |
21 | use super::time::Timespec; |
22 | use crate::ptr::null; |
23 | use crate::sync::atomic::Ordering::Relaxed; |
24 | |
25 | // Calculate the timeout as an absolute timespec. |
26 | // |
27 | // Overflows are rounded up to an infinite timeout (None). |
28 | let timespec = timeout |
29 | .and_then(|d| Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d)) |
30 | .and_then(|t| t.to_timespec()); |
31 | |
32 | loop { |
33 | // No need to wait if the value already changed. |
34 | if futex.load(Relaxed) != expected { |
35 | return true; |
36 | } |
37 | |
38 | let r = unsafe { |
39 | cfg_if::cfg_if! { |
40 | if #[cfg(target_os = "freebsd" )] { |
41 | // FreeBSD doesn't have futex(), but it has |
42 | // _umtx_op(UMTX_OP_WAIT_UINT_PRIVATE), which is nearly |
43 | // identical. It supports absolute timeouts through a flag |
44 | // in the _umtx_time struct. |
45 | let umtx_timeout = timespec.map(|t| libc::_umtx_time { |
46 | _timeout: t, |
47 | _flags: libc::UMTX_ABSTIME, |
48 | _clockid: libc::CLOCK_MONOTONIC as u32, |
49 | }); |
50 | let umtx_timeout_ptr = umtx_timeout.as_ref().map_or(null(), |t| t as *const _); |
51 | let umtx_timeout_size = umtx_timeout.as_ref().map_or(0, |t| crate::mem::size_of_val(t)); |
52 | libc::_umtx_op( |
53 | futex as *const AtomicU32 as *mut _, |
54 | libc::UMTX_OP_WAIT_UINT_PRIVATE, |
55 | expected as libc::c_ulong, |
56 | crate::ptr::without_provenance_mut(umtx_timeout_size), |
57 | umtx_timeout_ptr as *mut _, |
58 | ) |
59 | } else if #[cfg(any(target_os = "linux" , target_os = "android" ))] { |
60 | // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an |
61 | // absolute time rather than a relative time. |
62 | libc::syscall( |
63 | libc::SYS_futex, |
64 | futex as *const AtomicU32, |
65 | libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG, |
66 | expected, |
67 | timespec.as_ref().map_or(null(), |t| t as *const libc::timespec), |
68 | null::<u32>(), // This argument is unused for FUTEX_WAIT_BITSET. |
69 | !0u32, // A full bitmask, to make it behave like a regular FUTEX_WAIT. |
70 | ) |
71 | } else { |
72 | compile_error!("unknown target_os" ); |
73 | } |
74 | } |
75 | }; |
76 | |
77 | match (r < 0).then(super::os::errno) { |
78 | Some(libc::ETIMEDOUT) => return false, |
79 | Some(libc::EINTR) => continue, |
80 | _ => return true, |
81 | } |
82 | } |
83 | } |
84 | |
85 | /// Wake up one thread that's blocked on futex_wait on this futex. |
86 | /// |
87 | /// Returns true if this actually woke up such a thread, |
88 | /// or false if no thread was waiting on this futex. |
89 | /// |
90 | /// On some platforms, this always returns false. |
91 | #[cfg (any(target_os = "linux" , target_os = "android" ))] |
92 | pub fn futex_wake(futex: &AtomicU32) -> bool { |
93 | let ptr: *const AtomicU32 = futex as *const AtomicU32; |
94 | let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG; |
95 | unsafe { libc::syscall(libc::SYS_futex, ptr, op, 1) > 0 } |
96 | } |
97 | |
98 | /// Wake up all threads that are waiting on futex_wait on this futex. |
99 | #[cfg (any(target_os = "linux" , target_os = "android" ))] |
100 | pub fn futex_wake_all(futex: &AtomicU32) { |
101 | let ptr: *const AtomicU32 = futex as *const AtomicU32; |
102 | let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG; |
103 | unsafe { |
104 | libc::syscall(libc::SYS_futex, ptr, op, i32::MAX); |
105 | } |
106 | } |
107 | |
108 | // FreeBSD doesn't tell us how many threads are woken up, so this always returns false. |
109 | #[cfg (target_os = "freebsd" )] |
110 | pub fn futex_wake(futex: &AtomicU32) -> bool { |
111 | use crate::ptr::null_mut; |
112 | unsafe { |
113 | libc::_umtx_op( |
114 | futex as *const AtomicU32 as *mut _, |
115 | libc::UMTX_OP_WAKE_PRIVATE, |
116 | 1, |
117 | null_mut(), |
118 | null_mut(), |
119 | ) |
120 | }; |
121 | false |
122 | } |
123 | |
124 | #[cfg (target_os = "freebsd" )] |
125 | pub fn futex_wake_all(futex: &AtomicU32) { |
126 | use crate::ptr::null_mut; |
127 | unsafe { |
128 | libc::_umtx_op( |
129 | futex as *const AtomicU32 as *mut _, |
130 | libc::UMTX_OP_WAKE_PRIVATE, |
131 | i32::MAX as libc::c_ulong, |
132 | null_mut(), |
133 | null_mut(), |
134 | ) |
135 | }; |
136 | } |
137 | |
138 | #[cfg (target_os = "openbsd" )] |
139 | pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool { |
140 | use super::time::Timespec; |
141 | use crate::ptr::{null, null_mut}; |
142 | |
143 | // Overflows are rounded up to an infinite timeout (None). |
144 | let timespec = timeout |
145 | .and_then(|d| Timespec::zero().checked_add_duration(&d)) |
146 | .and_then(|t| t.to_timespec()); |
147 | |
148 | let r = unsafe { |
149 | libc::futex( |
150 | futex as *const AtomicU32 as *mut u32, |
151 | libc::FUTEX_WAIT, |
152 | expected as i32, |
153 | timespec.as_ref().map_or(null(), |t| t as *const libc::timespec), |
154 | null_mut(), |
155 | ) |
156 | }; |
157 | |
158 | r == 0 || super::os::errno() != libc::ETIMEDOUT |
159 | } |
160 | |
161 | #[cfg (target_os = "openbsd" )] |
162 | pub fn futex_wake(futex: &AtomicU32) -> bool { |
163 | use crate::ptr::{null, null_mut}; |
164 | unsafe { |
165 | libc::futex(futex as *const AtomicU32 as *mut u32, libc::FUTEX_WAKE, 1, null(), null_mut()) |
166 | > 0 |
167 | } |
168 | } |
169 | |
170 | #[cfg (target_os = "openbsd" )] |
171 | pub fn futex_wake_all(futex: &AtomicU32) { |
172 | use crate::ptr::{null, null_mut}; |
173 | unsafe { |
174 | libc::futex( |
175 | futex as *const AtomicU32 as *mut u32, |
176 | libc::FUTEX_WAKE, |
177 | i32::MAX, |
178 | null(), |
179 | null_mut(), |
180 | ); |
181 | } |
182 | } |
183 | |
184 | #[cfg (target_os = "dragonfly" )] |
185 | pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool { |
186 | // A timeout of 0 means infinite. |
187 | // We round smaller timeouts up to 1 millisecond. |
188 | // Overflows are rounded up to an infinite timeout. |
189 | let timeout_ms = |
190 | timeout.and_then(|d| Some(i32::try_from(d.as_millis()).ok()?.max(1))).unwrap_or(0); |
191 | |
192 | let r = unsafe { |
193 | libc::umtx_sleep(futex as *const AtomicU32 as *const i32, expected as i32, timeout_ms) |
194 | }; |
195 | |
196 | r == 0 || super::os::errno() != libc::ETIMEDOUT |
197 | } |
198 | |
199 | // DragonflyBSD doesn't tell us how many threads are woken up, so this always returns false. |
200 | #[cfg (target_os = "dragonfly" )] |
201 | pub fn futex_wake(futex: &AtomicU32) -> bool { |
202 | unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, 1) }; |
203 | false |
204 | } |
205 | |
206 | #[cfg (target_os = "dragonfly" )] |
207 | pub fn futex_wake_all(futex: &AtomicU32) { |
208 | unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, i32::MAX) }; |
209 | } |
210 | |
211 | #[cfg (target_os = "emscripten" )] |
212 | extern "C" { |
213 | fn emscripten_futex_wake(addr: *const AtomicU32, count: libc::c_int) -> libc::c_int; |
214 | fn emscripten_futex_wait( |
215 | addr: *const AtomicU32, |
216 | val: libc::c_uint, |
217 | max_wait_ms: libc::c_double, |
218 | ) -> libc::c_int; |
219 | } |
220 | |
221 | #[cfg (target_os = "emscripten" )] |
222 | pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool { |
223 | unsafe { |
224 | emscripten_futex_wait( |
225 | futex, |
226 | expected, |
227 | timeout.map_or(f64::INFINITY, |d| d.as_secs_f64() * 1000.0), |
228 | ) != -libc::ETIMEDOUT |
229 | } |
230 | } |
231 | |
232 | #[cfg (target_os = "emscripten" )] |
233 | pub fn futex_wake(futex: &AtomicU32) -> bool { |
234 | unsafe { emscripten_futex_wake(futex, 1) > 0 } |
235 | } |
236 | |
237 | #[cfg (target_os = "emscripten" )] |
238 | pub fn futex_wake_all(futex: &AtomicU32) { |
239 | unsafe { emscripten_futex_wake(futex, i32::MAX) }; |
240 | } |
241 | |
242 | #[cfg (target_os = "fuchsia" )] |
243 | pub mod zircon { |
244 | pub type zx_futex_t = crate::sync::atomic::AtomicU32; |
245 | pub type zx_handle_t = u32; |
246 | pub type zx_status_t = i32; |
247 | pub type zx_time_t = i64; |
248 | |
249 | pub const ZX_HANDLE_INVALID: zx_handle_t = 0; |
250 | |
251 | pub const ZX_TIME_INFINITE: zx_time_t = zx_time_t::MAX; |
252 | |
253 | pub const ZX_OK: zx_status_t = 0; |
254 | pub const ZX_ERR_INVALID_ARGS: zx_status_t = -10; |
255 | pub const ZX_ERR_BAD_HANDLE: zx_status_t = -11; |
256 | pub const ZX_ERR_WRONG_TYPE: zx_status_t = -12; |
257 | pub const ZX_ERR_BAD_STATE: zx_status_t = -20; |
258 | pub const ZX_ERR_TIMED_OUT: zx_status_t = -21; |
259 | |
260 | extern "C" { |
261 | pub fn zx_clock_get_monotonic() -> zx_time_t; |
262 | pub fn zx_futex_wait( |
263 | value_ptr: *const zx_futex_t, |
264 | current_value: zx_futex_t, |
265 | new_futex_owner: zx_handle_t, |
266 | deadline: zx_time_t, |
267 | ) -> zx_status_t; |
268 | pub fn zx_futex_wake(value_ptr: *const zx_futex_t, wake_count: u32) -> zx_status_t; |
269 | pub fn zx_futex_wake_single_owner(value_ptr: *const zx_futex_t) -> zx_status_t; |
270 | pub fn zx_thread_self() -> zx_handle_t; |
271 | } |
272 | } |
273 | |
274 | #[cfg (target_os = "fuchsia" )] |
275 | pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool { |
276 | // Sleep forever if the timeout is longer than fits in a i64. |
277 | let deadline = timeout |
278 | .and_then(|d| { |
279 | i64::try_from(d.as_nanos()) |
280 | .ok()? |
281 | .checked_add(unsafe { zircon::zx_clock_get_monotonic() }) |
282 | }) |
283 | .unwrap_or(zircon::ZX_TIME_INFINITE); |
284 | |
285 | unsafe { |
286 | zircon::zx_futex_wait(futex, AtomicU32::new(expected), zircon::ZX_HANDLE_INVALID, deadline) |
287 | != zircon::ZX_ERR_TIMED_OUT |
288 | } |
289 | } |
290 | |
291 | // Fuchsia doesn't tell us how many threads are woken up, so this always returns false. |
292 | #[cfg (target_os = "fuchsia" )] |
293 | pub fn futex_wake(futex: &AtomicU32) -> bool { |
294 | unsafe { zircon::zx_futex_wake(futex, 1) }; |
295 | false |
296 | } |
297 | |
298 | #[cfg (target_os = "fuchsia" )] |
299 | pub fn futex_wake_all(futex: &AtomicU32) { |
300 | unsafe { zircon::zx_futex_wake(futex, u32::MAX) }; |
301 | } |
302 | |