1#![cfg(any(
2 target_os = "linux",
3 target_os = "android",
4 all(target_os = "emscripten", target_feature = "atomics"),
5 target_os = "freebsd",
6 target_os = "openbsd",
7 target_os = "dragonfly",
8 target_os = "fuchsia",
9))]
10
11use crate::sync::atomic::AtomicU32;
12use crate::time::Duration;
13
14/// Wait for a futex_wake operation to wake us.
15///
16/// Returns directly if the futex doesn't hold the expected value.
17///
18/// Returns false on timeout, and true in all other cases.
19#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))]
20pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
21 use super::time::Timespec;
22 use crate::ptr::null;
23 use crate::sync::atomic::Ordering::Relaxed;
24
25 // Calculate the timeout as an absolute timespec.
26 //
27 // Overflows are rounded up to an infinite timeout (None).
28 let timespec = timeout
29 .and_then(|d| Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d))
30 .and_then(|t| t.to_timespec());
31
32 loop {
33 // No need to wait if the value already changed.
34 if futex.load(Relaxed) != expected {
35 return true;
36 }
37
38 let r = unsafe {
39 cfg_if::cfg_if! {
40 if #[cfg(target_os = "freebsd")] {
41 // FreeBSD doesn't have futex(), but it has
42 // _umtx_op(UMTX_OP_WAIT_UINT_PRIVATE), which is nearly
43 // identical. It supports absolute timeouts through a flag
44 // in the _umtx_time struct.
45 let umtx_timeout = timespec.map(|t| libc::_umtx_time {
46 _timeout: t,
47 _flags: libc::UMTX_ABSTIME,
48 _clockid: libc::CLOCK_MONOTONIC as u32,
49 });
50 let umtx_timeout_ptr = umtx_timeout.as_ref().map_or(null(), |t| t as *const _);
51 let umtx_timeout_size = umtx_timeout.as_ref().map_or(0, |t| crate::mem::size_of_val(t));
52 libc::_umtx_op(
53 futex as *const AtomicU32 as *mut _,
54 libc::UMTX_OP_WAIT_UINT_PRIVATE,
55 expected as libc::c_ulong,
56 crate::ptr::without_provenance_mut(umtx_timeout_size),
57 umtx_timeout_ptr as *mut _,
58 )
59 } else if #[cfg(any(target_os = "linux", target_os = "android"))] {
60 // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an
61 // absolute time rather than a relative time.
62 libc::syscall(
63 libc::SYS_futex,
64 futex as *const AtomicU32,
65 libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG,
66 expected,
67 timespec.as_ref().map_or(null(), |t| t as *const libc::timespec),
68 null::<u32>(), // This argument is unused for FUTEX_WAIT_BITSET.
69 !0u32, // A full bitmask, to make it behave like a regular FUTEX_WAIT.
70 )
71 } else {
72 compile_error!("unknown target_os");
73 }
74 }
75 };
76
77 match (r < 0).then(super::os::errno) {
78 Some(libc::ETIMEDOUT) => return false,
79 Some(libc::EINTR) => continue,
80 _ => return true,
81 }
82 }
83}
84
85/// Wake up one thread that's blocked on futex_wait on this futex.
86///
87/// Returns true if this actually woke up such a thread,
88/// or false if no thread was waiting on this futex.
89///
90/// On some platforms, this always returns false.
91#[cfg(any(target_os = "linux", target_os = "android"))]
92pub fn futex_wake(futex: &AtomicU32) -> bool {
93 let ptr: *const AtomicU32 = futex as *const AtomicU32;
94 let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG;
95 unsafe { libc::syscall(libc::SYS_futex, ptr, op, 1) > 0 }
96}
97
98/// Wake up all threads that are waiting on futex_wait on this futex.
99#[cfg(any(target_os = "linux", target_os = "android"))]
100pub fn futex_wake_all(futex: &AtomicU32) {
101 let ptr: *const AtomicU32 = futex as *const AtomicU32;
102 let op = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG;
103 unsafe {
104 libc::syscall(libc::SYS_futex, ptr, op, i32::MAX);
105 }
106}
107
108// FreeBSD doesn't tell us how many threads are woken up, so this always returns false.
109#[cfg(target_os = "freebsd")]
110pub fn futex_wake(futex: &AtomicU32) -> bool {
111 use crate::ptr::null_mut;
112 unsafe {
113 libc::_umtx_op(
114 futex as *const AtomicU32 as *mut _,
115 libc::UMTX_OP_WAKE_PRIVATE,
116 1,
117 null_mut(),
118 null_mut(),
119 )
120 };
121 false
122}
123
124#[cfg(target_os = "freebsd")]
125pub fn futex_wake_all(futex: &AtomicU32) {
126 use crate::ptr::null_mut;
127 unsafe {
128 libc::_umtx_op(
129 futex as *const AtomicU32 as *mut _,
130 libc::UMTX_OP_WAKE_PRIVATE,
131 i32::MAX as libc::c_ulong,
132 null_mut(),
133 null_mut(),
134 )
135 };
136}
137
138#[cfg(target_os = "openbsd")]
139pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
140 use super::time::Timespec;
141 use crate::ptr::{null, null_mut};
142
143 // Overflows are rounded up to an infinite timeout (None).
144 let timespec = timeout
145 .and_then(|d| Timespec::zero().checked_add_duration(&d))
146 .and_then(|t| t.to_timespec());
147
148 let r = unsafe {
149 libc::futex(
150 futex as *const AtomicU32 as *mut u32,
151 libc::FUTEX_WAIT,
152 expected as i32,
153 timespec.as_ref().map_or(null(), |t| t as *const libc::timespec),
154 null_mut(),
155 )
156 };
157
158 r == 0 || super::os::errno() != libc::ETIMEDOUT
159}
160
161#[cfg(target_os = "openbsd")]
162pub fn futex_wake(futex: &AtomicU32) -> bool {
163 use crate::ptr::{null, null_mut};
164 unsafe {
165 libc::futex(futex as *const AtomicU32 as *mut u32, libc::FUTEX_WAKE, 1, null(), null_mut())
166 > 0
167 }
168}
169
170#[cfg(target_os = "openbsd")]
171pub fn futex_wake_all(futex: &AtomicU32) {
172 use crate::ptr::{null, null_mut};
173 unsafe {
174 libc::futex(
175 futex as *const AtomicU32 as *mut u32,
176 libc::FUTEX_WAKE,
177 i32::MAX,
178 null(),
179 null_mut(),
180 );
181 }
182}
183
184#[cfg(target_os = "dragonfly")]
185pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
186 // A timeout of 0 means infinite.
187 // We round smaller timeouts up to 1 millisecond.
188 // Overflows are rounded up to an infinite timeout.
189 let timeout_ms =
190 timeout.and_then(|d| Some(i32::try_from(d.as_millis()).ok()?.max(1))).unwrap_or(0);
191
192 let r = unsafe {
193 libc::umtx_sleep(futex as *const AtomicU32 as *const i32, expected as i32, timeout_ms)
194 };
195
196 r == 0 || super::os::errno() != libc::ETIMEDOUT
197}
198
199// DragonflyBSD doesn't tell us how many threads are woken up, so this always returns false.
200#[cfg(target_os = "dragonfly")]
201pub fn futex_wake(futex: &AtomicU32) -> bool {
202 unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, 1) };
203 false
204}
205
206#[cfg(target_os = "dragonfly")]
207pub fn futex_wake_all(futex: &AtomicU32) {
208 unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, i32::MAX) };
209}
210
211#[cfg(target_os = "emscripten")]
212extern "C" {
213 fn emscripten_futex_wake(addr: *const AtomicU32, count: libc::c_int) -> libc::c_int;
214 fn emscripten_futex_wait(
215 addr: *const AtomicU32,
216 val: libc::c_uint,
217 max_wait_ms: libc::c_double,
218 ) -> libc::c_int;
219}
220
221#[cfg(target_os = "emscripten")]
222pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
223 unsafe {
224 emscripten_futex_wait(
225 futex,
226 expected,
227 timeout.map_or(f64::INFINITY, |d| d.as_secs_f64() * 1000.0),
228 ) != -libc::ETIMEDOUT
229 }
230}
231
232#[cfg(target_os = "emscripten")]
233pub fn futex_wake(futex: &AtomicU32) -> bool {
234 unsafe { emscripten_futex_wake(futex, 1) > 0 }
235}
236
237#[cfg(target_os = "emscripten")]
238pub fn futex_wake_all(futex: &AtomicU32) {
239 unsafe { emscripten_futex_wake(futex, i32::MAX) };
240}
241
242#[cfg(target_os = "fuchsia")]
243pub mod zircon {
244 pub type zx_futex_t = crate::sync::atomic::AtomicU32;
245 pub type zx_handle_t = u32;
246 pub type zx_status_t = i32;
247 pub type zx_time_t = i64;
248
249 pub const ZX_HANDLE_INVALID: zx_handle_t = 0;
250
251 pub const ZX_TIME_INFINITE: zx_time_t = zx_time_t::MAX;
252
253 pub const ZX_OK: zx_status_t = 0;
254 pub const ZX_ERR_INVALID_ARGS: zx_status_t = -10;
255 pub const ZX_ERR_BAD_HANDLE: zx_status_t = -11;
256 pub const ZX_ERR_WRONG_TYPE: zx_status_t = -12;
257 pub const ZX_ERR_BAD_STATE: zx_status_t = -20;
258 pub const ZX_ERR_TIMED_OUT: zx_status_t = -21;
259
260 extern "C" {
261 pub fn zx_clock_get_monotonic() -> zx_time_t;
262 pub fn zx_futex_wait(
263 value_ptr: *const zx_futex_t,
264 current_value: zx_futex_t,
265 new_futex_owner: zx_handle_t,
266 deadline: zx_time_t,
267 ) -> zx_status_t;
268 pub fn zx_futex_wake(value_ptr: *const zx_futex_t, wake_count: u32) -> zx_status_t;
269 pub fn zx_futex_wake_single_owner(value_ptr: *const zx_futex_t) -> zx_status_t;
270 pub fn zx_thread_self() -> zx_handle_t;
271 }
272}
273
274#[cfg(target_os = "fuchsia")]
275pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
276 // Sleep forever if the timeout is longer than fits in a i64.
277 let deadline = timeout
278 .and_then(|d| {
279 i64::try_from(d.as_nanos())
280 .ok()?
281 .checked_add(unsafe { zircon::zx_clock_get_monotonic() })
282 })
283 .unwrap_or(zircon::ZX_TIME_INFINITE);
284
285 unsafe {
286 zircon::zx_futex_wait(futex, AtomicU32::new(expected), zircon::ZX_HANDLE_INVALID, deadline)
287 != zircon::ZX_ERR_TIMED_OUT
288 }
289}
290
291// Fuchsia doesn't tell us how many threads are woken up, so this always returns false.
292#[cfg(target_os = "fuchsia")]
293pub fn futex_wake(futex: &AtomicU32) -> bool {
294 unsafe { zircon::zx_futex_wake(futex, 1) };
295 false
296}
297
298#[cfg(target_os = "fuchsia")]
299pub fn futex_wake_all(futex: &AtomicU32) {
300 unsafe { zircon::zx_futex_wake(futex, u32::MAX) };
301}
302