1#![cfg(any(
2 target_os = "linux",
3 target_os = "android",
4 all(target_os = "emscripten", target_feature = "atomics"),
5 target_os = "freebsd",
6 target_os = "openbsd",
7 target_os = "dragonfly",
8 target_os = "fuchsia",
9))]
10
11use crate::sync::atomic::AtomicU32;
12use crate::time::Duration;
13
14/// An atomic for use as a futex that is at least 32-bits but may be larger
15pub type Futex = AtomicU32;
16/// Must be the underlying type of Futex
17pub type Primitive = u32;
18
19/// An atomic for use as a futex that is at least 8-bits but may be larger.
20pub type SmallFutex = AtomicU32;
21/// Must be the underlying type of SmallFutex
22pub type SmallPrimitive = u32;
23
24/// Waits for a `futex_wake` operation to wake us.
25///
26/// Returns directly if the futex doesn't hold the expected value.
27///
28/// Returns false on timeout, and true in all other cases.
29#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))]
30pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
31 use super::time::Timespec;
32 use crate::ptr::null;
33 use crate::sync::atomic::Ordering::Relaxed;
34
35 // Calculate the timeout as an absolute timespec.
36 //
37 // Overflows are rounded up to an infinite timeout (None).
38 let timespec = timeout
39 .and_then(|d| Timespec::now(libc::CLOCK_MONOTONIC).checked_add_duration(&d))
40 .and_then(|t| t.to_timespec());
41
42 loop {
43 // No need to wait if the value already changed.
44 if futex.load(Relaxed) != expected {
45 return true;
46 }
47
48 let r = unsafe {
49 cfg_if::cfg_if! {
50 if #[cfg(target_os = "freebsd")] {
51 // FreeBSD doesn't have futex(), but it has
52 // _umtx_op(UMTX_OP_WAIT_UINT_PRIVATE), which is nearly
53 // identical. It supports absolute timeouts through a flag
54 // in the _umtx_time struct.
55 let umtx_timeout = timespec.map(|t| libc::_umtx_time {
56 _timeout: t,
57 _flags: libc::UMTX_ABSTIME,
58 _clockid: libc::CLOCK_MONOTONIC as u32,
59 });
60 let umtx_timeout_ptr = umtx_timeout.as_ref().map_or(null(), |t| t as *const _);
61 let umtx_timeout_size = umtx_timeout.as_ref().map_or(0, |t| size_of_val(t));
62 libc::_umtx_op(
63 futex as *const AtomicU32 as *mut _,
64 libc::UMTX_OP_WAIT_UINT_PRIVATE,
65 expected as libc::c_ulong,
66 crate::ptr::without_provenance_mut(umtx_timeout_size),
67 umtx_timeout_ptr as *mut _,
68 )
69 } else if #[cfg(any(target_os = "linux", target_os = "android"))] {
70 // Use FUTEX_WAIT_BITSET rather than FUTEX_WAIT to be able to give an
71 // absolute time rather than a relative time.
72 libc::syscall(
73 libc::SYS_futex,
74 futex as *const AtomicU32,
75 libc::FUTEX_WAIT_BITSET | libc::FUTEX_PRIVATE_FLAG,
76 expected,
77 timespec.as_ref().map_or(null(), |t| t as *const libc::timespec),
78 null::<u32>(), // This argument is unused for FUTEX_WAIT_BITSET.
79 !0u32, // A full bitmask, to make it behave like a regular FUTEX_WAIT.
80 )
81 } else {
82 compile_error!("unknown target_os");
83 }
84 }
85 };
86
87 match (r < 0).then(super::os::errno) {
88 Some(libc::ETIMEDOUT) => return false,
89 Some(libc::EINTR) => continue,
90 _ => return true,
91 }
92 }
93}
94
95/// Wakes up one thread that's blocked on `futex_wait` on this futex.
96///
97/// Returns true if this actually woke up such a thread,
98/// or false if no thread was waiting on this futex.
99///
100/// On some platforms, this always returns false.
101#[cfg(any(target_os = "linux", target_os = "android"))]
102pub fn futex_wake(futex: &AtomicU32) -> bool {
103 let ptr: *const AtomicU32 = futex as *const AtomicU32;
104 let op: i32 = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG;
105 unsafe { libc::syscall(num:libc::SYS_futex, ptr, op, 1) > 0 }
106}
107
108/// Wakes up all threads that are waiting on `futex_wait` on this futex.
109#[cfg(any(target_os = "linux", target_os = "android"))]
110pub fn futex_wake_all(futex: &AtomicU32) {
111 let ptr: *const AtomicU32 = futex as *const AtomicU32;
112 let op: i32 = libc::FUTEX_WAKE | libc::FUTEX_PRIVATE_FLAG;
113 unsafe {
114 libc::syscall(num:libc::SYS_futex, ptr, op, i32::MAX);
115 }
116}
117
118// FreeBSD doesn't tell us how many threads are woken up, so this always returns false.
119#[cfg(target_os = "freebsd")]
120pub fn futex_wake(futex: &AtomicU32) -> bool {
121 use crate::ptr::null_mut;
122 unsafe {
123 libc::_umtx_op(
124 futex as *const AtomicU32 as *mut _,
125 libc::UMTX_OP_WAKE_PRIVATE,
126 1,
127 null_mut(),
128 null_mut(),
129 )
130 };
131 false
132}
133
134#[cfg(target_os = "freebsd")]
135pub fn futex_wake_all(futex: &AtomicU32) {
136 use crate::ptr::null_mut;
137 unsafe {
138 libc::_umtx_op(
139 futex as *const AtomicU32 as *mut _,
140 libc::UMTX_OP_WAKE_PRIVATE,
141 i32::MAX as libc::c_ulong,
142 null_mut(),
143 null_mut(),
144 )
145 };
146}
147
148#[cfg(target_os = "openbsd")]
149pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
150 use super::time::Timespec;
151 use crate::ptr::{null, null_mut};
152
153 // Overflows are rounded up to an infinite timeout (None).
154 let timespec = timeout
155 .and_then(|d| Timespec::zero().checked_add_duration(&d))
156 .and_then(|t| t.to_timespec());
157
158 let r = unsafe {
159 libc::futex(
160 futex as *const AtomicU32 as *mut u32,
161 libc::FUTEX_WAIT,
162 expected as i32,
163 timespec.as_ref().map_or(null(), |t| t as *const libc::timespec),
164 null_mut(),
165 )
166 };
167
168 r == 0 || super::os::errno() != libc::ETIMEDOUT
169}
170
171#[cfg(target_os = "openbsd")]
172pub fn futex_wake(futex: &AtomicU32) -> bool {
173 use crate::ptr::{null, null_mut};
174 unsafe {
175 libc::futex(futex as *const AtomicU32 as *mut u32, libc::FUTEX_WAKE, 1, null(), null_mut())
176 > 0
177 }
178}
179
180#[cfg(target_os = "openbsd")]
181pub fn futex_wake_all(futex: &AtomicU32) {
182 use crate::ptr::{null, null_mut};
183 unsafe {
184 libc::futex(
185 futex as *const AtomicU32 as *mut u32,
186 libc::FUTEX_WAKE,
187 i32::MAX,
188 null(),
189 null_mut(),
190 );
191 }
192}
193
194#[cfg(target_os = "dragonfly")]
195pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
196 // A timeout of 0 means infinite.
197 // We round smaller timeouts up to 1 millisecond.
198 // Overflows are rounded up to an infinite timeout.
199 let timeout_ms =
200 timeout.and_then(|d| Some(i32::try_from(d.as_millis()).ok()?.max(1))).unwrap_or(0);
201
202 let r = unsafe {
203 libc::umtx_sleep(futex as *const AtomicU32 as *const i32, expected as i32, timeout_ms)
204 };
205
206 r == 0 || super::os::errno() != libc::ETIMEDOUT
207}
208
209// DragonflyBSD doesn't tell us how many threads are woken up, so this always returns false.
210#[cfg(target_os = "dragonfly")]
211pub fn futex_wake(futex: &AtomicU32) -> bool {
212 unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, 1) };
213 false
214}
215
216#[cfg(target_os = "dragonfly")]
217pub fn futex_wake_all(futex: &AtomicU32) {
218 unsafe { libc::umtx_wakeup(futex as *const AtomicU32 as *const i32, i32::MAX) };
219}
220
221#[cfg(target_os = "emscripten")]
222unsafe extern "C" {
223 fn emscripten_futex_wake(addr: *const AtomicU32, count: libc::c_int) -> libc::c_int;
224 fn emscripten_futex_wait(
225 addr: *const AtomicU32,
226 val: libc::c_uint,
227 max_wait_ms: libc::c_double,
228 ) -> libc::c_int;
229}
230
231#[cfg(target_os = "emscripten")]
232pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
233 unsafe {
234 emscripten_futex_wait(
235 futex,
236 expected,
237 timeout.map_or(f64::INFINITY, |d| d.as_secs_f64() * 1000.0),
238 ) != -libc::ETIMEDOUT
239 }
240}
241
242#[cfg(target_os = "emscripten")]
243pub fn futex_wake(futex: &AtomicU32) -> bool {
244 unsafe { emscripten_futex_wake(futex, 1) > 0 }
245}
246
247#[cfg(target_os = "emscripten")]
248pub fn futex_wake_all(futex: &AtomicU32) {
249 unsafe { emscripten_futex_wake(futex, i32::MAX) };
250}
251
252#[cfg(target_os = "fuchsia")]
253pub mod zircon {
254 pub type zx_futex_t = crate::sync::atomic::AtomicU32;
255 pub type zx_handle_t = u32;
256 pub type zx_status_t = i32;
257 pub type zx_time_t = i64;
258
259 pub const ZX_HANDLE_INVALID: zx_handle_t = 0;
260
261 pub const ZX_TIME_INFINITE: zx_time_t = zx_time_t::MAX;
262
263 pub const ZX_OK: zx_status_t = 0;
264 pub const ZX_ERR_INVALID_ARGS: zx_status_t = -10;
265 pub const ZX_ERR_BAD_HANDLE: zx_status_t = -11;
266 pub const ZX_ERR_WRONG_TYPE: zx_status_t = -12;
267 pub const ZX_ERR_BAD_STATE: zx_status_t = -20;
268 pub const ZX_ERR_TIMED_OUT: zx_status_t = -21;
269
270 unsafe extern "C" {
271 pub fn zx_clock_get_monotonic() -> zx_time_t;
272 pub fn zx_futex_wait(
273 value_ptr: *const zx_futex_t,
274 current_value: zx_futex_t,
275 new_futex_owner: zx_handle_t,
276 deadline: zx_time_t,
277 ) -> zx_status_t;
278 pub fn zx_futex_wake(value_ptr: *const zx_futex_t, wake_count: u32) -> zx_status_t;
279 pub fn zx_futex_wake_single_owner(value_ptr: *const zx_futex_t) -> zx_status_t;
280 pub fn zx_thread_self() -> zx_handle_t;
281 }
282}
283
284#[cfg(target_os = "fuchsia")]
285pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
286 // Sleep forever if the timeout is longer than fits in a i64.
287 let deadline = timeout
288 .and_then(|d| {
289 i64::try_from(d.as_nanos())
290 .ok()?
291 .checked_add(unsafe { zircon::zx_clock_get_monotonic() })
292 })
293 .unwrap_or(zircon::ZX_TIME_INFINITE);
294
295 unsafe {
296 zircon::zx_futex_wait(futex, AtomicU32::new(expected), zircon::ZX_HANDLE_INVALID, deadline)
297 != zircon::ZX_ERR_TIMED_OUT
298 }
299}
300
301// Fuchsia doesn't tell us how many threads are woken up, so this always returns false.
302#[cfg(target_os = "fuchsia")]
303pub fn futex_wake(futex: &AtomicU32) -> bool {
304 unsafe { zircon::zx_futex_wake(futex, 1) };
305 false
306}
307
308#[cfg(target_os = "fuchsia")]
309pub fn futex_wake_all(futex: &AtomicU32) {
310 unsafe { zircon::zx_futex_wake(futex, u32::MAX) };
311}
312