1//! Linux `futex`.
2//!
3//! Futex is a very low-level mechanism for implementing concurrency primitives
4//! such as mutexes, rwlocks, and condvars.
5//!
6//! # Examples
7//!
8//! ```
9//! use rustix::thread::futex;
10//! use std::sync::atomic::AtomicU32;
11//!
12//! # fn test(futex: &AtomicU32) -> rustix::io::Result<()> {
13//! // Wake up one waiter.
14//! futex::wake(futex, futex::Flags::PRIVATE, 1)?;
15//! # Ok(())
16//! # }
17//! ```
18//!
19//! # References
20//! - [Linux `futex` system call]
21//! - [Linux `futex` feature]
22//!
23//! [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
24//! [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
25#![allow(unsafe_code)]
26
27use core::num::NonZeroU32;
28use core::ptr;
29use core::sync::atomic::AtomicU32;
30
31use crate::backend::thread::futex::Operation;
32use crate::backend::thread::syscalls::{futex_timeout, futex_val2};
33use crate::fd::{FromRawFd, OwnedFd, RawFd};
34use crate::utils::option_as_ptr;
35use crate::{backend, io};
36
37pub use crate::timespec::{Nsecs, Secs, Timespec};
38
39pub use backend::thread::futex::{Flags, OWNER_DIED, WAITERS};
40
41/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT, val, timeout, NULL, 0)`
42///
43/// This is a very low-level feature for implementing synchronization
44/// primitives. See the references links.
45///
46/// # References
47/// - [Linux `futex` system call]
48/// - [Linux `futex` feature]
49///
50/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
51/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
52#[inline]
53pub fn wait(
54 uaddr: &AtomicU32,
55 flags: Flags,
56 val: u32,
57 timeout: Option<Timespec>,
58) -> io::Result<()> {
59 // SAFETY: The raw pointers come from references or null.
60 unsafe {
61 futex_timeout(
62 uaddr,
63 Operation::Wait,
64 flags,
65 val,
66 option_as_ptr(timeout.as_ref()),
67 ptr::null(),
68 0,
69 )
70 .map(|val: usize| {
71 debug_assert_eq!(
72 val, 0,
73 "The return value should always equal zero, if the call is successful"
74 );
75 })
76 }
77}
78
79/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE, val, NULL, NULL, 0)`
80///
81/// This is a very low-level feature for implementing synchronization
82/// primitives. See the references links.
83///
84/// # References
85/// - [Linux `futex` system call]
86/// - [Linux `futex` feature]
87///
88/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
89/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
90#[inline]
91pub fn wake(uaddr: &AtomicU32, flags: Flags, val: u32) -> io::Result<usize> {
92 // SAFETY: The raw pointers come from references or null.
93 unsafe { futex_val2(uaddr, op:Operation::Wake, flags, val, val2:0, uaddr2:ptr::null(), val3:0) }
94}
95
96/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_FD, val, NULL, NULL, 0)`
97///
98/// This is a very low-level feature for implementing synchronization
99/// primitives. See the references links.
100///
101/// # References
102/// - [Linux `futex` system call]
103/// - [Linux `futex` feature]
104///
105/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
106/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
107#[inline]
108pub fn fd(uaddr: &AtomicU32, flags: Flags, val: u32) -> io::Result<OwnedFd> {
109 // SAFETY: The raw pointers come from references or null.
110 unsafe {
111 futex_val2(uaddr, Operation::Fd, flags, val, 0, ptr::null(), 0).map(|val: usize| {
112 let fd: usize = val as RawFd;
113 debug_assert_eq!(fd as usize, val, "return value should be a valid fd");
114 OwnedFd::from_raw_fd(fd)
115 })
116 }
117}
118
119/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_REQUEUE, val, val2, uaddr2, 0)`
120///
121/// This is a very low-level feature for implementing synchronization
122/// primitives. See the references links.
123///
124/// # References
125/// - [Linux `futex` system call]
126/// - [Linux `futex` feature]
127///
128/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
129/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
130#[inline]
131pub fn requeue(
132 uaddr: &AtomicU32,
133 flags: Flags,
134 val: u32,
135 val2: u32,
136 uaddr2: &AtomicU32,
137) -> io::Result<usize> {
138 // SAFETY: The raw pointers come from references or null.
139 unsafe { futex_val2(uaddr, op:Operation::Requeue, flags, val, val2, uaddr2, val3:0) }
140}
141
142/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE, val, val2, uaddr2, val3)`
143///
144/// This is a very low-level feature for implementing synchronization
145/// primitives. See the references links.
146///
147/// # References
148/// - [Linux `futex` system call]
149/// - [Linux `futex` feature]
150///
151/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
152/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
153#[inline]
154pub fn cmp_requeue(
155 uaddr: &AtomicU32,
156 flags: Flags,
157 val: u32,
158 val2: u32,
159 uaddr2: &AtomicU32,
160 val3: u32,
161) -> io::Result<usize> {
162 // SAFETY: The raw pointers come from references or null.
163 unsafe { futex_val2(uaddr, op:Operation::CmpRequeue, flags, val, val2, uaddr2, val3) }
164}
165
166/// `FUTEX_OP_*` operations for use with [`wake_op`].
167#[derive(Debug, Copy, Clone, Eq, PartialEq)]
168#[repr(u32)]
169#[allow(clippy::identity_op)]
170pub enum WakeOp {
171 /// `FUTEX_OP_SET`: `uaddr2 = oparg;`
172 Set = 0,
173 /// `FUTEX_OP_ADD`: `uaddr2 += oparg;`
174 Add = 1,
175 /// `FUTEX_OP_OR`: `uaddr2 |= oparg;`
176 Or = 2,
177 /// `FUTEX_OP_ANDN`: `uaddr2 &= ~oparg;`
178 AndN = 3,
179 /// `FUTEX_OP_XOR`: `uaddr2 ^= oparg;`
180 XOr = 4,
181 /// `FUTEX_OP_SET | FUTEX_OP_ARG_SHIFT`: `uaddr2 = (oparg << 1);`
182 SetShift = 0 | 8,
183 /// `FUTEX_OP_ADD | FUTEX_OP_ARG_SHIFT`: `uaddr2 += (oparg << 1);`
184 AddShift = 1 | 8,
185 /// `FUTEX_OP_OR | FUTEX_OP_ARG_SHIFT`: `uaddr2 |= (oparg << 1);`
186 OrShift = 2 | 8,
187 /// `FUTEX_OP_ANDN | FUTEX_OP_ARG_SHIFT`: `uaddr2 &= !(oparg << 1);`
188 AndNShift = 3 | 8,
189 /// `FUTEX_OP_XOR | FUTEX_OP_ARG_SHIFT`: `uaddr2 ^= (oparg << 1);`
190 XOrShift = 4 | 8,
191}
192
193/// `FUTEX_OP_CMP_*` operations for use with [`wake_op`].
194#[derive(Debug, Copy, Clone, Eq, PartialEq)]
195#[repr(u32)]
196pub enum WakeOpCmp {
197 /// `FUTEX_OP_CMP_EQ`: `if oldval == cmparg { wake(); }`
198 Eq = 0,
199 /// `FUTEX_OP_CMP_EQ`: `if oldval != cmparg { wake(); }`
200 Ne = 1,
201 /// `FUTEX_OP_CMP_EQ`: `if oldval < cmparg { wake(); }`
202 Lt = 2,
203 /// `FUTEX_OP_CMP_EQ`: `if oldval <= cmparg { wake(); }`
204 Le = 3,
205 /// `FUTEX_OP_CMP_EQ`: `if oldval > cmparg { wake(); }`
206 Gt = 4,
207 /// `FUTEX_OP_CMP_EQ`: `if oldval >= cmparg { wake(); }`
208 Ge = 5,
209}
210
211/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE_OP, val, val2, uaddr2, val3)`
212///
213/// This is a very low-level feature for implementing synchronization
214/// primitives. See the references links.
215///
216/// # References
217/// - [Linux `futex` system call]
218/// - [Linux `futex` feature]
219///
220/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
221/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
222#[inline]
223#[allow(clippy::too_many_arguments)]
224pub fn wake_op(
225 uaddr: &AtomicU32,
226 flags: Flags,
227 val: u32,
228 val2: u32,
229 uaddr2: &AtomicU32,
230 op: WakeOp,
231 cmp: WakeOpCmp,
232 oparg: u16,
233 cmparg: u16,
234) -> io::Result<usize> {
235 if oparg >= 1 << 12 || cmparg >= 1 << 12 {
236 return Err(io::Errno::INVAL);
237 }
238
239 let val3: u32 =
240 ((op as u32) << 28) | ((cmp as u32) << 24) | ((oparg as u32) << 12) | (cmparg as u32);
241
242 // SAFETY: The raw pointers come from references or null.
243 unsafe { futex_val2(uaddr, op:Operation::WakeOp, flags, val, val2, uaddr2, val3) }
244}
245
246/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI, 0, timeout, NULL, 0)`
247///
248/// This is a very low-level feature for implementing synchronization
249/// primitives. See the references links.
250///
251/// # References
252/// - [Linux `futex` system call]
253/// - [Linux `futex` feature]
254///
255/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
256/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
257#[inline]
258pub fn lock_pi(uaddr: &AtomicU32, flags: Flags, timeout: Option<Timespec>) -> io::Result<()> {
259 // SAFETY: The raw pointers come from references or null.
260 unsafe {
261 futex_timeout(
262 uaddr,
263 Operation::LockPi,
264 flags,
265 0,
266 option_as_ptr(timeout.as_ref()),
267 ptr::null(),
268 0,
269 )
270 .map(|val: usize| {
271 debug_assert_eq!(
272 val, 0,
273 "The return value should always equal zero, if the call is successful"
274 );
275 })
276 }
277}
278
279/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_UNLOCK_PI, 0, NULL, NULL, 0)`
280///
281/// This is a very low-level feature for implementing synchronization
282/// primitives. See the references links.
283///
284/// # References
285/// - [Linux `futex` system call]
286/// - [Linux `futex` feature]
287///
288/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
289/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
290#[inline]
291pub fn unlock_pi(uaddr: &AtomicU32, flags: Flags) -> io::Result<()> {
292 // SAFETY: The raw pointers come from references or null.
293 unsafe {
294 futex_val2(uaddr, Operation::UnlockPi, flags, 0, 0, ptr::null(), 0).map(|val: usize| {
295 debug_assert_eq!(
296 val, 0,
297 "The return value should always equal zero, if the call is successful"
298 );
299 })
300 }
301}
302
303/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_TRYLOCK_PI, 0, NULL, NULL, 0)`
304///
305/// This is a very low-level feature for implementing synchronization
306/// primitives. See the references links.
307///
308/// # References
309/// - [Linux `futex` system call]
310/// - [Linux `futex` feature]
311///
312/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
313/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
314#[inline]
315pub fn trylock_pi(uaddr: &AtomicU32, flags: Flags) -> io::Result<bool> {
316 // SAFETY: The raw pointers come from references or null.
317 unsafe {
318 futex_val2(uaddr, Operation::TrylockPi, flags, 0, 0, ptr::null(), 0).map(|ret: usize| ret == 0)
319 }
320}
321
322/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT_BITSET, val, timeout, NULL, val3)`
323///
324/// This is a very low-level feature for implementing synchronization
325/// primitives. See the references links.
326///
327/// # References
328/// - [Linux `futex` system call]
329/// - [Linux `futex` feature]
330///
331/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
332/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
333#[inline]
334pub fn wait_bitset(
335 uaddr: &AtomicU32,
336 flags: Flags,
337 val: u32,
338 timeout: Option<Timespec>,
339 val3: NonZeroU32,
340) -> io::Result<()> {
341 // SAFETY: The raw pointers come from references or null.
342 unsafe {
343 futex_timeout(
344 uaddr,
345 Operation::WaitBitset,
346 flags,
347 val,
348 option_as_ptr(timeout.as_ref()),
349 ptr::null(),
350 val3.get(),
351 )
352 .map(|val: usize| {
353 debug_assert_eq!(
354 val, 0,
355 "The return value should always equal zero, if the call is successful"
356 );
357 })
358 }
359}
360
361/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAKE_BITSET, val, NULL, NULL, val3)`
362///
363/// This is a very low-level feature for implementing synchronization
364/// primitives. See the references links.
365///
366/// # References
367/// - [Linux `futex` system call]
368/// - [Linux `futex` feature]
369///
370/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
371/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
372#[inline]
373pub fn wake_bitset(
374 uaddr: &AtomicU32,
375 flags: Flags,
376 val: u32,
377 val3: NonZeroU32,
378) -> io::Result<usize> {
379 // SAFETY: The raw pointers come from references or null.
380 unsafe {
381 futex_val2(
382 uaddr,
383 op:Operation::WakeBitset,
384 flags,
385 val,
386 val2:0,
387 uaddr2:ptr::null(),
388 val3.get(),
389 )
390 }
391}
392
393/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_WAIT_REQUEUE_PI, val, timeout, uaddr2, 0)`
394///
395/// This is a very low-level feature for implementing synchronization
396/// primitives. See the references links.
397///
398/// # References
399/// - [Linux `futex` system call]
400/// - [Linux `futex` feature]
401///
402/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
403/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
404#[inline]
405pub fn wait_requeue_pi(
406 uaddr: &AtomicU32,
407 flags: Flags,
408 val: u32,
409 timeout: Option<Timespec>,
410 uaddr2: &AtomicU32,
411) -> io::Result<()> {
412 // SAFETY: The raw pointers come from references or null.
413 unsafe {
414 futex_timeout(
415 uaddr,
416 Operation::WaitRequeuePi,
417 flags,
418 val,
419 option_as_ptr(timeout.as_ref()),
420 uaddr2,
421 0,
422 )
423 .map(|val: usize| {
424 debug_assert_eq!(
425 val, 0,
426 "The return value should always equal zero, if the call is successful"
427 );
428 })
429 }
430}
431
432/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_CMP_REQUEUE_PI, 1, val2, uaddr2, val3)`
433///
434/// This is a very low-level feature for implementing synchronization
435/// primitives. See the references links.
436///
437/// # References
438/// - [Linux `futex` system call]
439/// - [Linux `futex` feature]
440///
441/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
442/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
443#[inline]
444pub fn cmp_requeue_pi(
445 uaddr: &AtomicU32,
446 flags: Flags,
447 val2: u32,
448 uaddr2: &AtomicU32,
449 val3: u32,
450) -> io::Result<usize> {
451 // SAFETY: The raw pointers come from references or null.
452 unsafe { futex_val2(uaddr, op:Operation::CmpRequeuePi, flags, val:1, val2, uaddr2, val3) }
453}
454
455/// Equivalent to `syscall(SYS_futex, uaddr, FUTEX_LOCK_PI2, 0, timeout, NULL, 0)`
456///
457/// This is a very low-level feature for implementing synchronization
458/// primitives. See the references links.
459///
460/// # References
461/// - [Linux `futex` system call]
462/// - [Linux `futex` feature]
463///
464/// [Linux `futex` system call]: https://man7.org/linux/man-pages/man2/futex.2.html
465/// [Linux `futex` feature]: https://man7.org/linux/man-pages/man7/futex.7.html
466#[inline]
467pub fn lock_pi2(uaddr: &AtomicU32, flags: Flags, timeout: Option<Timespec>) -> io::Result<()> {
468 // SAFETY: The raw pointers come from references or null.
469 unsafe {
470 futex_timeout(
471 uaddr,
472 Operation::LockPi2,
473 flags,
474 0,
475 option_as_ptr(timeout.as_ref()),
476 ptr::null(),
477 0,
478 )
479 .map(|val: usize| {
480 debug_assert_eq!(
481 val, 0,
482 "The return value should always equal zero, if the call is successful"
483 );
484 })
485 }
486}
487