1use crate::sync::atomic::Ordering;
2
3#[cfg(test)]
4use stdarch_test::assert_instr;
5
6/// Compares and exchange 16 bytes (128 bits) of data atomically.
7///
8/// This intrinsic corresponds to the `cmpxchg16b` instruction on `x86_64`
9/// processors. It performs an atomic compare-and-swap, updating the `ptr`
10/// memory location to `val` if the current value in memory equals `old`.
11///
12/// # Return value
13///
14/// This function returns the previous value at the memory location. If it is
15/// equal to `old` then the memory was updated to `new`.
16///
17/// # Memory Orderings
18///
19/// This atomic operation has the same semantics of memory orderings as
20/// `AtomicUsize::compare_exchange` does, only operating on 16 bytes of memory
21/// instead of just a pointer.
22///
23/// The failure ordering must be [`Ordering::SeqCst`], [`Ordering::Acquire`] or
24/// [`Ordering::Relaxed`].
25///
26/// For more information on memory orderings here see the `compare_exchange`
27/// documentation for other `Atomic*` types in the standard library.
28///
29/// # Unsafety
30///
31/// This method is unsafe because it takes a raw pointer and will attempt to
32/// read and possibly write the memory at the pointer. The pointer must also be
33/// aligned on a 16-byte boundary.
34///
35/// This method also requires the `cmpxchg16b` CPU feature to be available at
36/// runtime to work correctly. If the CPU running the binary does not actually
37/// support `cmpxchg16b` and the program enters an execution path that
38/// eventually would reach this function the behavior is undefined.
39#[inline]
40#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
41#[cfg_attr(test, assert_instr(cmpxchg16b, success = Ordering::SeqCst, failure = Ordering::SeqCst))]
42#[target_feature(enable = "cmpxchg16b")]
43#[stable(feature = "cmpxchg16b_intrinsic", since = "1.67.0")]
44pub unsafe fn cmpxchg16b(
45 dst: *mut u128,
46 old: u128,
47 new: u128,
48 success: Ordering,
49 failure: Ordering,
50) -> u128 {
51 use crate::{intrinsics, sync::atomic::Ordering::*};
52
53 debug_assert!(dst as usize % 16 == 0);
54
55 // Copied from `atomic_compare_exchange` in `core`.
56 // https://github.com/rust-lang/rust/blob/f8a2e49/library/core/src/sync/atomic.rs#L3046-L3079
57 let (val, _ok) = match (success, failure) {
58 (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
59 (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
60 (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
61 (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
62 (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
63 (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
64 (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
65 (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
66 (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
67 (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
68 (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
69 (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
70 (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
71 (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
72 (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
73 (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
74 (_, Release) => panic!("there is no such thing as a release failure ordering"),
75
76 // `atomic::Ordering` is non_exhaustive. It warns when `core_arch` is built as a part of `core`.
77 #[allow(unreachable_patterns)]
78 (_, _) => unreachable!(),
79 };
80 val
81}
82