1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7 all(
8 portable_atomic_no_atomic_load_store,
9 not(all(target_arch = "bpf", not(feature = "critical-section"))),
10 ),
11 portable_atomic_unsafe_assume_single_core,
12 target_arch = "avr",
13 target_arch = "msp430",
14)))]
15#[cfg_attr(
16 portable_atomic_no_cfg_target_has_atomic,
17 cfg(not(all(
18 any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
19 portable_atomic_no_atomic_cas,
20 )))
21)]
22#[cfg_attr(
23 not(portable_atomic_no_cfg_target_has_atomic),
24 cfg(not(all(
25 any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
26 not(target_has_atomic = "ptr"),
27 )))
28)]
29mod core_atomic;
30
31// aarch64 128-bit atomics
32#[cfg(all(
33 target_arch = "aarch64",
34 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
35))]
36// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
37#[cfg_attr(
38 all(any(miri, portable_atomic_sanitize_thread), portable_atomic_new_atomic_intrinsics),
39 path = "atomic128/intrinsics.rs"
40)]
41#[cfg_attr(
42 not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_new_atomic_intrinsics)),
43 path = "atomic128/aarch64.rs"
44)]
45mod aarch64;
46
47// x86_64 128-bit atomics
48#[cfg(all(
49 target_arch = "x86_64",
50 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
51 any(
52 target_feature = "cmpxchg16b",
53 portable_atomic_target_feature = "cmpxchg16b",
54 all(
55 feature = "fallback",
56 not(portable_atomic_no_cmpxchg16b_target_feature),
57 not(portable_atomic_no_outline_atomics),
58 not(any(target_env = "sgx", miri)),
59 ),
60 ),
61))]
62// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
63#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")]
64#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/x86_64.rs")]
65mod x86_64;
66
67// powerpc64 128-bit atomics
68#[cfg(all(
69 target_arch = "powerpc64",
70 portable_atomic_unstable_asm_experimental_arch,
71 any(
72 target_feature = "quadword-atomics",
73 portable_atomic_target_feature = "quadword-atomics",
74 all(
75 feature = "fallback",
76 not(portable_atomic_no_outline_atomics),
77 any(test, portable_atomic_outline_atomics), // TODO(powerpc64): currently disabled by default
78 any(
79 all(
80 target_os = "linux",
81 any(
82 target_env = "gnu",
83 all(
84 any(target_env = "musl", target_env = "ohos"),
85 not(target_feature = "crt-static"),
86 ),
87 portable_atomic_outline_atomics,
88 ),
89 ),
90 target_os = "android",
91 target_os = "freebsd",
92 ),
93 not(any(miri, portable_atomic_sanitize_thread)),
94 ),
95 ),
96))]
97// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
98#[cfg_attr(
99 all(any(miri, portable_atomic_sanitize_thread), portable_atomic_llvm_15),
100 path = "atomic128/intrinsics.rs"
101)]
102#[cfg_attr(
103 not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_llvm_15)),
104 path = "atomic128/powerpc64.rs"
105)]
106mod powerpc64;
107
108// s390x 128-bit atomics
109#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))]
110// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly.
111#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")]
112#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/s390x.rs")]
113mod s390x;
114
115// pre-v6 ARM Linux 64-bit atomics
116#[cfg(feature = "fallback")]
117// Miri and Sanitizer do not support inline assembly.
118#[cfg(all(
119 target_arch = "arm",
120 not(any(miri, portable_atomic_sanitize_thread)),
121 not(portable_atomic_no_asm),
122 any(target_os = "linux", target_os = "android"),
123 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
124 not(portable_atomic_no_outline_atomics),
125))]
126#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
127#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
128mod arm_linux;
129
130// MSP430 atomics
131#[cfg(target_arch = "msp430")]
132pub(crate) mod msp430;
133
134// atomic load/store for RISC-V without A-extension
135#[cfg(any(test, not(feature = "critical-section")))]
136#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
137#[cfg_attr(
138 not(portable_atomic_no_cfg_target_has_atomic),
139 cfg(any(test, not(target_has_atomic = "ptr")))
140)]
141#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
142mod riscv;
143
144// x86-specific optimizations
145// Miri and Sanitizer do not support inline assembly.
146#[cfg(all(
147 any(target_arch = "x86", target_arch = "x86_64"),
148 not(any(miri, portable_atomic_sanitize_thread)),
149 not(portable_atomic_no_asm),
150))]
151mod x86;
152
153// -----------------------------------------------------------------------------
154// Lock-based fallback implementations
155
156#[cfg(feature = "fallback")]
157#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
158#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
159#[cfg(any(
160 test,
161 not(any(
162 all(
163 target_arch = "aarch64",
164 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
165 ),
166 all(
167 target_arch = "x86_64",
168 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
169 any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
170 ),
171 all(
172 target_arch = "powerpc64",
173 portable_atomic_unstable_asm_experimental_arch,
174 any(
175 target_feature = "quadword-atomics",
176 portable_atomic_target_feature = "quadword-atomics",
177 ),
178 ),
179 all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch),
180 ))
181))]
182mod fallback;
183
184// -----------------------------------------------------------------------------
185// Critical section based fallback implementations
186
187// On AVR, we always use critical section based fallback implementation.
188// AVR can be safely assumed to be single-core, so this is sound.
189// https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074
190// MSP430 as well.
191#[cfg(any(
192 all(test, target_os = "none"),
193 portable_atomic_unsafe_assume_single_core,
194 feature = "critical-section",
195 target_arch = "avr",
196 target_arch = "msp430",
197))]
198#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
199#[cfg_attr(
200 not(portable_atomic_no_cfg_target_has_atomic),
201 cfg(any(test, not(target_has_atomic = "ptr")))
202)]
203#[cfg(any(
204 target_arch = "arm",
205 target_arch = "avr",
206 target_arch = "msp430",
207 target_arch = "riscv32",
208 target_arch = "riscv64",
209 target_arch = "xtensa",
210 feature = "critical-section",
211))]
212mod interrupt;
213
214// -----------------------------------------------------------------------------
215// Atomic float implementations
216
217#[cfg(feature = "float")]
218pub(crate) mod float;
219
220// -----------------------------------------------------------------------------
221
222#[cfg(not(any(
223 portable_atomic_no_atomic_load_store,
224 portable_atomic_unsafe_assume_single_core,
225 target_arch = "avr",
226 target_arch = "msp430",
227)))]
228#[cfg_attr(
229 portable_atomic_no_cfg_target_has_atomic,
230 cfg(not(all(
231 any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
232 portable_atomic_no_atomic_cas,
233 )))
234)]
235#[cfg_attr(
236 not(portable_atomic_no_cfg_target_has_atomic),
237 cfg(not(all(
238 any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"),
239 not(target_has_atomic = "ptr"),
240 )))
241)]
242items! {
243 pub(crate) use self::core_atomic::{
244 AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
245 AtomicUsize,
246 };
247 #[cfg_attr(
248 portable_atomic_no_cfg_target_has_atomic,
249 cfg(any(
250 not(portable_atomic_no_atomic_64),
251 not(any(target_pointer_width = "16", target_pointer_width = "32")),
252 ))
253 )]
254 #[cfg_attr(
255 not(portable_atomic_no_cfg_target_has_atomic),
256 cfg(any(
257 target_has_atomic = "64",
258 not(any(target_pointer_width = "16", target_pointer_width = "32")),
259 ))
260 )]
261 pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
262}
263// bpf
264#[cfg(all(
265 target_arch = "bpf",
266 portable_atomic_no_atomic_load_store,
267 not(feature = "critical-section"),
268))]
269pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
270
271// RISC-V without A-extension & !(assume single core | critical section)
272#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
273#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
274#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
275#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
276items! {
277 pub(crate) use self::riscv::{
278 AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
279 AtomicUsize,
280 };
281 #[cfg(target_arch = "riscv64")]
282 pub(crate) use self::riscv::{AtomicI64, AtomicU64};
283}
284
285// no core atomic CAS & (assume single core | critical section) => critical section based fallback
286#[cfg(any(
287 portable_atomic_unsafe_assume_single_core,
288 feature = "critical-section",
289 target_arch = "avr",
290 target_arch = "msp430",
291))]
292#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
293#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
294items! {
295 pub(crate) use self::interrupt::{
296 AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
297 };
298 #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
299 pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
300 #[cfg(any(
301 not(any(target_pointer_width = "16", target_pointer_width = "32")),
302 feature = "fallback",
303 ))]
304 pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
305 #[cfg(feature = "fallback")]
306 pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
307}
308
309// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
310#[cfg(feature = "fallback")]
311#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
312#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
313items! {
314 #[cfg(not(all(
315 target_arch = "arm",
316 not(any(miri, portable_atomic_sanitize_thread)),
317 not(portable_atomic_no_asm),
318 any(target_os = "linux", target_os = "android"),
319 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
320 not(portable_atomic_no_outline_atomics),
321 )))]
322 #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
323 #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
324 pub(crate) use self::fallback::{AtomicI64, AtomicU64};
325 #[cfg(not(any(
326 all(
327 target_arch = "aarch64",
328 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
329 ),
330 all(
331 target_arch = "x86_64",
332 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
333 any(
334 target_feature = "cmpxchg16b",
335 portable_atomic_target_feature = "cmpxchg16b",
336 all(
337 feature = "fallback",
338 not(portable_atomic_no_cmpxchg16b_target_feature),
339 not(portable_atomic_no_outline_atomics),
340 not(any(target_env = "sgx", miri)),
341 ),
342 ),
343 ),
344 all(
345 target_arch = "powerpc64",
346 portable_atomic_unstable_asm_experimental_arch,
347 any(
348 target_feature = "quadword-atomics",
349 portable_atomic_target_feature = "quadword-atomics",
350 all(
351 feature = "fallback",
352 not(portable_atomic_no_outline_atomics),
353 portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default
354 any(
355 all(
356 target_os = "linux",
357 any(
358 target_env = "gnu",
359 all(
360 any(target_env = "musl", target_env = "ohos"),
361 not(target_feature = "crt-static"),
362 ),
363 portable_atomic_outline_atomics,
364 ),
365 ),
366 target_os = "android",
367 target_os = "freebsd",
368 ),
369 not(any(miri, portable_atomic_sanitize_thread)),
370 ),
371 ),
372 ),
373 all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch),
374 )))]
375 pub(crate) use self::fallback::{AtomicI128, AtomicU128};
376}
377
378// 64-bit atomics (platform-specific)
379// pre-v6 ARM Linux
380#[cfg(feature = "fallback")]
381#[cfg(all(
382 target_arch = "arm",
383 not(any(miri, portable_atomic_sanitize_thread)),
384 not(portable_atomic_no_asm),
385 any(target_os = "linux", target_os = "android"),
386 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
387 not(portable_atomic_no_outline_atomics),
388))]
389#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
390#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
391pub(crate) use self::arm_linux::{AtomicI64, AtomicU64};
392
393// 128-bit atomics (platform-specific)
394// aarch64
395#[cfg(all(
396 target_arch = "aarch64",
397 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
398))]
399pub(crate) use self::aarch64::{AtomicI128, AtomicU128};
400// x86_64 & (cmpxchg16b | outline-atomics)
401#[cfg(all(
402 target_arch = "x86_64",
403 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
404 any(
405 target_feature = "cmpxchg16b",
406 portable_atomic_target_feature = "cmpxchg16b",
407 all(
408 feature = "fallback",
409 not(portable_atomic_no_cmpxchg16b_target_feature),
410 not(portable_atomic_no_outline_atomics),
411 not(any(target_env = "sgx", miri)),
412 ),
413 ),
414))]
415pub(crate) use self::x86_64::{AtomicI128, AtomicU128};
416// powerpc64 & (pwr8 | outline-atomics)
417#[cfg(all(
418 target_arch = "powerpc64",
419 portable_atomic_unstable_asm_experimental_arch,
420 any(
421 target_feature = "quadword-atomics",
422 portable_atomic_target_feature = "quadword-atomics",
423 all(
424 feature = "fallback",
425 not(portable_atomic_no_outline_atomics),
426 portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default
427 any(
428 all(
429 target_os = "linux",
430 any(
431 target_env = "gnu",
432 all(
433 any(target_env = "musl", target_env = "ohos"),
434 not(target_feature = "crt-static"),
435 ),
436 portable_atomic_outline_atomics,
437 ),
438 ),
439 target_os = "android",
440 target_os = "freebsd",
441 ),
442 not(any(miri, portable_atomic_sanitize_thread)),
443 ),
444 ),
445))]
446pub(crate) use self::powerpc64::{AtomicI128, AtomicU128};
447// s390x
448#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))]
449pub(crate) use self::s390x::{AtomicI128, AtomicU128};
450