1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7 all(
8 portable_atomic_no_atomic_load_store,
9 not(all(target_arch = "bpf", not(feature = "critical-section"))),
10 ),
11 target_arch = "avr",
12 target_arch = "msp430",
13)))]
14#[cfg_attr(
15 portable_atomic_no_cfg_target_has_atomic,
16 cfg(not(all(
17 any(
18 target_arch = "riscv32",
19 target_arch = "riscv64",
20 feature = "critical-section",
21 portable_atomic_unsafe_assume_single_core,
22 ),
23 portable_atomic_no_atomic_cas,
24 )))
25)]
26#[cfg_attr(
27 not(portable_atomic_no_cfg_target_has_atomic),
28 cfg(not(all(
29 any(
30 target_arch = "riscv32",
31 target_arch = "riscv64",
32 feature = "critical-section",
33 portable_atomic_unsafe_assume_single_core,
34 ),
35 not(target_has_atomic = "ptr"),
36 )))
37)]
38mod core_atomic;
39
40// AVR
41#[cfg(target_arch = "avr")]
42#[cfg(not(portable_atomic_no_asm))]
43#[cfg(not(feature = "critical-section"))]
44mod avr;
45
46// MSP430
47#[cfg(target_arch = "msp430")]
48pub(crate) mod msp430;
49
50// RISC-V without A-extension
51#[cfg(any(test, not(feature = "critical-section")))]
52#[cfg_attr(
53 portable_atomic_no_cfg_target_has_atomic,
54 cfg(any(
55 all(test, not(any(miri, portable_atomic_sanitize_thread))),
56 portable_atomic_no_atomic_cas,
57 ))
58)]
59#[cfg_attr(
60 not(portable_atomic_no_cfg_target_has_atomic),
61 cfg(any(
62 all(test, not(any(miri, portable_atomic_sanitize_thread))),
63 not(target_has_atomic = "ptr"),
64 ))
65)]
66#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
67mod riscv;
68
69// x86-specific optimizations
70// Miri and Sanitizer do not support inline assembly.
71#[cfg(all(
72 any(target_arch = "x86", target_arch = "x86_64"),
73 not(any(miri, portable_atomic_sanitize_thread)),
74 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
75))]
76mod x86;
77
78// 64-bit atomic implementations on 32-bit architectures
79#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
80mod atomic64;
81
82// 128-bit atomic implementations on 64-bit architectures
83#[cfg(any(
84 target_arch = "aarch64",
85 target_arch = "arm64ec",
86 target_arch = "powerpc64",
87 target_arch = "riscv64",
88 target_arch = "s390x",
89 target_arch = "x86_64",
90))]
91mod atomic128;
92
93// -----------------------------------------------------------------------------
94// Lock-based fallback implementations
95
96#[cfg(feature = "fallback")]
97#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
98#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
99#[cfg(any(
100 test,
101 not(any(
102 all(
103 target_arch = "aarch64",
104 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
105 ),
106 all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
107 all(
108 target_arch = "x86_64",
109 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
110 any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
111 ),
112 all(
113 target_arch = "riscv64",
114 not(portable_atomic_no_asm),
115 any(
116 target_feature = "experimental-zacas",
117 portable_atomic_target_feature = "experimental-zacas",
118 ),
119 ),
120 all(
121 target_arch = "powerpc64",
122 portable_atomic_unstable_asm_experimental_arch,
123 any(
124 target_feature = "quadword-atomics",
125 portable_atomic_target_feature = "quadword-atomics",
126 ),
127 ),
128 all(target_arch = "s390x", not(portable_atomic_no_asm)),
129 ))
130))]
131mod fallback;
132
133// -----------------------------------------------------------------------------
134// Critical section based fallback implementations
135
136// On AVR, we always use critical section based fallback implementation.
137// AVR can be safely assumed to be single-core, so this is sound.
138// https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074
139// MSP430 as well.
140#[cfg(any(
141 all(test, target_os = "none"),
142 portable_atomic_unsafe_assume_single_core,
143 feature = "critical-section",
144 target_arch = "avr",
145 target_arch = "msp430",
146))]
147#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
148#[cfg_attr(
149 not(portable_atomic_no_cfg_target_has_atomic),
150 cfg(any(test, not(target_has_atomic = "ptr")))
151)]
152#[cfg(any(
153 target_arch = "arm",
154 target_arch = "avr",
155 target_arch = "msp430",
156 target_arch = "riscv32",
157 target_arch = "riscv64",
158 target_arch = "xtensa",
159 feature = "critical-section",
160))]
161mod interrupt;
162
163// -----------------------------------------------------------------------------
164// Atomic float implementations
165
166#[cfg(feature = "float")]
167#[allow(clippy::float_arithmetic)]
168pub(crate) mod float;
169
170// -----------------------------------------------------------------------------
171
172// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
173#[cfg(not(any(
174 portable_atomic_no_atomic_load_store,
175 target_arch = "avr",
176 target_arch = "msp430",
177)))]
178#[cfg_attr(
179 portable_atomic_no_cfg_target_has_atomic,
180 cfg(not(all(
181 any(
182 target_arch = "riscv32",
183 target_arch = "riscv64",
184 feature = "critical-section",
185 portable_atomic_unsafe_assume_single_core,
186 ),
187 portable_atomic_no_atomic_cas,
188 )))
189)]
190#[cfg_attr(
191 not(portable_atomic_no_cfg_target_has_atomic),
192 cfg(not(all(
193 any(
194 target_arch = "riscv32",
195 target_arch = "riscv64",
196 feature = "critical-section",
197 portable_atomic_unsafe_assume_single_core,
198 ),
199 not(target_has_atomic = "ptr"),
200 )))
201)]
202items! {
203 pub(crate) use self::core_atomic::{
204 AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
205 AtomicUsize,
206 };
207 #[cfg_attr(
208 portable_atomic_no_cfg_target_has_atomic,
209 cfg(any(
210 not(portable_atomic_no_atomic_64),
211 not(any(target_pointer_width = "16", target_pointer_width = "32")),
212 ))
213 )]
214 #[cfg_attr(
215 not(portable_atomic_no_cfg_target_has_atomic),
216 cfg(any(
217 target_has_atomic = "64",
218 not(any(target_pointer_width = "16", target_pointer_width = "32")),
219 ))
220 )]
221 pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
222}
223// bpf & !(critical section) => core atomic
224#[cfg(all(
225 target_arch = "bpf",
226 portable_atomic_no_atomic_load_store,
227 not(feature = "critical-section"),
228))]
229pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
230
231// RISC-V without A-extension & !(assume single core | critical section)
232#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
233#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
234#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
235#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
236items! {
237 pub(crate) use self::riscv::{
238 AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
239 AtomicUsize,
240 };
241 #[cfg(target_arch = "riscv64")]
242 pub(crate) use self::riscv::{AtomicI64, AtomicU64};
243}
244
245// no core atomic CAS & (assume single core | critical section) => critical section based fallback
246#[cfg(any(
247 portable_atomic_unsafe_assume_single_core,
248 feature = "critical-section",
249 target_arch = "avr",
250 target_arch = "msp430",
251))]
252#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
253#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
254items! {
255 pub(crate) use self::interrupt::{
256 AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
257 };
258 #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
259 pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
260 #[cfg(any(
261 not(any(target_pointer_width = "16", target_pointer_width = "32")),
262 feature = "fallback",
263 ))]
264 pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
265 #[cfg(feature = "fallback")]
266 pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
267}
268
269// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
270#[cfg(feature = "fallback")]
271#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
272#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
273items! {
274 #[cfg(not(any(
275 all(
276 target_arch = "arm",
277 not(any(miri, portable_atomic_sanitize_thread)),
278 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
279 any(target_os = "linux", target_os = "android"),
280 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
281 not(portable_atomic_no_outline_atomics),
282 ),
283 all(
284 target_arch = "riscv32",
285 not(any(miri, portable_atomic_sanitize_thread)),
286 not(portable_atomic_no_asm),
287 any(
288 target_feature = "experimental-zacas",
289 portable_atomic_target_feature = "experimental-zacas",
290 all(
291 feature = "fallback",
292 not(portable_atomic_no_outline_atomics),
293 any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
294 any(target_os = "linux", target_os = "android"),
295 ),
296 ),
297 ),
298 )))]
299 #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
300 #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
301 pub(crate) use self::fallback::{AtomicI64, AtomicU64};
302 #[cfg(not(any(
303 all(
304 target_arch = "aarch64",
305 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
306 ),
307 all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
308 all(
309 target_arch = "x86_64",
310 not(all(
311 any(miri, portable_atomic_sanitize_thread),
312 portable_atomic_no_cmpxchg16b_intrinsic,
313 )),
314 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
315 any(
316 target_feature = "cmpxchg16b",
317 portable_atomic_target_feature = "cmpxchg16b",
318 all(
319 feature = "fallback",
320 not(portable_atomic_no_outline_atomics),
321 not(any(target_env = "sgx", miri)),
322 ),
323 ),
324 ),
325 all(
326 target_arch = "riscv64",
327 not(portable_atomic_no_asm),
328 any(
329 target_feature = "experimental-zacas",
330 portable_atomic_target_feature = "experimental-zacas",
331 all(
332 feature = "fallback",
333 not(portable_atomic_no_outline_atomics),
334 any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
335 any(target_os = "linux", target_os = "android"),
336 not(any(miri, portable_atomic_sanitize_thread)),
337 ),
338 ),
339 ),
340 all(
341 target_arch = "powerpc64",
342 portable_atomic_unstable_asm_experimental_arch,
343 any(
344 target_feature = "quadword-atomics",
345 portable_atomic_target_feature = "quadword-atomics",
346 all(
347 feature = "fallback",
348 not(portable_atomic_no_outline_atomics),
349 any(
350 all(
351 target_os = "linux",
352 any(
353 all(
354 target_env = "gnu",
355 any(target_endian = "little", not(target_feature = "crt-static")),
356 ),
357 all(
358 any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
359 not(target_feature = "crt-static"),
360 ),
361 portable_atomic_outline_atomics,
362 ),
363 ),
364 target_os = "android",
365 target_os = "freebsd",
366 target_os = "openbsd",
367 ),
368 not(any(miri, portable_atomic_sanitize_thread)),
369 ),
370 ),
371 ),
372 all(target_arch = "s390x", not(portable_atomic_no_asm)),
373 )))]
374 pub(crate) use self::fallback::{AtomicI128, AtomicU128};
375}
376
377// 64-bit atomics (platform-specific)
378// pre-v6 Arm Linux
379#[cfg(feature = "fallback")]
380#[cfg(all(
381 target_arch = "arm",
382 not(any(miri, portable_atomic_sanitize_thread)),
383 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
384 any(target_os = "linux", target_os = "android"),
385 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
386 not(portable_atomic_no_outline_atomics),
387))]
388#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
389#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
390pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
391// riscv32 & (zacas | outline-atomics)
392#[cfg(all(
393 target_arch = "riscv32",
394 not(any(miri, portable_atomic_sanitize_thread)),
395 not(portable_atomic_no_asm),
396 any(
397 target_feature = "experimental-zacas",
398 portable_atomic_target_feature = "experimental-zacas",
399 all(
400 feature = "fallback",
401 not(portable_atomic_no_outline_atomics),
402 any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
403 any(target_os = "linux", target_os = "android"),
404 ),
405 ),
406))]
407pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
408
409// 128-bit atomics (platform-specific)
410// AArch64
411#[cfg(any(
412 all(target_arch = "aarch64", any(not(portable_atomic_no_asm), portable_atomic_unstable_asm)),
413 all(target_arch = "arm64ec", not(portable_atomic_no_asm))
414))]
415pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
416// x86_64 & (cmpxchg16b | outline-atomics)
417#[cfg(all(
418 target_arch = "x86_64",
419 not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
420 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
421 any(
422 target_feature = "cmpxchg16b",
423 portable_atomic_target_feature = "cmpxchg16b",
424 all(
425 feature = "fallback",
426 not(portable_atomic_no_outline_atomics),
427 not(any(target_env = "sgx", miri)),
428 ),
429 ),
430))]
431pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
432// riscv64 & (zacas | outline-atomics)
433#[cfg(all(
434 target_arch = "riscv64",
435 not(portable_atomic_no_asm),
436 any(
437 target_feature = "experimental-zacas",
438 portable_atomic_target_feature = "experimental-zacas",
439 all(
440 feature = "fallback",
441 not(portable_atomic_no_outline_atomics),
442 any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
443 any(target_os = "linux", target_os = "android"),
444 not(any(miri, portable_atomic_sanitize_thread)),
445 ),
446 ),
447))]
448pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
449// powerpc64 & (pwr8 | outline-atomics)
450#[cfg(all(
451 target_arch = "powerpc64",
452 portable_atomic_unstable_asm_experimental_arch,
453 any(
454 target_feature = "quadword-atomics",
455 portable_atomic_target_feature = "quadword-atomics",
456 all(
457 feature = "fallback",
458 not(portable_atomic_no_outline_atomics),
459 any(
460 all(
461 target_os = "linux",
462 any(
463 all(
464 target_env = "gnu",
465 any(target_endian = "little", not(target_feature = "crt-static")),
466 ),
467 all(
468 any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
469 not(target_feature = "crt-static"),
470 ),
471 portable_atomic_outline_atomics,
472 ),
473 ),
474 target_os = "android",
475 target_os = "freebsd",
476 target_os = "openbsd",
477 ),
478 not(any(miri, portable_atomic_sanitize_thread)),
479 ),
480 ),
481))]
482pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
483// s390x
484#[cfg(all(target_arch = "s390x", not(portable_atomic_no_asm)))]
485pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
486