1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7 all(
8 portable_atomic_no_atomic_load_store,
9 not(all(target_arch = "bpf", not(feature = "critical-section"))),
10 ),
11 target_arch = "avr",
12 target_arch = "msp430",
13)))]
14#[cfg_attr(
15 portable_atomic_no_cfg_target_has_atomic,
16 cfg(not(all(
17 any(
18 target_arch = "riscv32",
19 target_arch = "riscv64",
20 feature = "critical-section",
21 portable_atomic_unsafe_assume_single_core,
22 ),
23 portable_atomic_no_atomic_cas,
24 )))
25)]
26#[cfg_attr(
27 not(portable_atomic_no_cfg_target_has_atomic),
28 cfg(not(all(
29 any(
30 target_arch = "riscv32",
31 target_arch = "riscv64",
32 feature = "critical-section",
33 portable_atomic_unsafe_assume_single_core,
34 ),
35 not(target_has_atomic = "ptr"),
36 )))
37)]
38mod core_atomic;
39
40// AVR
41#[cfg(target_arch = "avr")]
42#[cfg(not(portable_atomic_no_asm))]
43#[cfg(not(feature = "critical-section"))]
44mod avr;
45
46// MSP430
47#[cfg(target_arch = "msp430")]
48pub(crate) mod msp430;
49
50// RISC-V without A-extension
51#[cfg(any(test, not(feature = "critical-section")))]
52#[cfg_attr(
53 portable_atomic_no_cfg_target_has_atomic,
54 cfg(any(
55 all(test, not(any(miri, portable_atomic_sanitize_thread))),
56 portable_atomic_no_atomic_cas,
57 ))
58)]
59#[cfg_attr(
60 not(portable_atomic_no_cfg_target_has_atomic),
61 cfg(any(
62 all(test, not(any(miri, portable_atomic_sanitize_thread))),
63 not(target_has_atomic = "ptr"),
64 ))
65)]
66#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
67mod riscv;
68
69// x86-specific optimizations
70// Miri and Sanitizer do not support inline assembly.
71#[cfg(all(
72 any(target_arch = "x86", target_arch = "x86_64"),
73 not(any(miri, portable_atomic_sanitize_thread)),
74 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
75))]
76mod x86;
77
78// 64-bit atomic implementations on 32-bit architectures
79#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
80mod atomic64;
81
82// 128-bit atomic implementations on 64-bit architectures
83#[cfg(any(
84 target_arch = "aarch64",
85 target_arch = "arm64ec",
86 target_arch = "powerpc64",
87 target_arch = "riscv64",
88 target_arch = "s390x",
89 target_arch = "x86_64",
90))]
91mod atomic128;
92
93// -----------------------------------------------------------------------------
94// Lock-based fallback implementations
95
96#[cfg(feature = "fallback")]
97#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
98#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
99#[cfg(any(
100 test,
101 not(any(
102 all(
103 target_arch = "aarch64",
104 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
105 ),
106 all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
107 all(
108 target_arch = "x86_64",
109 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
110 any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
111 ),
112 all(
113 target_arch = "riscv64",
114 not(any(miri, portable_atomic_sanitize_thread)),
115 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
116 any(target_feature = "zacas", portable_atomic_target_feature = "zacas"),
117 ),
118 all(
119 target_arch = "powerpc64",
120 portable_atomic_unstable_asm_experimental_arch,
121 any(
122 target_feature = "quadword-atomics",
123 portable_atomic_target_feature = "quadword-atomics",
124 ),
125 ),
126 all(target_arch = "s390x", not(portable_atomic_no_asm)),
127 ))
128))]
129mod fallback;
130
131// -----------------------------------------------------------------------------
132// Critical section based fallback implementations
133
134// On AVR, we always use critical section based fallback implementation.
135// AVR can be safely assumed to be single-core, so this is sound.
136// MSP430 as well.
137// See the module-level comments of interrupt module for more.
138#[cfg(any(
139 all(test, target_os = "none"),
140 portable_atomic_unsafe_assume_single_core,
141 feature = "critical-section",
142 target_arch = "avr",
143 target_arch = "msp430",
144))]
145#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
146#[cfg_attr(
147 not(portable_atomic_no_cfg_target_has_atomic),
148 cfg(any(test, not(target_has_atomic = "ptr")))
149)]
150#[cfg(any(
151 target_arch = "arm",
152 target_arch = "avr",
153 target_arch = "msp430",
154 target_arch = "riscv32",
155 target_arch = "riscv64",
156 target_arch = "xtensa",
157 feature = "critical-section",
158))]
159mod interrupt;
160
161// -----------------------------------------------------------------------------
162// Atomic float implementations
163
164#[cfg(feature = "float")]
165pub(crate) mod float;
166
167// -----------------------------------------------------------------------------
168
169// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
170#[cfg(not(any(
171 portable_atomic_no_atomic_load_store,
172 target_arch = "avr",
173 target_arch = "msp430",
174)))]
175#[cfg_attr(
176 portable_atomic_no_cfg_target_has_atomic,
177 cfg(not(all(
178 any(
179 target_arch = "riscv32",
180 target_arch = "riscv64",
181 feature = "critical-section",
182 portable_atomic_unsafe_assume_single_core,
183 ),
184 portable_atomic_no_atomic_cas,
185 )))
186)]
187#[cfg_attr(
188 not(portable_atomic_no_cfg_target_has_atomic),
189 cfg(not(all(
190 any(
191 target_arch = "riscv32",
192 target_arch = "riscv64",
193 feature = "critical-section",
194 portable_atomic_unsafe_assume_single_core,
195 ),
196 not(target_has_atomic = "ptr"),
197 )))
198)]
199items! {
200 pub(crate) use self::core_atomic::{
201 AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
202 AtomicUsize,
203 };
204 #[cfg_attr(
205 portable_atomic_no_cfg_target_has_atomic,
206 cfg(any(
207 not(portable_atomic_no_atomic_64),
208 not(any(target_pointer_width = "16", target_pointer_width = "32")),
209 ))
210 )]
211 #[cfg_attr(
212 not(portable_atomic_no_cfg_target_has_atomic),
213 cfg(any(
214 target_has_atomic = "64",
215 not(any(target_pointer_width = "16", target_pointer_width = "32")),
216 ))
217 )]
218 pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
219}
220// bpf & !(critical section) => core atomic
221#[cfg(all(
222 target_arch = "bpf",
223 portable_atomic_no_atomic_load_store,
224 not(feature = "critical-section"),
225))]
226pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
227
228// RISC-V without A-extension & !(assume single core | critical section)
229#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
230#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
231#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
232#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
233items! {
234 pub(crate) use self::riscv::{
235 AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
236 AtomicUsize,
237 };
238 #[cfg(target_arch = "riscv64")]
239 pub(crate) use self::riscv::{AtomicI64, AtomicU64};
240}
241
242// no core atomic CAS & (assume single core | critical section) => critical section based fallback
243#[cfg(any(
244 portable_atomic_unsafe_assume_single_core,
245 feature = "critical-section",
246 target_arch = "avr",
247 target_arch = "msp430",
248))]
249#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
250#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
251items! {
252 pub(crate) use self::interrupt::{
253 AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
254 };
255 #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
256 pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
257 #[cfg(any(
258 not(any(target_pointer_width = "16", target_pointer_width = "32")),
259 feature = "fallback",
260 ))]
261 pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
262 #[cfg(feature = "fallback")]
263 pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
264}
265
266// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
267#[cfg(feature = "fallback")]
268#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
269#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
270items! {
271 #[cfg(not(any(
272 all(
273 target_arch = "arm",
274 not(any(miri, portable_atomic_sanitize_thread)),
275 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
276 any(target_os = "linux", target_os = "android"),
277 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
278 not(portable_atomic_no_outline_atomics),
279 ),
280 all(
281 target_arch = "riscv32",
282 not(any(miri, portable_atomic_sanitize_thread)),
283 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
284 any(
285 target_feature = "zacas",
286 portable_atomic_target_feature = "zacas",
287 all(
288 feature = "fallback",
289 not(portable_atomic_no_outline_atomics),
290 any(target_os = "linux", target_os = "android"),
291 ),
292 ),
293 ),
294 )))]
295 #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
296 #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
297 pub(crate) use self::fallback::{AtomicI64, AtomicU64};
298 #[cfg(not(any(
299 all(
300 target_arch = "aarch64",
301 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
302 ),
303 all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
304 all(
305 target_arch = "x86_64",
306 not(all(
307 any(miri, portable_atomic_sanitize_thread),
308 portable_atomic_no_cmpxchg16b_intrinsic,
309 )),
310 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
311 any(
312 target_feature = "cmpxchg16b",
313 portable_atomic_target_feature = "cmpxchg16b",
314 all(
315 feature = "fallback",
316 not(portable_atomic_no_outline_atomics),
317 not(any(target_env = "sgx", miri)),
318 ),
319 ),
320 ),
321 all(
322 target_arch = "riscv64",
323 not(any(miri, portable_atomic_sanitize_thread)),
324 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
325 any(
326 target_feature = "zacas",
327 portable_atomic_target_feature = "zacas",
328 all(
329 feature = "fallback",
330 not(portable_atomic_no_outline_atomics),
331 any(target_os = "linux", target_os = "android"),
332 ),
333 ),
334 ),
335 all(
336 target_arch = "powerpc64",
337 portable_atomic_unstable_asm_experimental_arch,
338 any(
339 target_feature = "quadword-atomics",
340 portable_atomic_target_feature = "quadword-atomics",
341 all(
342 feature = "fallback",
343 not(portable_atomic_no_outline_atomics),
344 any(
345 all(
346 target_os = "linux",
347 any(
348 all(
349 target_env = "gnu",
350 any(target_endian = "little", not(target_feature = "crt-static")),
351 ),
352 all(
353 any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
354 not(target_feature = "crt-static"),
355 ),
356 portable_atomic_outline_atomics,
357 ),
358 ),
359 target_os = "android",
360 target_os = "freebsd",
361 target_os = "openbsd",
362 all(
363 target_os = "aix",
364 not(portable_atomic_pre_llvm_20),
365 portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
366 ),
367 ),
368 not(any(miri, portable_atomic_sanitize_thread)),
369 ),
370 ),
371 ),
372 all(target_arch = "s390x", not(portable_atomic_no_asm)),
373 )))]
374 pub(crate) use self::fallback::{AtomicI128, AtomicU128};
375}
376
377// 64-bit atomics (platform-specific)
378// pre-v6 Arm Linux
379#[cfg(feature = "fallback")]
380#[cfg(all(
381 target_arch = "arm",
382 not(any(miri, portable_atomic_sanitize_thread)),
383 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
384 any(target_os = "linux", target_os = "android"),
385 not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
386 not(portable_atomic_no_outline_atomics),
387))]
388#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
389#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
390pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
391// riscv32 & (zacas | outline-atomics)
392#[cfg(all(
393 target_arch = "riscv32",
394 not(any(miri, portable_atomic_sanitize_thread)),
395 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
396 any(
397 target_feature = "zacas",
398 portable_atomic_target_feature = "zacas",
399 all(
400 feature = "fallback",
401 not(portable_atomic_no_outline_atomics),
402 any(target_os = "linux", target_os = "android"),
403 ),
404 ),
405))]
406pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
407
408// 128-bit atomics (platform-specific)
409// AArch64
410#[cfg(any(
411 all(target_arch = "aarch64", any(not(portable_atomic_no_asm), portable_atomic_unstable_asm)),
412 all(target_arch = "arm64ec", not(portable_atomic_no_asm))
413))]
414pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
415// x86_64 & (cmpxchg16b | outline-atomics)
416#[cfg(all(
417 target_arch = "x86_64",
418 not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
419 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
420 any(
421 target_feature = "cmpxchg16b",
422 portable_atomic_target_feature = "cmpxchg16b",
423 all(
424 feature = "fallback",
425 not(portable_atomic_no_outline_atomics),
426 not(any(target_env = "sgx", miri)),
427 ),
428 ),
429))]
430pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
431// riscv64 & (zacas | outline-atomics)
432#[cfg(all(
433 target_arch = "riscv64",
434 not(any(miri, portable_atomic_sanitize_thread)),
435 any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
436 any(
437 target_feature = "zacas",
438 portable_atomic_target_feature = "zacas",
439 all(
440 feature = "fallback",
441 not(portable_atomic_no_outline_atomics),
442 any(target_os = "linux", target_os = "android"),
443 ),
444 ),
445))]
446pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
447// powerpc64 & (pwr8 | outline-atomics)
448#[cfg(all(
449 target_arch = "powerpc64",
450 portable_atomic_unstable_asm_experimental_arch,
451 any(
452 target_feature = "quadword-atomics",
453 portable_atomic_target_feature = "quadword-atomics",
454 all(
455 feature = "fallback",
456 not(portable_atomic_no_outline_atomics),
457 any(
458 all(
459 target_os = "linux",
460 any(
461 all(
462 target_env = "gnu",
463 any(target_endian = "little", not(target_feature = "crt-static")),
464 ),
465 all(
466 any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
467 not(target_feature = "crt-static"),
468 ),
469 portable_atomic_outline_atomics,
470 ),
471 ),
472 target_os = "android",
473 target_os = "freebsd",
474 target_os = "openbsd",
475 all(
476 target_os = "aix",
477 not(portable_atomic_pre_llvm_20),
478 portable_atomic_outline_atomics, // TODO(aix): currently disabled by default
479 ),
480 ),
481 not(any(miri, portable_atomic_sanitize_thread)),
482 ),
483 ),
484))]
485pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
486// s390x
487#[cfg(all(target_arch = "s390x", not(portable_atomic_no_asm)))]
488pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
489