1 | use crate::cell::Cell; |
2 | use crate::ops::{Deref, DerefMut}; |
3 | |
4 | /// Pads and aligns a value to the length of a cache line. |
5 | #[derive (Clone, Copy, Default, Hash, PartialEq, Eq)] |
6 | // Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache |
7 | // lines at a time, so we have to align to 128 bytes rather than 64. |
8 | // |
9 | // Sources: |
10 | // - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf |
11 | // - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107 |
12 | // |
13 | // ARM's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size. |
14 | // |
15 | // Sources: |
16 | // - https://www.mono-project.com/news/2016/09/12/arm64-icache/ |
17 | // |
18 | // powerpc64 has 128-byte cache line size. |
19 | // |
20 | // Sources: |
21 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9 |
22 | #[cfg_attr ( |
23 | any(target_arch = "x86_64" , target_arch = "aarch64" , target_arch = "powerpc64" ,), |
24 | repr(align(128)) |
25 | )] |
26 | // arm, mips, mips64, and riscv64 have 32-byte cache line size. |
27 | // |
28 | // Sources: |
29 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7 |
30 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7 |
31 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7 |
32 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9 |
33 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7 |
34 | #[cfg_attr ( |
35 | any( |
36 | target_arch = "arm" , |
37 | target_arch = "mips" , |
38 | target_arch = "mips32r6" , |
39 | target_arch = "mips64" , |
40 | target_arch = "mips64r6" , |
41 | target_arch = "riscv64" , |
42 | ), |
43 | repr(align(32)) |
44 | )] |
45 | // s390x has 256-byte cache line size. |
46 | // |
47 | // Sources: |
48 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7 |
49 | #[cfg_attr (target_arch = "s390x" , repr(align(256)))] |
50 | // x86 and wasm have 64-byte cache line size. |
51 | // |
52 | // Sources: |
53 | // - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9 |
54 | // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7 |
55 | // |
56 | // All others are assumed to have 64-byte cache line size. |
57 | #[cfg_attr ( |
58 | not(any( |
59 | target_arch = "x86_64" , |
60 | target_arch = "aarch64" , |
61 | target_arch = "powerpc64" , |
62 | target_arch = "arm" , |
63 | target_arch = "mips" , |
64 | target_arch = "mips32r6" , |
65 | target_arch = "mips64" , |
66 | target_arch = "mips64r6" , |
67 | target_arch = "riscv64" , |
68 | target_arch = "s390x" , |
69 | )), |
70 | repr(align(64)) |
71 | )] |
72 | pub struct CachePadded<T> { |
73 | value: T, |
74 | } |
75 | |
76 | impl<T> CachePadded<T> { |
77 | /// Pads and aligns a value to the length of a cache line. |
78 | pub fn new(value: T) -> CachePadded<T> { |
79 | CachePadded::<T> { value } |
80 | } |
81 | } |
82 | |
83 | impl<T> Deref for CachePadded<T> { |
84 | type Target = T; |
85 | |
86 | fn deref(&self) -> &T { |
87 | &self.value |
88 | } |
89 | } |
90 | |
91 | impl<T> DerefMut for CachePadded<T> { |
92 | fn deref_mut(&mut self) -> &mut T { |
93 | &mut self.value |
94 | } |
95 | } |
96 | |
97 | const SPIN_LIMIT: u32 = 6; |
98 | |
99 | /// Performs quadratic backoff in spin loops. |
100 | pub struct Backoff { |
101 | step: Cell<u32>, |
102 | } |
103 | |
104 | impl Backoff { |
105 | /// Creates a new `Backoff`. |
106 | pub fn new() -> Self { |
107 | Backoff { step: Cell::new(0) } |
108 | } |
109 | |
110 | /// Backs off using lightweight spinning. |
111 | /// |
112 | /// This method should be used for retrying an operation because another thread made |
113 | /// progress. i.e. on CAS failure. |
114 | #[inline ] |
115 | pub fn spin_light(&self) { |
116 | let step = self.step.get().min(SPIN_LIMIT); |
117 | for _ in 0..step.pow(2) { |
118 | crate::hint::spin_loop(); |
119 | } |
120 | |
121 | self.step.set(self.step.get() + 1); |
122 | } |
123 | |
124 | /// Backs off using heavyweight spinning. |
125 | /// |
126 | /// This method should be used in blocking loops where parking the thread is not an option. |
127 | #[inline ] |
128 | pub fn spin_heavy(&self) { |
129 | if self.step.get() <= SPIN_LIMIT { |
130 | for _ in 0..self.step.get().pow(2) { |
131 | crate::hint::spin_loop() |
132 | } |
133 | } else { |
134 | crate::thread::yield_now(); |
135 | } |
136 | |
137 | self.step.set(self.step.get() + 1); |
138 | } |
139 | } |
140 | |