1//! x86/x86-64 CPU feature detection support.
2//!
3//! Portable, `no_std`-friendly implementation that relies on the x86 `CPUID`
4//! instruction for feature detection.
5
6/// Evaluate the given `$body` expression any of the supplied target features
7/// are not enabled. Otherwise returns true.
8///
9/// The `$body` expression is not evaluated on SGX targets, and returns false
10/// on these targets unless *all* supplied target features are enabled.
11#[macro_export]
12#[doc(hidden)]
13macro_rules! __unless_target_features {
14 ($($tf:tt),+ => $body:expr ) => {{
15 #[cfg(not(all($(target_feature=$tf,)*)))]
16 {
17 #[cfg(not(any(target_env = "sgx", target_os = "none", target_os = "uefi")))]
18 $body
19
20 // CPUID is not available on SGX. Freestanding and UEFI targets
21 // do not support SIMD features with default compilation flags.
22 #[cfg(any(target_env = "sgx", target_os = "none", target_os = "uefi"))]
23 false
24 }
25
26 #[cfg(all($(target_feature=$tf,)*))]
27 true
28 }};
29}
30
31/// Use CPUID to detect the presence of all supplied target features.
32#[macro_export]
33#[doc(hidden)]
34macro_rules! __detect_target_features {
35 ($($tf:tt),+) => {{
36 #[cfg(target_arch = "x86")]
37 use core::arch::x86::{__cpuid, __cpuid_count, CpuidResult};
38 #[cfg(target_arch = "x86_64")]
39 use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult};
40
41 // These wrappers are workarounds around
42 // https://github.com/rust-lang/rust/issues/101346
43 //
44 // DO NOT remove it until MSRV is bumped to a version
45 // with the issue fix (at least 1.64).
46 #[inline(never)]
47 unsafe fn cpuid(leaf: u32) -> CpuidResult {
48 __cpuid(leaf)
49 }
50
51 #[inline(never)]
52 unsafe fn cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult {
53 __cpuid_count(leaf, sub_leaf)
54 }
55
56 let cr = unsafe {
57 [cpuid(1), cpuid_count(7, 0)]
58 };
59
60 $($crate::check!(cr, $tf) & )+ true
61 }};
62}
63
64/// Check that OS supports required SIMD registers
65#[macro_export]
66#[doc(hidden)]
67macro_rules! __xgetbv {
68 ($cr:expr, $mask:expr) => {{
69 #[cfg(target_arch = "x86")]
70 use core::arch::x86 as arch;
71 #[cfg(target_arch = "x86_64")]
72 use core::arch::x86_64 as arch;
73
74 // Check bits 26 and 27
75 let xmask = 0b11 << 26;
76 let xsave = $cr[0].ecx & xmask == xmask;
77 if xsave {
78 let xcr0 = unsafe { arch::_xgetbv(arch::_XCR_XFEATURE_ENABLED_MASK) };
79 (xcr0 & $mask) == $mask
80 } else {
81 false
82 }
83 }};
84}
85
86macro_rules! __expand_check_macro {
87 ($(($name:tt, $reg_cap:tt $(, $i:expr, $reg:ident, $offset:expr)*)),* $(,)?) => {
88 #[macro_export]
89 #[doc(hidden)]
90 macro_rules! check {
91 $(
92 ($cr:expr, $name) => {{
93 // Register bits are listed here:
94 // https://wiki.osdev.org/CPU_Registers_x86#Extended_Control_Registers
95 let reg_cap = match $reg_cap {
96 // Bit 1
97 "xmm" => $crate::__xgetbv!($cr, 0b10),
98 // Bits 1 and 2
99 "ymm" => $crate::__xgetbv!($cr, 0b110),
100 // Bits 1, 2, 5, 6, and 7
101 "zmm" => $crate::__xgetbv!($cr, 0b1110_0110),
102 _ => true,
103 };
104 reg_cap
105 $(
106 & ($cr[$i].$reg & (1 << $offset) != 0)
107 )*
108 }};
109 )*
110 }
111 };
112}
113
114__expand_check_macro! {
115 ("sse3", "", 0, ecx, 0),
116 ("pclmulqdq", "", 0, ecx, 1),
117 ("ssse3", "", 0, ecx, 9),
118 ("fma", "ymm", 0, ecx, 12, 0, ecx, 28),
119 ("sse4.1", "", 0, ecx, 19),
120 ("sse4.2", "", 0, ecx, 20),
121 ("popcnt", "", 0, ecx, 23),
122 ("aes", "", 0, ecx, 25),
123 ("avx", "xmm", 0, ecx, 28),
124 ("rdrand", "", 0, ecx, 30),
125
126 ("mmx", "", 0, edx, 23),
127 ("sse", "", 0, edx, 25),
128 ("sse2", "", 0, edx, 26),
129
130 ("sgx", "", 1, ebx, 2),
131 ("bmi1", "", 1, ebx, 3),
132 ("bmi2", "", 1, ebx, 8),
133 ("avx2", "ymm", 1, ebx, 5, 0, ecx, 28),
134 ("avx512f", "zmm", 1, ebx, 16),
135 ("avx512dq", "zmm", 1, ebx, 17),
136 ("rdseed", "", 1, ebx, 18),
137 ("adx", "", 1, ebx, 19),
138 ("avx512ifma", "zmm", 1, ebx, 21),
139 ("avx512pf", "zmm", 1, ebx, 26),
140 ("avx512er", "zmm", 1, ebx, 27),
141 ("avx512cd", "zmm", 1, ebx, 28),
142 ("sha", "", 1, ebx, 29),
143 ("avx512bw", "zmm", 1, ebx, 30),
144 ("avx512vl", "zmm", 1, ebx, 31),
145 ("avx512vbmi", "zmm", 1, ecx, 1),
146 ("avx512vbmi2", "zmm", 1, ecx, 6),
147 ("gfni", "zmm", 1, ecx, 8),
148 ("vaes", "zmm", 1, ecx, 9),
149 ("vpclmulqdq", "zmm", 1, ecx, 10),
150 ("avx512bitalg", "zmm", 1, ecx, 12),
151 ("avx512vpopcntdq", "zmm", 1, ecx, 14),
152}
153