1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3/*
4Run-time CPU feature detection on x86_64 by using CPUID.
5
6Adapted from https://github.com/rust-lang/stdarch.
7*/
8
9#![cfg_attr(portable_atomic_sanitize_thread, allow(dead_code))]
10
11// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932
12// SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
13#[cfg(any(target_env = "sgx", miri))]
14compile_error!("internal error: this module is not supported on this environment");
15
16include!("common.rs");
17
18#[cfg(not(portable_atomic_no_asm))]
19use core::arch::asm;
20use core::arch::x86_64::CpuidResult;
21
22// Workaround for https://github.com/rust-lang/rust/issues/101346
23// It is not clear if our use cases are affected, but we implement this just in case.
24//
25// Refs:
26// - https://www.felixcloutier.com/x86/cpuid
27// - https://en.wikipedia.org/wiki/CPUID
28// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs
29#[cfg(not(target_env = "sgx"))]
30fn __cpuid(leaf: u32) -> CpuidResult {
31 let eax: u32;
32 let mut ebx: u32;
33 let ecx: u32;
34 let edx: u32;
35 // SAFETY: Calling `__cpuid`` is safe on all x86_64 CPUs except for SGX,
36 // which doesn't support `cpuid`.
37 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L109
38 unsafe {
39 asm!(
40 "mov {ebx_tmp:r}, rbx", // save rbx which is reserved by LLVM
41 "cpuid",
42 "xchg {ebx_tmp:r}, rbx", // restore rbx
43 ebx_tmp = out(reg) ebx,
44 inout("eax") leaf => eax,
45 inout("ecx") 0 => ecx,
46 out("edx") edx,
47 options(nostack, preserves_flags),
48 );
49 }
50 CpuidResult { eax, ebx, ecx, edx }
51}
52
53// https://en.wikipedia.org/wiki/CPUID
54const _VENDOR_ID_INTEL: [u32; 3] = _vender(b"GenuineIntel"); // Intel
55const _VENDOR_ID_INTEL2: [u32; 3] = _vender(b"GenuineIotel"); // Intel https://github.com/InstLatx64/InstLatx64/commit/8fdd319884c67d2c6ec1ca0c595b42c1c4b8d803
56const _VENDOR_ID_AMD: [u32; 3] = _vender(b"AuthenticAMD"); // AMD
57const _VENDOR_ID_CENTAUR: [u32; 3] = _vender(b"CentaurHauls"); // Centaur/VIA/Zhaoxin
58const _VENDOR_ID_ZHAOXIN: [u32; 3] = _vender(b" Shanghai "); // Zhaoxin
59const fn _vender(b: &[u8; 12]) -> [u32; 3] {
60 [
61 u32::from_ne_bytes([b[0], b[1], b[2], b[3]]),
62 u32::from_ne_bytes([b[4], b[5], b[6], b[7]]),
63 u32::from_ne_bytes([b[8], b[9], b[10], b[11]]),
64 ]
65}
66fn _vendor_id() -> [u32; 3] {
67 let CpuidResult { ebx: u32, ecx: u32, edx: u32, .. } = __cpuid(leaf:0);
68 [ebx, edx, ecx]
69}
70fn _vendor_has_vmovdqa_atomic(vendor_id: [u32; 3], family: u32) -> bool {
71 // VMOVDQA is atomic on Intel, AMD, and Zhaoxin CPUs with AVX.
72 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.
73 vendor_id == _VENDOR_ID_INTEL
74 || vendor_id == _VENDOR_ID_INTEL2
75 || vendor_id == _VENDOR_ID_AMD
76 || vendor_id == _VENDOR_ID_ZHAOXIN
77 || vendor_id == _VENDOR_ID_CENTAUR && family > 6
78}
79
80#[cold]
81fn _detect(info: &mut CpuInfo) {
82 let CpuidResult {
83 #[cfg(target_feature = "sse")]
84 eax: proc_info_eax,
85 ecx: proc_info_ecx,
86 ..
87 } = __cpuid(1);
88
89 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L111
90 if test(proc_info_ecx, 13) {
91 info.set(CpuInfo::HAS_CMPXCHG16B);
92 }
93
94 // We only use VMOVDQA when SSE is enabled. See atomic_load_vmovdqa() in atomic128/x86_64.rs for more.
95 #[cfg(target_feature = "sse")]
96 {
97 use core::arch::x86_64::_xgetbv;
98
99 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L131-L224
100 let cpu_xsave = test(proc_info_ecx, 26);
101 if cpu_xsave {
102 let cpu_osxsave = test(proc_info_ecx, 27);
103 if cpu_osxsave {
104 // SAFETY: Calling `_xgetbv`` is safe because the CPU has `xsave` support
105 // and OS has set `osxsave`.
106 let xcr0 = unsafe { _xgetbv(0) };
107 let os_avx_support = xcr0 & 6 == 6;
108 if os_avx_support && test(proc_info_ecx, 28) {
109 let vendor_id = _vendor_id();
110 let family = (proc_info_eax >> 8) & 0x0F;
111 if _vendor_has_vmovdqa_atomic(vendor_id, family) {
112 info.set(CpuInfo::HAS_VMOVDQA_ATOMIC);
113 }
114 }
115 }
116 }
117 }
118}
119
120#[allow(
121 clippy::alloc_instead_of_core,
122 clippy::std_instead_of_alloc,
123 clippy::std_instead_of_core,
124 clippy::undocumented_unsafe_blocks,
125 clippy::wildcard_imports
126)]
127#[cfg(test)]
128mod tests {
129 use std::{
130 io::{self, Write},
131 mem, str,
132 };
133
134 use super::*;
135
136 #[test]
137 #[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)]
138 fn test_cpuid() {
139 assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().has_cmpxchg16b());
140 let vendor_id = _vendor_id();
141 {
142 let stdout = io::stderr();
143 let mut stdout = stdout.lock();
144 let _ = writeln!(
145 stdout,
146 "\n vendor_id: {} (ebx: {:x}, edx: {:x}, ecx: {:x})",
147 str::from_utf8(&unsafe { mem::transmute::<[u32; 3], [u8; 12]>(vendor_id) })
148 .unwrap(),
149 vendor_id[0],
150 vendor_id[1],
151 vendor_id[2],
152 );
153 }
154 let CpuidResult { eax: proc_info_eax, .. } = __cpuid(1);
155 let family = (proc_info_eax >> 8) & 0x0F;
156 if _vendor_has_vmovdqa_atomic(vendor_id, family) {
157 assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic());
158 } else {
159 assert!(!detect().has_vmovdqa_atomic());
160 }
161 assert_eq!(
162 unsafe { mem::transmute::<[u32; 3], [u8; 12]>(_VENDOR_ID_INTEL) },
163 *b"GenuineIntel"
164 );
165 }
166}
167