1#if __has_include(<sys/auxv.h>)
2#include <sys/auxv.h>
3#define HAVE_SYS_AUXV_H
4#endif
5
6
7
8static void __init_cpu_features_constructor(unsigned long hwcap,
9 const __ifunc_arg_t *arg) {
10#define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F
11#define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
12#define extractBits(val, start, number) \
13 (val & ((1ULL << number) - 1ULL) << start) >> start
14 unsigned long hwcap2 = 0;
15 if (hwcap & _IFUNC_ARG_HWCAP)
16 hwcap2 = arg->_hwcap2;
17 if (hwcap & HWCAP_CRC32)
18 setCPUFeature(FEAT_CRC);
19 if (hwcap & HWCAP_PMULL)
20 setCPUFeature(FEAT_PMULL);
21 if (hwcap & HWCAP_FLAGM)
22 setCPUFeature(FEAT_FLAGM);
23 if (hwcap2 & HWCAP2_FLAGM2) {
24 setCPUFeature(FEAT_FLAGM);
25 setCPUFeature(FEAT_FLAGM2);
26 }
27 if (hwcap & HWCAP_SM3 && hwcap & HWCAP_SM4)
28 setCPUFeature(FEAT_SM4);
29 if (hwcap & HWCAP_ASIMDDP)
30 setCPUFeature(FEAT_DOTPROD);
31 if (hwcap & HWCAP_ASIMDFHM)
32 setCPUFeature(FEAT_FP16FML);
33 if (hwcap & HWCAP_FPHP) {
34 setCPUFeature(FEAT_FP16);
35 setCPUFeature(FEAT_FP);
36 }
37 if (hwcap & HWCAP_DIT)
38 setCPUFeature(FEAT_DIT);
39 if (hwcap & HWCAP_ASIMDRDM)
40 setCPUFeature(FEAT_RDM);
41 if (hwcap & HWCAP_ILRCPC)
42 setCPUFeature(FEAT_RCPC2);
43 if (hwcap & HWCAP_AES)
44 setCPUFeature(FEAT_AES);
45 if (hwcap & HWCAP_SHA1)
46 setCPUFeature(FEAT_SHA1);
47 if (hwcap & HWCAP_SHA2)
48 setCPUFeature(FEAT_SHA2);
49 if (hwcap & HWCAP_JSCVT)
50 setCPUFeature(FEAT_JSCVT);
51 if (hwcap & HWCAP_FCMA)
52 setCPUFeature(FEAT_FCMA);
53 if (hwcap & HWCAP_SB)
54 setCPUFeature(FEAT_SB);
55 if (hwcap & HWCAP_SSBS)
56 setCPUFeature(FEAT_SSBS2);
57 if (hwcap2 & HWCAP2_MTE) {
58 setCPUFeature(FEAT_MEMTAG);
59 setCPUFeature(FEAT_MEMTAG2);
60 }
61 if (hwcap2 & HWCAP2_MTE3) {
62 setCPUFeature(FEAT_MEMTAG);
63 setCPUFeature(FEAT_MEMTAG2);
64 setCPUFeature(FEAT_MEMTAG3);
65 }
66 if (hwcap2 & HWCAP2_SVEAES)
67 setCPUFeature(FEAT_SVE_AES);
68 if (hwcap2 & HWCAP2_SVEPMULL) {
69 setCPUFeature(FEAT_SVE_AES);
70 setCPUFeature(FEAT_SVE_PMULL128);
71 }
72 if (hwcap2 & HWCAP2_SVEBITPERM)
73 setCPUFeature(FEAT_SVE_BITPERM);
74 if (hwcap2 & HWCAP2_SVESHA3)
75 setCPUFeature(FEAT_SVE_SHA3);
76 if (hwcap2 & HWCAP2_SVESM4)
77 setCPUFeature(FEAT_SVE_SM4);
78 if (hwcap2 & HWCAP2_DCPODP)
79 setCPUFeature(FEAT_DPB2);
80 if (hwcap & HWCAP_ATOMICS)
81 setCPUFeature(FEAT_LSE);
82 if (hwcap2 & HWCAP2_RNG)
83 setCPUFeature(FEAT_RNG);
84 if (hwcap2 & HWCAP2_I8MM)
85 setCPUFeature(FEAT_I8MM);
86 if (hwcap2 & HWCAP2_EBF16)
87 setCPUFeature(FEAT_EBF16);
88 if (hwcap2 & HWCAP2_SVE_EBF16)
89 setCPUFeature(FEAT_SVE_EBF16);
90 if (hwcap2 & HWCAP2_DGH)
91 setCPUFeature(FEAT_DGH);
92 if (hwcap2 & HWCAP2_FRINT)
93 setCPUFeature(FEAT_FRINTTS);
94 if (hwcap2 & HWCAP2_SVEI8MM)
95 setCPUFeature(FEAT_SVE_I8MM);
96 if (hwcap2 & HWCAP2_SVEF32MM)
97 setCPUFeature(FEAT_SVE_F32MM);
98 if (hwcap2 & HWCAP2_SVEF64MM)
99 setCPUFeature(FEAT_SVE_F64MM);
100 if (hwcap2 & HWCAP2_BTI)
101 setCPUFeature(FEAT_BTI);
102 if (hwcap2 & HWCAP2_RPRES)
103 setCPUFeature(FEAT_RPRES);
104 if (hwcap2 & HWCAP2_WFXT)
105 setCPUFeature(FEAT_WFXT);
106 if (hwcap2 & HWCAP2_SME)
107 setCPUFeature(FEAT_SME);
108 if (hwcap2 & HWCAP2_SME_I16I64)
109 setCPUFeature(FEAT_SME_I64);
110 if (hwcap2 & HWCAP2_SME_F64F64)
111 setCPUFeature(FEAT_SME_F64);
112 if (hwcap2 & HWCAP2_MOPS)
113 setCPUFeature(FEAT_MOPS);
114 if (hwcap & HWCAP_CPUID) {
115 unsigned long ftr;
116 getCPUFeature(ID_AA64PFR1_EL1, ftr);
117 // ID_AA64PFR1_EL1.MTE >= 0b0001
118 if (extractBits(ftr, 8, 4) >= 0x1)
119 setCPUFeature(FEAT_MEMTAG);
120 // ID_AA64PFR1_EL1.SSBS == 0b0001
121 if (extractBits(ftr, 4, 4) == 0x1)
122 setCPUFeature(FEAT_SSBS);
123 // ID_AA64PFR1_EL1.SME == 0b0010
124 if (extractBits(ftr, 24, 4) == 0x2)
125 setCPUFeature(FEAT_SME2);
126 getCPUFeature(ID_AA64PFR0_EL1, ftr);
127 // ID_AA64PFR0_EL1.FP != 0b1111
128 if (extractBits(ftr, 16, 4) != 0xF) {
129 setCPUFeature(FEAT_FP);
130 // ID_AA64PFR0_EL1.AdvSIMD has the same value as ID_AA64PFR0_EL1.FP
131 setCPUFeature(FEAT_SIMD);
132 }
133 // ID_AA64PFR0_EL1.SVE != 0b0000
134 if (extractBits(ftr, 32, 4) != 0x0) {
135 // get ID_AA64ZFR0_EL1, that name supported
136 // if sve enabled only
137 getCPUFeature(S3_0_C0_C4_4, ftr);
138 // ID_AA64ZFR0_EL1.SVEver == 0b0000
139 if (extractBits(ftr, 0, 4) == 0x0)
140 setCPUFeature(FEAT_SVE);
141 // ID_AA64ZFR0_EL1.SVEver == 0b0001
142 if (extractBits(ftr, 0, 4) == 0x1)
143 setCPUFeature(FEAT_SVE2);
144 // ID_AA64ZFR0_EL1.BF16 != 0b0000
145 if (extractBits(ftr, 20, 4) != 0x0)
146 setCPUFeature(FEAT_SVE_BF16);
147 }
148 getCPUFeature(ID_AA64ISAR0_EL1, ftr);
149 // ID_AA64ISAR0_EL1.SHA3 != 0b0000
150 if (extractBits(ftr, 32, 4) != 0x0)
151 setCPUFeature(FEAT_SHA3);
152 getCPUFeature(ID_AA64ISAR1_EL1, ftr);
153 // ID_AA64ISAR1_EL1.DPB >= 0b0001
154 if (extractBits(ftr, 0, 4) >= 0x1)
155 setCPUFeature(FEAT_DPB);
156 // ID_AA64ISAR1_EL1.LRCPC != 0b0000
157 if (extractBits(ftr, 20, 4) != 0x0)
158 setCPUFeature(FEAT_RCPC);
159 // ID_AA64ISAR1_EL1.LRCPC == 0b0011
160 if (extractBits(ftr, 20, 4) == 0x3)
161 setCPUFeature(FEAT_RCPC3);
162 // ID_AA64ISAR1_EL1.SPECRES == 0b0001
163 if (extractBits(ftr, 40, 4) == 0x2)
164 setCPUFeature(FEAT_PREDRES);
165 // ID_AA64ISAR1_EL1.BF16 != 0b0000
166 if (extractBits(ftr, 44, 4) != 0x0)
167 setCPUFeature(FEAT_BF16);
168 // ID_AA64ISAR1_EL1.LS64 >= 0b0001
169 if (extractBits(ftr, 60, 4) >= 0x1)
170 setCPUFeature(FEAT_LS64);
171 // ID_AA64ISAR1_EL1.LS64 >= 0b0010
172 if (extractBits(ftr, 60, 4) >= 0x2)
173 setCPUFeature(FEAT_LS64_V);
174 // ID_AA64ISAR1_EL1.LS64 >= 0b0011
175 if (extractBits(ftr, 60, 4) >= 0x3)
176 setCPUFeature(FEAT_LS64_ACCDATA);
177 } else {
178 // Set some features in case of no CPUID support
179 if (hwcap & (HWCAP_FP | HWCAP_FPHP)) {
180 setCPUFeature(FEAT_FP);
181 // FP and AdvSIMD fields have the same value
182 setCPUFeature(FEAT_SIMD);
183 }
184 if (hwcap & HWCAP_DCPOP || hwcap2 & HWCAP2_DCPODP)
185 setCPUFeature(FEAT_DPB);
186 if (hwcap & HWCAP_LRCPC || hwcap & HWCAP_ILRCPC)
187 setCPUFeature(FEAT_RCPC);
188 if (hwcap2 & HWCAP2_BF16 || hwcap2 & HWCAP2_EBF16)
189 setCPUFeature(FEAT_BF16);
190 if (hwcap2 & HWCAP2_SVEBF16)
191 setCPUFeature(FEAT_SVE_BF16);
192 if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE)
193 setCPUFeature(FEAT_SVE2);
194 if (hwcap & HWCAP_SHA3)
195 setCPUFeature(FEAT_SHA3);
196 }
197 setCPUFeature(FEAT_INIT);
198}
199

source code of compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc