1 | //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the AArch64 implementation of the TargetRegisterInfo |
10 | // class. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "AArch64RegisterInfo.h" |
15 | #include "AArch64FrameLowering.h" |
16 | #include "AArch64InstrInfo.h" |
17 | #include "AArch64MachineFunctionInfo.h" |
18 | #include "AArch64Subtarget.h" |
19 | #include "MCTargetDesc/AArch64AddressingModes.h" |
20 | #include "MCTargetDesc/AArch64InstPrinter.h" |
21 | #include "llvm/ADT/BitVector.h" |
22 | #include "llvm/BinaryFormat/Dwarf.h" |
23 | #include "llvm/CodeGen/MachineFrameInfo.h" |
24 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
25 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
26 | #include "llvm/CodeGen/RegisterScavenging.h" |
27 | #include "llvm/CodeGen/TargetFrameLowering.h" |
28 | #include "llvm/IR/DebugInfoMetadata.h" |
29 | #include "llvm/IR/DiagnosticInfo.h" |
30 | #include "llvm/IR/Function.h" |
31 | #include "llvm/Support/raw_ostream.h" |
32 | #include "llvm/Target/TargetOptions.h" |
33 | #include "llvm/TargetParser/Triple.h" |
34 | |
35 | using namespace llvm; |
36 | |
37 | #define GET_CC_REGISTER_LISTS |
38 | #include "AArch64GenCallingConv.inc" |
39 | #define GET_REGINFO_TARGET_DESC |
40 | #include "AArch64GenRegisterInfo.inc" |
41 | |
42 | AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT) |
43 | : AArch64GenRegisterInfo(AArch64::LR), TT(TT) { |
44 | AArch64_MC::initLLVMToCVRegMapping(this); |
45 | } |
46 | |
47 | /// Return whether the register needs a CFI entry. Not all unwinders may know |
48 | /// about SVE registers, so we assume the lowest common denominator, i.e. the |
49 | /// callee-saves required by the base ABI. For the SVE registers z8-z15 only the |
50 | /// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is |
51 | /// returned in \p RegToUseForCFI. |
52 | bool AArch64RegisterInfo::regNeedsCFI(unsigned Reg, |
53 | unsigned &RegToUseForCFI) const { |
54 | if (AArch64::PPRRegClass.contains(Reg)) |
55 | return false; |
56 | |
57 | if (AArch64::ZPRRegClass.contains(Reg)) { |
58 | RegToUseForCFI = getSubReg(Reg, AArch64::dsub); |
59 | for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) { |
60 | if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI) |
61 | return true; |
62 | } |
63 | return false; |
64 | } |
65 | |
66 | RegToUseForCFI = Reg; |
67 | return true; |
68 | } |
69 | |
70 | const MCPhysReg * |
71 | AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { |
72 | assert(MF && "Invalid MachineFunction pointer." ); |
73 | |
74 | if (MF->getFunction().getCallingConv() == CallingConv::GHC) |
75 | // GHC set of callee saved regs is empty as all those regs are |
76 | // used for passing STG regs around |
77 | return CSR_AArch64_NoRegs_SaveList; |
78 | if (MF->getFunction().getCallingConv() == CallingConv::AnyReg) |
79 | return CSR_AArch64_AllRegs_SaveList; |
80 | |
81 | if (MF->getFunction().getCallingConv() == CallingConv::ARM64EC_Thunk_X64) |
82 | return CSR_Win_AArch64_Arm64EC_Thunk_SaveList; |
83 | |
84 | // Darwin has its own CSR_AArch64_AAPCS_SaveList, which means most CSR save |
85 | // lists depending on that will need to have their Darwin variant as well. |
86 | if (MF->getSubtarget<AArch64Subtarget>().isTargetDarwin()) |
87 | return getDarwinCalleeSavedRegs(MF); |
88 | |
89 | if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check) |
90 | return CSR_Win_AArch64_CFGuard_Check_SaveList; |
91 | if (MF->getSubtarget<AArch64Subtarget>().isTargetWindows()) { |
92 | if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() |
93 | ->supportSwiftError() && |
94 | MF->getFunction().getAttributes().hasAttrSomewhere( |
95 | Attribute::Kind: SwiftError)) |
96 | return CSR_Win_AArch64_AAPCS_SwiftError_SaveList; |
97 | if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) |
98 | return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList; |
99 | return CSR_Win_AArch64_AAPCS_SaveList; |
100 | } |
101 | if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) |
102 | return CSR_AArch64_AAVPCS_SaveList; |
103 | if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall) |
104 | return CSR_AArch64_SVE_AAPCS_SaveList; |
105 | if (MF->getFunction().getCallingConv() == |
106 | CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
107 | report_fatal_error( |
108 | reason: "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " |
109 | "only supported to improve calls to SME ACLE save/restore/disable-za " |
110 | "functions, and is not intended to be used beyond that scope." ); |
111 | if (MF->getFunction().getCallingConv() == |
112 | CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
113 | report_fatal_error( |
114 | reason: "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " |
115 | "only supported to improve calls to SME ACLE __arm_sme_state " |
116 | "and is not intended to be used beyond that scope." ); |
117 | if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() |
118 | ->supportSwiftError() && |
119 | MF->getFunction().getAttributes().hasAttrSomewhere( |
120 | Attribute::Kind: SwiftError)) |
121 | return CSR_AArch64_AAPCS_SwiftError_SaveList; |
122 | if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) |
123 | return CSR_AArch64_AAPCS_SwiftTail_SaveList; |
124 | if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) |
125 | return CSR_AArch64_RT_MostRegs_SaveList; |
126 | if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll) |
127 | return CSR_AArch64_RT_AllRegs_SaveList; |
128 | if (MF->getFunction().getCallingConv() == CallingConv::Win64) |
129 | // This is for OSes other than Windows; Windows is a separate case further |
130 | // above. |
131 | return CSR_AArch64_AAPCS_X18_SaveList; |
132 | if (MF->getInfo<AArch64FunctionInfo>()->isSVECC()) |
133 | return CSR_AArch64_SVE_AAPCS_SaveList; |
134 | return CSR_AArch64_AAPCS_SaveList; |
135 | } |
136 | |
137 | const MCPhysReg * |
138 | AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const { |
139 | assert(MF && "Invalid MachineFunction pointer." ); |
140 | assert(MF->getSubtarget<AArch64Subtarget>().isTargetDarwin() && |
141 | "Invalid subtarget for getDarwinCalleeSavedRegs" ); |
142 | |
143 | if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check) |
144 | report_fatal_error( |
145 | reason: "Calling convention CFGuard_Check is unsupported on Darwin." ); |
146 | if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) |
147 | return CSR_Darwin_AArch64_AAVPCS_SaveList; |
148 | if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall) |
149 | report_fatal_error( |
150 | reason: "Calling convention SVE_VectorCall is unsupported on Darwin." ); |
151 | if (MF->getFunction().getCallingConv() == |
152 | CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
153 | report_fatal_error( |
154 | reason: "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " |
155 | "only supported to improve calls to SME ACLE save/restore/disable-za " |
156 | "functions, and is not intended to be used beyond that scope." ); |
157 | if (MF->getFunction().getCallingConv() == |
158 | CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
159 | report_fatal_error( |
160 | reason: "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " |
161 | "only supported to improve calls to SME ACLE __arm_sme_state " |
162 | "and is not intended to be used beyond that scope." ); |
163 | if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS) |
164 | return MF->getInfo<AArch64FunctionInfo>()->isSplitCSR() |
165 | ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList |
166 | : CSR_Darwin_AArch64_CXX_TLS_SaveList; |
167 | if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() |
168 | ->supportSwiftError() && |
169 | MF->getFunction().getAttributes().hasAttrSomewhere( |
170 | Attribute::Kind: SwiftError)) |
171 | return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList; |
172 | if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) |
173 | return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList; |
174 | if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) |
175 | return CSR_Darwin_AArch64_RT_MostRegs_SaveList; |
176 | if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll) |
177 | return CSR_Darwin_AArch64_RT_AllRegs_SaveList; |
178 | if (MF->getFunction().getCallingConv() == CallingConv::Win64) |
179 | return CSR_Darwin_AArch64_AAPCS_Win64_SaveList; |
180 | return CSR_Darwin_AArch64_AAPCS_SaveList; |
181 | } |
182 | |
183 | const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy( |
184 | const MachineFunction *MF) const { |
185 | assert(MF && "Invalid MachineFunction pointer." ); |
186 | if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS && |
187 | MF->getInfo<AArch64FunctionInfo>()->isSplitCSR()) |
188 | return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList; |
189 | return nullptr; |
190 | } |
191 | |
192 | void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs( |
193 | MachineFunction &MF) const { |
194 | const MCPhysReg *CSRs = getCalleeSavedRegs(&MF); |
195 | SmallVector<MCPhysReg, 32> UpdatedCSRs; |
196 | for (const MCPhysReg *I = CSRs; *I; ++I) |
197 | UpdatedCSRs.push_back(Elt: *I); |
198 | |
199 | for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) { |
200 | if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) { |
201 | UpdatedCSRs.push_back(AArch64::Elt: GPR64commonRegClass.getRegister(i)); |
202 | } |
203 | } |
204 | // Register lists are zero-terminated. |
205 | UpdatedCSRs.push_back(Elt: 0); |
206 | MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs); |
207 | } |
208 | |
209 | const TargetRegisterClass * |
210 | AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC, |
211 | unsigned Idx) const { |
212 | // edge case for GPR/FPR register classes |
213 | if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub) |
214 | return &AArch64::FPR32RegClass; |
215 | else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub) |
216 | return &AArch64::FPR64RegClass; |
217 | |
218 | // Forward to TableGen's default version. |
219 | return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx); |
220 | } |
221 | |
222 | const uint32_t * |
223 | AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF, |
224 | CallingConv::ID CC) const { |
225 | assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() && |
226 | "Invalid subtarget for getDarwinCallPreservedMask" ); |
227 | |
228 | if (CC == CallingConv::CXX_FAST_TLS) |
229 | return CSR_Darwin_AArch64_CXX_TLS_RegMask; |
230 | if (CC == CallingConv::AArch64_VectorCall) |
231 | return CSR_Darwin_AArch64_AAVPCS_RegMask; |
232 | if (CC == CallingConv::AArch64_SVE_VectorCall) |
233 | report_fatal_error( |
234 | reason: "Calling convention SVE_VectorCall is unsupported on Darwin." ); |
235 | if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
236 | report_fatal_error( |
237 | reason: "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " |
238 | "unsupported on Darwin." ); |
239 | if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
240 | report_fatal_error( |
241 | reason: "Calling convention AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " |
242 | "unsupported on Darwin." ); |
243 | if (CC == CallingConv::CFGuard_Check) |
244 | report_fatal_error( |
245 | reason: "Calling convention CFGuard_Check is unsupported on Darwin." ); |
246 | if (MF.getSubtarget<AArch64Subtarget>() |
247 | .getTargetLowering() |
248 | ->supportSwiftError() && |
249 | MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError)) |
250 | return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask; |
251 | if (CC == CallingConv::SwiftTail) |
252 | return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask; |
253 | if (CC == CallingConv::PreserveMost) |
254 | return CSR_Darwin_AArch64_RT_MostRegs_RegMask; |
255 | if (CC == CallingConv::PreserveAll) |
256 | return CSR_Darwin_AArch64_RT_AllRegs_RegMask; |
257 | return CSR_Darwin_AArch64_AAPCS_RegMask; |
258 | } |
259 | |
260 | const uint32_t * |
261 | AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF, |
262 | CallingConv::ID CC) const { |
263 | bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack); |
264 | if (CC == CallingConv::GHC) |
265 | // This is academic because all GHC calls are (supposed to be) tail calls |
266 | return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask; |
267 | if (CC == CallingConv::AnyReg) |
268 | return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask; |
269 | |
270 | // All the following calling conventions are handled differently on Darwin. |
271 | if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) { |
272 | if (SCS) |
273 | report_fatal_error(reason: "ShadowCallStack attribute not supported on Darwin." ); |
274 | return getDarwinCallPreservedMask(MF, CC); |
275 | } |
276 | |
277 | if (CC == CallingConv::AArch64_VectorCall) |
278 | return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask; |
279 | if (CC == CallingConv::AArch64_SVE_VectorCall) |
280 | return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask |
281 | : CSR_AArch64_SVE_AAPCS_RegMask; |
282 | if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
283 | return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; |
284 | if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
285 | return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; |
286 | if (CC == CallingConv::CFGuard_Check) |
287 | return CSR_Win_AArch64_CFGuard_Check_RegMask; |
288 | if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering() |
289 | ->supportSwiftError() && |
290 | MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError)) |
291 | return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask |
292 | : CSR_AArch64_AAPCS_SwiftError_RegMask; |
293 | if (CC == CallingConv::SwiftTail) { |
294 | if (SCS) |
295 | report_fatal_error(reason: "ShadowCallStack attribute not supported with swifttail" ); |
296 | return CSR_AArch64_AAPCS_SwiftTail_RegMask; |
297 | } |
298 | if (CC == CallingConv::PreserveMost) |
299 | return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask |
300 | : CSR_AArch64_RT_MostRegs_RegMask; |
301 | else if (CC == CallingConv::PreserveAll) |
302 | return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask |
303 | : CSR_AArch64_RT_AllRegs_RegMask; |
304 | |
305 | else |
306 | return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask; |
307 | } |
308 | |
309 | const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask( |
310 | const MachineFunction &MF) const { |
311 | if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux()) |
312 | return CSR_AArch64_AAPCS_RegMask; |
313 | |
314 | return nullptr; |
315 | } |
316 | |
317 | const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const { |
318 | if (TT.isOSDarwin()) |
319 | return CSR_Darwin_AArch64_TLS_RegMask; |
320 | |
321 | assert(TT.isOSBinFormatELF() && "Invalid target" ); |
322 | return CSR_AArch64_TLS_ELF_RegMask; |
323 | } |
324 | |
325 | void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF, |
326 | const uint32_t **Mask) const { |
327 | uint32_t *UpdatedMask = MF.allocateRegMask(); |
328 | unsigned RegMaskSize = MachineOperand::getRegMaskSize(NumRegs: getNumRegs()); |
329 | memcpy(dest: UpdatedMask, src: *Mask, n: sizeof(UpdatedMask[0]) * RegMaskSize); |
330 | |
331 | for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) { |
332 | if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) { |
333 | for (MCPhysReg SubReg : |
334 | subregs_inclusive(AArch64::GPR64commonRegClass.getRegister(i))) { |
335 | // See TargetRegisterInfo::getCallPreservedMask for how to interpret the |
336 | // register mask. |
337 | UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32); |
338 | } |
339 | } |
340 | } |
341 | *Mask = UpdatedMask; |
342 | } |
343 | |
344 | const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const { |
345 | return CSR_AArch64_SMStartStop_RegMask; |
346 | } |
347 | |
348 | const uint32_t * |
349 | AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const { |
350 | return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; |
351 | } |
352 | |
353 | const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const { |
354 | return CSR_AArch64_NoRegs_RegMask; |
355 | } |
356 | |
357 | const uint32_t * |
358 | AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF, |
359 | CallingConv::ID CC) const { |
360 | // This should return a register mask that is the same as that returned by |
361 | // getCallPreservedMask but that additionally preserves the register used for |
362 | // the first i64 argument (which must also be the register used to return a |
363 | // single i64 return value) |
364 | // |
365 | // In case that the calling convention does not use the same register for |
366 | // both, the function should return NULL (does not currently apply) |
367 | assert(CC != CallingConv::GHC && "should not be GHC calling convention." ); |
368 | if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) |
369 | return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask; |
370 | return CSR_AArch64_AAPCS_ThisReturn_RegMask; |
371 | } |
372 | |
373 | const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const { |
374 | return CSR_AArch64_StackProbe_Windows_RegMask; |
375 | } |
376 | |
377 | std::optional<std::string> |
378 | AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF, |
379 | MCRegister PhysReg) const { |
380 | if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19)) |
381 | return std::string("X19 is used as the frame base pointer register." ); |
382 | |
383 | if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) { |
384 | bool warn = false; |
385 | if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) || |
386 | MCRegisterInfo::regsOverlap(PhysReg, AArch64::X14) || |
387 | MCRegisterInfo::regsOverlap(PhysReg, AArch64::X23) || |
388 | MCRegisterInfo::regsOverlap(PhysReg, AArch64::X24) || |
389 | MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28)) |
390 | warn = true; |
391 | |
392 | for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i) |
393 | if (MCRegisterInfo::regsOverlap(PhysReg, i)) |
394 | warn = true; |
395 | |
396 | if (warn) |
397 | return std::string(AArch64InstPrinter::getRegisterName(Reg: PhysReg)) + |
398 | " is clobbered by asynchronous signals when using Arm64EC." ; |
399 | } |
400 | |
401 | return {}; |
402 | } |
403 | |
404 | BitVector |
405 | AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const { |
406 | const AArch64FrameLowering *TFI = getFrameLowering(MF); |
407 | |
408 | // FIXME: avoid re-calculating this every time. |
409 | BitVector Reserved(getNumRegs()); |
410 | markSuperRegs(Reserved, AArch64::WSP); |
411 | markSuperRegs(Reserved, AArch64::WZR); |
412 | |
413 | if (TFI->hasFP(MF) || TT.isOSDarwin()) |
414 | markSuperRegs(Reserved, AArch64::W29); |
415 | |
416 | if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) { |
417 | // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous |
418 | // signals, so we can't ever use them. |
419 | markSuperRegs(Reserved, AArch64::W13); |
420 | markSuperRegs(Reserved, AArch64::W14); |
421 | markSuperRegs(Reserved, AArch64::W23); |
422 | markSuperRegs(Reserved, AArch64::W24); |
423 | markSuperRegs(Reserved, AArch64::W28); |
424 | for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i) |
425 | markSuperRegs(Reserved, i); |
426 | } |
427 | |
428 | for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { |
429 | if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i)) |
430 | markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); |
431 | } |
432 | |
433 | if (hasBasePointer(MF)) |
434 | markSuperRegs(Reserved, AArch64::W19); |
435 | |
436 | // SLH uses register W16/X16 as the taint register. |
437 | if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) |
438 | markSuperRegs(Reserved, AArch64::W16); |
439 | |
440 | // FFR is modelled as global state that cannot be allocated. |
441 | if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) |
442 | Reserved.set(AArch64::FFR); |
443 | |
444 | // SME tiles are not allocatable. |
445 | if (MF.getSubtarget<AArch64Subtarget>().hasSME()) { |
446 | for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA)) |
447 | Reserved.set(SubReg); |
448 | } |
449 | |
450 | // VG cannot be allocated |
451 | Reserved.set(AArch64::VG); |
452 | |
453 | if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) { |
454 | for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true); |
455 | SubReg.isValid(); ++SubReg) |
456 | Reserved.set(*SubReg); |
457 | } |
458 | |
459 | markSuperRegs(Reserved, AArch64::FPCR); |
460 | markSuperRegs(Reserved, AArch64::FPSR); |
461 | |
462 | if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) { |
463 | markSuperRegs(Reserved, AArch64::X27); |
464 | markSuperRegs(Reserved, AArch64::X28); |
465 | markSuperRegs(Reserved, AArch64::W27); |
466 | markSuperRegs(Reserved, AArch64::W28); |
467 | } |
468 | |
469 | assert(checkAllSuperRegsMarked(Reserved)); |
470 | return Reserved; |
471 | } |
472 | |
473 | BitVector |
474 | AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const { |
475 | BitVector Reserved = getStrictlyReservedRegs(MF); |
476 | |
477 | for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { |
478 | if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i)) |
479 | markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); |
480 | } |
481 | |
482 | assert(checkAllSuperRegsMarked(Reserved)); |
483 | return Reserved; |
484 | } |
485 | |
486 | bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF, |
487 | MCRegister Reg) const { |
488 | return getReservedRegs(MF)[Reg]; |
489 | } |
490 | |
491 | bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF, |
492 | MCRegister Reg) const { |
493 | return getStrictlyReservedRegs(MF)[Reg]; |
494 | } |
495 | |
496 | bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const { |
497 | return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) { |
498 | return isStrictlyReservedReg(MF, r); |
499 | }); |
500 | } |
501 | |
502 | void AArch64RegisterInfo::emitReservedArgRegCallError( |
503 | const MachineFunction &MF) const { |
504 | const Function &F = MF.getFunction(); |
505 | F.getContext().diagnose(DI: DiagnosticInfoUnsupported{F, ("AArch64 doesn't support" |
506 | " function calls if any of the argument registers is reserved." )}); |
507 | } |
508 | |
509 | bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF, |
510 | MCRegister PhysReg) const { |
511 | // SLH uses register X16 as the taint register but it will fallback to a different |
512 | // method if the user clobbers it. So X16 is not reserved for inline asm but is |
513 | // for normal codegen. |
514 | if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening) && |
515 | MCRegisterInfo::regsOverlap(PhysReg, AArch64::X16)) |
516 | return true; |
517 | |
518 | // ZA/ZT0 registers are reserved but may be permitted in the clobber list. |
519 | if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0) |
520 | return true; |
521 | |
522 | return !isReservedReg(MF, PhysReg); |
523 | } |
524 | |
525 | const TargetRegisterClass * |
526 | AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF, |
527 | unsigned Kind) const { |
528 | return &AArch64::GPR64spRegClass; |
529 | } |
530 | |
531 | const TargetRegisterClass * |
532 | AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { |
533 | if (RC == &AArch64::CCRRegClass) |
534 | return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV. |
535 | return RC; |
536 | } |
537 | |
538 | unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; } |
539 | |
540 | bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { |
541 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
542 | |
543 | // In the presence of variable sized objects or funclets, if the fixed stack |
544 | // size is large enough that referencing from the FP won't result in things |
545 | // being in range relatively often, we can use a base pointer to allow access |
546 | // from the other direction like the SP normally works. |
547 | // |
548 | // Furthermore, if both variable sized objects are present, and the |
549 | // stack needs to be dynamically re-aligned, the base pointer is the only |
550 | // reliable way to reference the locals. |
551 | if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) { |
552 | if (hasStackRealignment(MF)) |
553 | return true; |
554 | |
555 | if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) { |
556 | const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); |
557 | // Frames that have variable sized objects and scalable SVE objects, |
558 | // should always use a basepointer. |
559 | if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE()) |
560 | return true; |
561 | } |
562 | |
563 | // Conservatively estimate whether the negative offset from the frame |
564 | // pointer will be sufficient to reach. If a function has a smallish |
565 | // frame, it's less likely to have lots of spills and callee saved |
566 | // space, so it's all more likely to be within range of the frame pointer. |
567 | // If it's wrong, we'll materialize the constant and still get to the |
568 | // object; it's just suboptimal. Negative offsets use the unscaled |
569 | // load/store instructions, which have a 9-bit signed immediate. |
570 | return MFI.getLocalFrameSize() >= 256; |
571 | } |
572 | |
573 | return false; |
574 | } |
575 | |
576 | bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF, |
577 | MCRegister Reg) const { |
578 | CallingConv::ID CC = MF.getFunction().getCallingConv(); |
579 | const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); |
580 | bool IsVarArg = STI.isCallingConvWin64(CC: MF.getFunction().getCallingConv()); |
581 | |
582 | auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) { |
583 | return llvm::is_contained(Range&: RegList, Element: Reg); |
584 | }; |
585 | |
586 | switch (CC) { |
587 | default: |
588 | report_fatal_error(reason: "Unsupported calling convention." ); |
589 | case CallingConv::GHC: |
590 | return HasReg(CC_AArch64_GHC_ArgRegs, Reg); |
591 | case CallingConv::C: |
592 | case CallingConv::Fast: |
593 | case CallingConv::PreserveMost: |
594 | case CallingConv::PreserveAll: |
595 | case CallingConv::CXX_FAST_TLS: |
596 | case CallingConv::Swift: |
597 | case CallingConv::SwiftTail: |
598 | case CallingConv::Tail: |
599 | if (STI.isTargetWindows()) { |
600 | if (IsVarArg) |
601 | return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg); |
602 | switch (CC) { |
603 | default: |
604 | return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
605 | case CallingConv::Swift: |
606 | case CallingConv::SwiftTail: |
607 | return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) || |
608 | HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
609 | } |
610 | } |
611 | if (!STI.isTargetDarwin()) { |
612 | switch (CC) { |
613 | default: |
614 | return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg); |
615 | case CallingConv::Swift: |
616 | case CallingConv::SwiftTail: |
617 | return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) || |
618 | HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg); |
619 | } |
620 | } |
621 | if (!IsVarArg) { |
622 | switch (CC) { |
623 | default: |
624 | return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg); |
625 | case CallingConv::Swift: |
626 | case CallingConv::SwiftTail: |
627 | return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) || |
628 | HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg); |
629 | } |
630 | } |
631 | if (STI.isTargetILP32()) |
632 | return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg); |
633 | return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg); |
634 | case CallingConv::Win64: |
635 | if (IsVarArg) |
636 | HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg); |
637 | return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
638 | case CallingConv::CFGuard_Check: |
639 | return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg); |
640 | case CallingConv::AArch64_VectorCall: |
641 | case CallingConv::AArch64_SVE_VectorCall: |
642 | case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0: |
643 | case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2: |
644 | if (STI.isTargetWindows()) |
645 | return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
646 | return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg); |
647 | } |
648 | } |
649 | |
650 | Register |
651 | AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const { |
652 | const AArch64FrameLowering *TFI = getFrameLowering(MF); |
653 | return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP; |
654 | } |
655 | |
656 | bool AArch64RegisterInfo::requiresRegisterScavenging( |
657 | const MachineFunction &MF) const { |
658 | return true; |
659 | } |
660 | |
661 | bool AArch64RegisterInfo::requiresVirtualBaseRegisters( |
662 | const MachineFunction &MF) const { |
663 | return true; |
664 | } |
665 | |
666 | bool |
667 | AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const { |
668 | // This function indicates whether the emergency spillslot should be placed |
669 | // close to the beginning of the stackframe (closer to FP) or the end |
670 | // (closer to SP). |
671 | // |
672 | // The beginning works most reliably if we have a frame pointer. |
673 | // In the presence of any non-constant space between FP and locals, |
674 | // (e.g. in case of stack realignment or a scalable SVE area), it is |
675 | // better to use SP or BP. |
676 | const AArch64FrameLowering &TFI = *getFrameLowering(MF); |
677 | const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); |
678 | assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() || |
679 | AFI->hasCalculatedStackSizeSVE()) && |
680 | "Expected SVE area to be calculated by this point" ); |
681 | return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE(); |
682 | } |
683 | |
684 | bool AArch64RegisterInfo::requiresFrameIndexScavenging( |
685 | const MachineFunction &MF) const { |
686 | return true; |
687 | } |
688 | |
689 | bool |
690 | AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const { |
691 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
692 | if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack()) |
693 | return true; |
694 | return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken(); |
695 | } |
696 | |
697 | /// needsFrameBaseReg - Returns true if the instruction's frame index |
698 | /// reference would be better served by a base register other than FP |
699 | /// or SP. Used by LocalStackFrameAllocation to determine which frame index |
700 | /// references it should create new base registers for. |
701 | bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI, |
702 | int64_t Offset) const { |
703 | for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i) |
704 | assert(i < MI->getNumOperands() && |
705 | "Instr doesn't have FrameIndex operand!" ); |
706 | |
707 | // It's the load/store FI references that cause issues, as it can be difficult |
708 | // to materialize the offset if it won't fit in the literal field. Estimate |
709 | // based on the size of the local frame and some conservative assumptions |
710 | // about the rest of the stack frame (note, this is pre-regalloc, so |
711 | // we don't know everything for certain yet) whether this offset is likely |
712 | // to be out of range of the immediate. Return true if so. |
713 | |
714 | // We only generate virtual base registers for loads and stores, so |
715 | // return false for everything else. |
716 | if (!MI->mayLoad() && !MI->mayStore()) |
717 | return false; |
718 | |
719 | // Without a virtual base register, if the function has variable sized |
720 | // objects, all fixed-size local references will be via the frame pointer, |
721 | // Approximate the offset and see if it's legal for the instruction. |
722 | // Note that the incoming offset is based on the SP value at function entry, |
723 | // so it'll be negative. |
724 | MachineFunction &MF = *MI->getParent()->getParent(); |
725 | const AArch64FrameLowering *TFI = getFrameLowering(MF); |
726 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
727 | |
728 | // Estimate an offset from the frame pointer. |
729 | // Conservatively assume all GPR callee-saved registers get pushed. |
730 | // FP, LR, X19-X28, D8-D15. 64-bits each. |
731 | int64_t FPOffset = Offset - 16 * 20; |
732 | // Estimate an offset from the stack pointer. |
733 | // The incoming offset is relating to the SP at the start of the function, |
734 | // but when we access the local it'll be relative to the SP after local |
735 | // allocation, so adjust our SP-relative offset by that allocation size. |
736 | Offset += MFI.getLocalFrameSize(); |
737 | // Assume that we'll have at least some spill slots allocated. |
738 | // FIXME: This is a total SWAG number. We should run some statistics |
739 | // and pick a real one. |
740 | Offset += 128; // 128 bytes of spill slots |
741 | |
742 | // If there is a frame pointer, try using it. |
743 | // The FP is only available if there is no dynamic realignment. We |
744 | // don't know for sure yet whether we'll need that, so we guess based |
745 | // on whether there are any local variables that would trigger it. |
746 | if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset)) |
747 | return false; |
748 | |
749 | // If we can reference via the stack pointer or base pointer, try that. |
750 | // FIXME: This (and the code that resolves the references) can be improved |
751 | // to only disallow SP relative references in the live range of |
752 | // the VLA(s). In practice, it's unclear how much difference that |
753 | // would make, but it may be worth doing. |
754 | if (isFrameOffsetLegal(MI, AArch64::SP, Offset)) |
755 | return false; |
756 | |
757 | // If even offset 0 is illegal, we don't want a virtual base register. |
758 | if (!isFrameOffsetLegal(MI, AArch64::SP, 0)) |
759 | return false; |
760 | |
761 | // The offset likely isn't legal; we want to allocate a virtual base register. |
762 | return true; |
763 | } |
764 | |
765 | bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, |
766 | Register BaseReg, |
767 | int64_t Offset) const { |
768 | assert(MI && "Unable to get the legal offset for nil instruction." ); |
769 | StackOffset SaveOffset = StackOffset::getFixed(Fixed: Offset); |
770 | return isAArch64FrameOffsetLegal(MI: *MI, Offset&: SaveOffset) & AArch64FrameOffsetIsLegal; |
771 | } |
772 | |
773 | /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx |
774 | /// at the beginning of the basic block. |
775 | Register |
776 | AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, |
777 | int FrameIdx, |
778 | int64_t Offset) const { |
779 | MachineBasicBlock::iterator Ins = MBB->begin(); |
780 | DebugLoc DL; // Defaults to "unknown" |
781 | if (Ins != MBB->end()) |
782 | DL = Ins->getDebugLoc(); |
783 | const MachineFunction &MF = *MBB->getParent(); |
784 | const AArch64InstrInfo *TII = |
785 | MF.getSubtarget<AArch64Subtarget>().getInstrInfo(); |
786 | const MCInstrDesc &MCID = TII->get(AArch64::ADDXri); |
787 | MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); |
788 | Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); |
789 | MRI.constrainRegClass(Reg: BaseReg, RC: TII->getRegClass(MCID, 0, this, MF)); |
790 | unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0); |
791 | |
792 | BuildMI(BB&: *MBB, I: Ins, MIMD: DL, MCID, DestReg: BaseReg) |
793 | .addFrameIndex(Idx: FrameIdx) |
794 | .addImm(Val: Offset) |
795 | .addImm(Val: Shifter); |
796 | |
797 | return BaseReg; |
798 | } |
799 | |
800 | void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, |
801 | int64_t Offset) const { |
802 | // ARM doesn't need the general 64-bit offsets |
803 | StackOffset Off = StackOffset::getFixed(Fixed: Offset); |
804 | |
805 | unsigned i = 0; |
806 | while (!MI.getOperand(i).isFI()) { |
807 | ++i; |
808 | assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!" ); |
809 | } |
810 | |
811 | const MachineFunction *MF = MI.getParent()->getParent(); |
812 | const AArch64InstrInfo *TII = |
813 | MF->getSubtarget<AArch64Subtarget>().getInstrInfo(); |
814 | bool Done = rewriteAArch64FrameIndex(MI, FrameRegIdx: i, FrameReg: BaseReg, Offset&: Off, TII); |
815 | assert(Done && "Unable to resolve frame index!" ); |
816 | (void)Done; |
817 | } |
818 | |
819 | // Create a scratch register for the frame index elimination in an instruction. |
820 | // This function has special handling of stack tagging loop pseudos, in which |
821 | // case it can also change the instruction opcode. |
822 | static Register |
823 | createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum, |
824 | const AArch64InstrInfo *TII) { |
825 | // ST*Gloop have a reserved scratch register in operand 1. Use it, and also |
826 | // replace the instruction with the writeback variant because it will now |
827 | // satisfy the operand constraints for it. |
828 | Register ScratchReg; |
829 | if (MI.getOpcode() == AArch64::STGloop || |
830 | MI.getOpcode() == AArch64::STZGloop) { |
831 | assert(FIOperandNum == 3 && |
832 | "Wrong frame index operand for STGloop/STZGloop" ); |
833 | unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback |
834 | : AArch64::STZGloop_wback; |
835 | ScratchReg = MI.getOperand(i: 1).getReg(); |
836 | MI.getOperand(i: 3).ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true); |
837 | MI.setDesc(TII->get(Op)); |
838 | MI.tieOperands(DefIdx: 1, UseIdx: 3); |
839 | } else { |
840 | ScratchReg = |
841 | MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); |
842 | MI.getOperand(i: FIOperandNum) |
843 | .ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true); |
844 | } |
845 | return ScratchReg; |
846 | } |
847 | |
848 | void AArch64RegisterInfo::getOffsetOpcodes( |
849 | const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const { |
850 | // The smallest scalable element supported by scaled SVE addressing |
851 | // modes are predicates, which are 2 scalable bytes in size. So the scalable |
852 | // byte offset must always be a multiple of 2. |
853 | assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset" ); |
854 | |
855 | // Add fixed-sized offset using existing DIExpression interface. |
856 | DIExpression::appendOffset(Ops, Offset: Offset.getFixed()); |
857 | |
858 | unsigned VG = getDwarfRegNum(AArch64::VG, true); |
859 | int64_t VGSized = Offset.getScalable() / 2; |
860 | if (VGSized > 0) { |
861 | Ops.push_back(Elt: dwarf::DW_OP_constu); |
862 | Ops.push_back(Elt: VGSized); |
863 | Ops.append(IL: {dwarf::DW_OP_bregx, VG, 0ULL}); |
864 | Ops.push_back(Elt: dwarf::DW_OP_mul); |
865 | Ops.push_back(Elt: dwarf::DW_OP_plus); |
866 | } else if (VGSized < 0) { |
867 | Ops.push_back(Elt: dwarf::DW_OP_constu); |
868 | Ops.push_back(Elt: -VGSized); |
869 | Ops.append(IL: {dwarf::DW_OP_bregx, VG, 0ULL}); |
870 | Ops.push_back(Elt: dwarf::DW_OP_mul); |
871 | Ops.push_back(Elt: dwarf::DW_OP_minus); |
872 | } |
873 | } |
874 | |
875 | bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, |
876 | int SPAdj, unsigned FIOperandNum, |
877 | RegScavenger *RS) const { |
878 | assert(SPAdj == 0 && "Unexpected" ); |
879 | |
880 | MachineInstr &MI = *II; |
881 | MachineBasicBlock &MBB = *MI.getParent(); |
882 | MachineFunction &MF = *MBB.getParent(); |
883 | const MachineFrameInfo &MFI = MF.getFrameInfo(); |
884 | const AArch64InstrInfo *TII = |
885 | MF.getSubtarget<AArch64Subtarget>().getInstrInfo(); |
886 | const AArch64FrameLowering *TFI = getFrameLowering(MF); |
887 | int FrameIndex = MI.getOperand(i: FIOperandNum).getIndex(); |
888 | bool Tagged = |
889 | MI.getOperand(i: FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED; |
890 | Register FrameReg; |
891 | |
892 | // Special handling of dbg_value, stackmap patchpoint statepoint instructions. |
893 | if (MI.getOpcode() == TargetOpcode::STACKMAP || |
894 | MI.getOpcode() == TargetOpcode::PATCHPOINT || |
895 | MI.getOpcode() == TargetOpcode::STATEPOINT) { |
896 | StackOffset Offset = |
897 | TFI->resolveFrameIndexReference(MF, FI: FrameIndex, FrameReg, |
898 | /*PreferFP=*/true, |
899 | /*ForSimm=*/false); |
900 | Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: FIOperandNum + 1).getImm()); |
901 | MI.getOperand(i: FIOperandNum).ChangeToRegister(Reg: FrameReg, isDef: false /*isDef*/); |
902 | MI.getOperand(i: FIOperandNum + 1).ChangeToImmediate(ImmVal: Offset.getFixed()); |
903 | return false; |
904 | } |
905 | |
906 | if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) { |
907 | MachineOperand &FI = MI.getOperand(i: FIOperandNum); |
908 | StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FI: FrameIndex); |
909 | assert(!Offset.getScalable() && |
910 | "Frame offsets with a scalable component are not supported" ); |
911 | FI.ChangeToImmediate(ImmVal: Offset.getFixed()); |
912 | return false; |
913 | } |
914 | |
915 | StackOffset Offset; |
916 | if (MI.getOpcode() == AArch64::TAGPstack) { |
917 | // TAGPstack must use the virtual frame register in its 3rd operand. |
918 | const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); |
919 | FrameReg = MI.getOperand(i: 3).getReg(); |
920 | Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) + |
921 | AFI->getTaggedBasePointerOffset()); |
922 | } else if (Tagged) { |
923 | StackOffset SPOffset = StackOffset::getFixed( |
924 | Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) + (int64_t)MFI.getStackSize()); |
925 | if (MFI.hasVarSizedObjects() || |
926 | isAArch64FrameOffsetLegal(MI, Offset&: SPOffset, OutUseUnscaledOp: nullptr, OutUnscaledOp: nullptr, EmittableOffset: nullptr) != |
927 | (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) { |
928 | // Can't update to SP + offset in place. Precalculate the tagged pointer |
929 | // in a scratch register. |
930 | Offset = TFI->resolveFrameIndexReference( |
931 | MF, FI: FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); |
932 | Register ScratchReg = |
933 | MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); |
934 | emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, |
935 | TII); |
936 | BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg) |
937 | .addReg(ScratchReg) |
938 | .addReg(ScratchReg) |
939 | .addImm(0); |
940 | MI.getOperand(i: FIOperandNum) |
941 | .ChangeToRegister(Reg: ScratchReg, isDef: false, isImp: false, isKill: true); |
942 | return false; |
943 | } |
944 | FrameReg = AArch64::SP; |
945 | Offset = StackOffset::getFixed(Fixed: MFI.getObjectOffset(ObjectIdx: FrameIndex) + |
946 | (int64_t)MFI.getStackSize()); |
947 | } else { |
948 | Offset = TFI->resolveFrameIndexReference( |
949 | MF, FI: FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); |
950 | } |
951 | |
952 | // Modify MI as necessary to handle as much of 'Offset' as possible |
953 | if (rewriteAArch64FrameIndex(MI, FrameRegIdx: FIOperandNum, FrameReg, Offset, TII)) |
954 | return true; |
955 | |
956 | assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) && |
957 | "Emergency spill slot is out of reach" ); |
958 | |
959 | // If we get here, the immediate doesn't fit into the instruction. We folded |
960 | // as much as possible above. Handle the rest, providing a register that is |
961 | // SP+LargeImm. |
962 | Register ScratchReg = |
963 | createScratchRegisterForInstruction(MI, FIOperandNum, TII); |
964 | emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII); |
965 | return false; |
966 | } |
967 | |
968 | unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, |
969 | MachineFunction &MF) const { |
970 | const AArch64FrameLowering *TFI = getFrameLowering(MF); |
971 | |
972 | switch (RC->getID()) { |
973 | default: |
974 | return 0; |
975 | case AArch64::GPR32RegClassID: |
976 | case AArch64::GPR32spRegClassID: |
977 | case AArch64::GPR32allRegClassID: |
978 | case AArch64::GPR64spRegClassID: |
979 | case AArch64::GPR64allRegClassID: |
980 | case AArch64::GPR64RegClassID: |
981 | case AArch64::GPR32commonRegClassID: |
982 | case AArch64::GPR64commonRegClassID: |
983 | return 32 - 1 // XZR/SP |
984 | - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP |
985 | - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved() |
986 | - hasBasePointer(MF); // X19 |
987 | case AArch64::FPR8RegClassID: |
988 | case AArch64::FPR16RegClassID: |
989 | case AArch64::FPR32RegClassID: |
990 | case AArch64::FPR64RegClassID: |
991 | case AArch64::FPR128RegClassID: |
992 | return 32; |
993 | |
994 | case AArch64::MatrixIndexGPR32_8_11RegClassID: |
995 | case AArch64::MatrixIndexGPR32_12_15RegClassID: |
996 | return 4; |
997 | |
998 | case AArch64::DDRegClassID: |
999 | case AArch64::DDDRegClassID: |
1000 | case AArch64::DDDDRegClassID: |
1001 | case AArch64::QQRegClassID: |
1002 | case AArch64::QQQRegClassID: |
1003 | case AArch64::QQQQRegClassID: |
1004 | return 32; |
1005 | |
1006 | case AArch64::FPR128_loRegClassID: |
1007 | case AArch64::FPR64_loRegClassID: |
1008 | case AArch64::FPR16_loRegClassID: |
1009 | return 16; |
1010 | case AArch64::FPR128_0to7RegClassID: |
1011 | return 8; |
1012 | } |
1013 | } |
1014 | |
1015 | unsigned AArch64RegisterInfo::getLocalAddressRegister( |
1016 | const MachineFunction &MF) const { |
1017 | const auto &MFI = MF.getFrameInfo(); |
1018 | if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects()) |
1019 | return AArch64::SP; |
1020 | else if (hasStackRealignment(MF)) |
1021 | return getBaseRegister(); |
1022 | return getFrameRegister(MF); |
1023 | } |
1024 | |
1025 | /// SrcRC and DstRC will be morphed into NewRC if this returns true |
1026 | bool AArch64RegisterInfo::shouldCoalesce( |
1027 | MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg, |
1028 | const TargetRegisterClass *DstRC, unsigned DstSubReg, |
1029 | const TargetRegisterClass *NewRC, LiveIntervals &LIS) const { |
1030 | MachineRegisterInfo &MRI = MI->getMF()->getRegInfo(); |
1031 | |
1032 | if (MI->isCopy() && |
1033 | ((DstRC->getID() == AArch64::GPR64RegClassID) || |
1034 | (DstRC->getID() == AArch64::GPR64commonRegClassID)) && |
1035 | MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg()) |
1036 | // Do not coalesce in the case of a 32-bit subregister copy |
1037 | // which implements a 32 to 64 bit zero extension |
1038 | // which relies on the upper 32 bits being zeroed. |
1039 | return false; |
1040 | |
1041 | auto IsCoalescerBarrier = [](const MachineInstr &MI) { |
1042 | switch (MI.getOpcode()) { |
1043 | case AArch64::COALESCER_BARRIER_FPR16: |
1044 | case AArch64::COALESCER_BARRIER_FPR32: |
1045 | case AArch64::COALESCER_BARRIER_FPR64: |
1046 | case AArch64::COALESCER_BARRIER_FPR128: |
1047 | return true; |
1048 | default: |
1049 | return false; |
1050 | } |
1051 | }; |
1052 | |
1053 | // For calls that temporarily have to toggle streaming mode as part of the |
1054 | // call-sequence, we need to be more careful when coalescing copy instructions |
1055 | // so that we don't end up coalescing the NEON/FP result or argument register |
1056 | // with a whole Z-register, such that after coalescing the register allocator |
1057 | // will try to spill/reload the entire Z register. |
1058 | // |
1059 | // We do this by checking if the node has any defs/uses that are |
1060 | // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to |
1061 | // instruct the coalescer to avoid coalescing the copy. |
1062 | if (MI->isCopy() && SubReg != DstSubReg && |
1063 | (AArch64::ZPRRegClass.hasSubClassEq(DstRC) || |
1064 | AArch64::ZPRRegClass.hasSubClassEq(SrcRC))) { |
1065 | unsigned SrcReg = MI->getOperand(i: 1).getReg(); |
1066 | if (any_of(Range: MRI.def_instructions(Reg: SrcReg), P: IsCoalescerBarrier)) |
1067 | return false; |
1068 | unsigned DstReg = MI->getOperand(i: 0).getReg(); |
1069 | if (any_of(Range: MRI.use_nodbg_instructions(Reg: DstReg), P: IsCoalescerBarrier)) |
1070 | return false; |
1071 | } |
1072 | |
1073 | return true; |
1074 | } |
1075 | |
1076 | bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo( |
1077 | MCRegister R) const { |
1078 | return R == AArch64::VG; |
1079 | } |
1080 | |