1 | //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains a printer that converts from our internal representation |
10 | // of machine-dependent LLVM code to the AArch64 assembly language. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #include "AArch64.h" |
15 | #include "AArch64MCInstLower.h" |
16 | #include "AArch64MachineFunctionInfo.h" |
17 | #include "AArch64RegisterInfo.h" |
18 | #include "AArch64Subtarget.h" |
19 | #include "AArch64TargetObjectFile.h" |
20 | #include "MCTargetDesc/AArch64AddressingModes.h" |
21 | #include "MCTargetDesc/AArch64InstPrinter.h" |
22 | #include "MCTargetDesc/AArch64MCExpr.h" |
23 | #include "MCTargetDesc/AArch64MCTargetDesc.h" |
24 | #include "MCTargetDesc/AArch64TargetStreamer.h" |
25 | #include "TargetInfo/AArch64TargetInfo.h" |
26 | #include "Utils/AArch64BaseInfo.h" |
27 | #include "llvm/ADT/SmallString.h" |
28 | #include "llvm/ADT/SmallVector.h" |
29 | #include "llvm/ADT/StringRef.h" |
30 | #include "llvm/ADT/Twine.h" |
31 | #include "llvm/BinaryFormat/COFF.h" |
32 | #include "llvm/BinaryFormat/ELF.h" |
33 | #include "llvm/BinaryFormat/MachO.h" |
34 | #include "llvm/CodeGen/AsmPrinter.h" |
35 | #include "llvm/CodeGen/FaultMaps.h" |
36 | #include "llvm/CodeGen/MachineBasicBlock.h" |
37 | #include "llvm/CodeGen/MachineFunction.h" |
38 | #include "llvm/CodeGen/MachineInstr.h" |
39 | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
40 | #include "llvm/CodeGen/MachineModuleInfoImpls.h" |
41 | #include "llvm/CodeGen/MachineOperand.h" |
42 | #include "llvm/CodeGen/StackMaps.h" |
43 | #include "llvm/CodeGen/TargetRegisterInfo.h" |
44 | #include "llvm/IR/DataLayout.h" |
45 | #include "llvm/IR/DebugInfoMetadata.h" |
46 | #include "llvm/MC/MCAsmInfo.h" |
47 | #include "llvm/MC/MCContext.h" |
48 | #include "llvm/MC/MCInst.h" |
49 | #include "llvm/MC/MCInstBuilder.h" |
50 | #include "llvm/MC/MCSectionELF.h" |
51 | #include "llvm/MC/MCSectionMachO.h" |
52 | #include "llvm/MC/MCStreamer.h" |
53 | #include "llvm/MC/MCSymbol.h" |
54 | #include "llvm/MC/TargetRegistry.h" |
55 | #include "llvm/Support/Casting.h" |
56 | #include "llvm/Support/CommandLine.h" |
57 | #include "llvm/Support/ErrorHandling.h" |
58 | #include "llvm/Support/raw_ostream.h" |
59 | #include "llvm/Target/TargetMachine.h" |
60 | #include "llvm/TargetParser/Triple.h" |
61 | #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" |
62 | #include <algorithm> |
63 | #include <cassert> |
64 | #include <cstdint> |
65 | #include <map> |
66 | #include <memory> |
67 | |
68 | using namespace llvm; |
69 | |
70 | #define DEBUG_TYPE "asm-printer" |
71 | |
72 | namespace { |
73 | |
74 | class AArch64AsmPrinter : public AsmPrinter { |
75 | AArch64MCInstLower MCInstLowering; |
76 | FaultMaps FM; |
77 | const AArch64Subtarget *STI; |
78 | bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false; |
79 | |
80 | public: |
81 | AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) |
82 | : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this), |
83 | FM(*this) {} |
84 | |
85 | StringRef getPassName() const override { return "AArch64 Assembly Printer" ; } |
86 | |
87 | /// Wrapper for MCInstLowering.lowerOperand() for the |
88 | /// tblgen'erated pseudo lowering. |
89 | bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const { |
90 | return MCInstLowering.lowerOperand(MO, MCOp); |
91 | } |
92 | |
93 | void emitStartOfAsmFile(Module &M) override; |
94 | void emitJumpTableInfo() override; |
95 | std::tuple<const MCSymbol *, uint64_t, const MCSymbol *, |
96 | codeview::JumpTableEntrySize> |
97 | getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, |
98 | const MCSymbol *BranchLabel) const override; |
99 | |
100 | void emitFunctionEntryLabel() override; |
101 | |
102 | void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI); |
103 | |
104 | void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI); |
105 | |
106 | void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, |
107 | const MachineInstr &MI); |
108 | void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, |
109 | const MachineInstr &MI); |
110 | void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, |
111 | const MachineInstr &MI); |
112 | void LowerFAULTING_OP(const MachineInstr &MI); |
113 | |
114 | void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); |
115 | void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); |
116 | void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); |
117 | void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed); |
118 | |
119 | typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t> |
120 | HwasanMemaccessTuple; |
121 | std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols; |
122 | void LowerKCFI_CHECK(const MachineInstr &MI); |
123 | void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI); |
124 | void emitHwasanMemaccessSymbols(Module &M); |
125 | |
126 | void emitSled(const MachineInstr &MI, SledKind Kind); |
127 | |
128 | /// tblgen'erated driver function for lowering simple MI->MC |
129 | /// pseudo instructions. |
130 | bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, |
131 | const MachineInstr *MI); |
132 | |
133 | void emitInstruction(const MachineInstr *MI) override; |
134 | |
135 | void emitFunctionHeaderComment() override; |
136 | |
137 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
138 | AsmPrinter::getAnalysisUsage(AU); |
139 | AU.setPreservesAll(); |
140 | } |
141 | |
142 | bool runOnMachineFunction(MachineFunction &MF) override { |
143 | AArch64FI = MF.getInfo<AArch64FunctionInfo>(); |
144 | STI = &MF.getSubtarget<AArch64Subtarget>(); |
145 | |
146 | SetupMachineFunction(MF); |
147 | |
148 | if (STI->isTargetCOFF()) { |
149 | bool Local = MF.getFunction().hasLocalLinkage(); |
150 | COFF::SymbolStorageClass Scl = |
151 | Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL; |
152 | int Type = |
153 | COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT; |
154 | |
155 | OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym); |
156 | OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl); |
157 | OutStreamer->emitCOFFSymbolType(Type); |
158 | OutStreamer->endCOFFSymbolDef(); |
159 | } |
160 | |
161 | // Emit the rest of the function body. |
162 | emitFunctionBody(); |
163 | |
164 | // Emit the XRay table for this function. |
165 | emitXRayTable(); |
166 | |
167 | // We didn't modify anything. |
168 | return false; |
169 | } |
170 | |
171 | const MCExpr *lowerConstant(const Constant *CV) override; |
172 | |
173 | private: |
174 | void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O); |
175 | bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O); |
176 | bool printAsmRegInClass(const MachineOperand &MO, |
177 | const TargetRegisterClass *RC, unsigned AltName, |
178 | raw_ostream &O); |
179 | |
180 | bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, |
181 | const char *, raw_ostream &O) override; |
182 | bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum, |
183 | const char *, raw_ostream &O) override; |
184 | |
185 | void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS); |
186 | |
187 | void emitFunctionBodyEnd() override; |
188 | |
189 | MCSymbol *GetCPISymbol(unsigned CPID) const override; |
190 | void emitEndOfAsmFile(Module &M) override; |
191 | |
192 | AArch64FunctionInfo *AArch64FI = nullptr; |
193 | |
194 | /// Emit the LOHs contained in AArch64FI. |
195 | void emitLOHs(); |
196 | |
197 | /// Emit instruction to set float register to zero. |
198 | void emitFMov0(const MachineInstr &MI); |
199 | |
200 | using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; |
201 | |
202 | MInstToMCSymbol LOHInstToLabel; |
203 | |
204 | bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { |
205 | return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; |
206 | } |
207 | |
208 | const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override { |
209 | assert(STI); |
210 | return STI; |
211 | } |
212 | void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, |
213 | MCSymbol *LazyPointer) override; |
214 | void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, |
215 | MCSymbol *LazyPointer) override; |
216 | }; |
217 | |
218 | } // end anonymous namespace |
219 | |
220 | void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) { |
221 | const Triple &TT = TM.getTargetTriple(); |
222 | |
223 | if (TT.isOSBinFormatCOFF()) { |
224 | // Emit an absolute @feat.00 symbol |
225 | MCSymbol *S = MMI->getContext().getOrCreateSymbol(Name: StringRef("@feat.00" )); |
226 | OutStreamer->beginCOFFSymbolDef(Symbol: S); |
227 | OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_STATIC); |
228 | OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_NULL); |
229 | OutStreamer->endCOFFSymbolDef(); |
230 | int64_t Feat00Value = 0; |
231 | |
232 | if (M.getModuleFlag(Key: "cfguard" )) { |
233 | // Object is CFG-aware. |
234 | Feat00Value |= COFF::Feat00Flags::GuardCF; |
235 | } |
236 | |
237 | if (M.getModuleFlag(Key: "ehcontguard" )) { |
238 | // Object also has EHCont. |
239 | Feat00Value |= COFF::Feat00Flags::GuardEHCont; |
240 | } |
241 | |
242 | if (M.getModuleFlag(Key: "ms-kernel" )) { |
243 | // Object is compiled with /kernel. |
244 | Feat00Value |= COFF::Feat00Flags::Kernel; |
245 | } |
246 | |
247 | OutStreamer->emitSymbolAttribute(Symbol: S, Attribute: MCSA_Global); |
248 | OutStreamer->emitAssignment( |
249 | Symbol: S, Value: MCConstantExpr::create(Value: Feat00Value, Ctx&: MMI->getContext())); |
250 | } |
251 | |
252 | if (!TT.isOSBinFormatELF()) |
253 | return; |
254 | |
255 | // Assemble feature flags that may require creation of a note section. |
256 | unsigned Flags = 0; |
257 | if (const auto *BTE = mdconst::extract_or_null<ConstantInt>( |
258 | MD: M.getModuleFlag(Key: "branch-target-enforcement" ))) |
259 | if (BTE->getZExtValue()) |
260 | Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI; |
261 | |
262 | if (const auto *GCS = mdconst::extract_or_null<ConstantInt>( |
263 | MD: M.getModuleFlag(Key: "guarded-control-stack" ))) |
264 | if (GCS->getZExtValue()) |
265 | Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS; |
266 | |
267 | if (const auto *Sign = mdconst::extract_or_null<ConstantInt>( |
268 | MD: M.getModuleFlag(Key: "sign-return-address" ))) |
269 | if (Sign->getZExtValue()) |
270 | Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC; |
271 | |
272 | uint64_t PAuthABIPlatform = -1; |
273 | if (const auto *PAP = mdconst::extract_or_null<ConstantInt>( |
274 | MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform" ))) |
275 | PAuthABIPlatform = PAP->getZExtValue(); |
276 | uint64_t PAuthABIVersion = -1; |
277 | if (const auto *PAV = mdconst::extract_or_null<ConstantInt>( |
278 | MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version" ))) |
279 | PAuthABIVersion = PAV->getZExtValue(); |
280 | |
281 | // Emit a .note.gnu.property section with the flags. |
282 | auto *TS = |
283 | static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); |
284 | TS->emitNoteSection(Flags, PAuthABIPlatform, PAuthABIVersion); |
285 | } |
286 | |
287 | void AArch64AsmPrinter::() { |
288 | const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>(); |
289 | std::optional<std::string> OutlinerString = FI->getOutliningStyle(); |
290 | if (OutlinerString != std::nullopt) |
291 | OutStreamer->getCommentOS() << ' ' << OutlinerString; |
292 | } |
293 | |
294 | void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) |
295 | { |
296 | const Function &F = MF->getFunction(); |
297 | if (F.hasFnAttribute(Kind: "patchable-function-entry" )) { |
298 | unsigned Num; |
299 | if (F.getFnAttribute(Kind: "patchable-function-entry" ) |
300 | .getValueAsString() |
301 | .getAsInteger(Radix: 10, Result&: Num)) |
302 | return; |
303 | emitNops(N: Num); |
304 | return; |
305 | } |
306 | |
307 | emitSled(MI, Kind: SledKind::FUNCTION_ENTER); |
308 | } |
309 | |
310 | void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) { |
311 | emitSled(MI, Kind: SledKind::FUNCTION_EXIT); |
312 | } |
313 | |
314 | void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) { |
315 | emitSled(MI, Kind: SledKind::TAIL_CALL); |
316 | } |
317 | |
318 | void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) { |
319 | static const int8_t NoopsInSledCount = 7; |
320 | // We want to emit the following pattern: |
321 | // |
322 | // .Lxray_sled_N: |
323 | // ALIGN |
324 | // B #32 |
325 | // ; 7 NOP instructions (28 bytes) |
326 | // .tmpN |
327 | // |
328 | // We need the 28 bytes (7 instructions) because at runtime, we'd be patching |
329 | // over the full 32 bytes (8 instructions) with the following pattern: |
330 | // |
331 | // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack |
332 | // LDR W17, #12 ; W17 := function ID |
333 | // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit |
334 | // BLR X16 ; call the tracing trampoline |
335 | // ;DATA: 32 bits of function ID |
336 | // ;DATA: lower 32 bits of the address of the trampoline |
337 | // ;DATA: higher 32 bits of the address of the trampoline |
338 | // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack |
339 | // |
340 | OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: &getSubtargetInfo()); |
341 | auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_" , AlwaysAddSuffix: true); |
342 | OutStreamer->emitLabel(Symbol: CurSled); |
343 | auto Target = OutContext.createTempSymbol(); |
344 | |
345 | // Emit "B #32" instruction, which jumps over the next 28 bytes. |
346 | // The operand has to be the number of 4-byte instructions to jump over, |
347 | // including the current instruction. |
348 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8)); |
349 | |
350 | for (int8_t I = 0; I < NoopsInSledCount; I++) |
351 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
352 | |
353 | OutStreamer->emitLabel(Symbol: Target); |
354 | recordSled(Sled: CurSled, MI, Kind, Version: 2); |
355 | } |
356 | |
357 | // Emit the following code for Intrinsic::{xray_customevent,xray_typedevent} |
358 | // (built-in functions __xray_customevent/__xray_typedevent). |
359 | // |
360 | // .Lxray_event_sled_N: |
361 | // b 1f |
362 | // save x0 and x1 (and also x2 for TYPED_EVENT_CALL) |
363 | // set up x0 and x1 (and also x2 for TYPED_EVENT_CALL) |
364 | // bl __xray_CustomEvent or __xray_TypedEvent |
365 | // restore x0 and x1 (and also x2 for TYPED_EVENT_CALL) |
366 | // 1: |
367 | // |
368 | // There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL. |
369 | // |
370 | // Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT. |
371 | // After patching, b .+N will become a nop. |
372 | void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, |
373 | bool Typed) { |
374 | auto &O = *OutStreamer; |
375 | MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_" , AlwaysAddSuffix: true); |
376 | O.emitLabel(Symbol: CurSled); |
377 | MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs) |
378 | .addReg(AArch64::X0) |
379 | .addReg(AArch64::XZR) |
380 | .addReg(MI.getOperand(0).getReg()) |
381 | .addImm(0); |
382 | MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs) |
383 | .addReg(AArch64::X1) |
384 | .addReg(AArch64::XZR) |
385 | .addReg(MI.getOperand(1).getReg()) |
386 | .addImm(0); |
387 | bool MachO = TM.getTargetTriple().isOSBinFormatMachO(); |
388 | auto *Sym = MCSymbolRefExpr::create( |
389 | Symbol: OutContext.getOrCreateSymbol( |
390 | Name: Twine(MachO ? "_" : "" ) + |
391 | (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent" )), |
392 | Ctx&: OutContext); |
393 | if (Typed) { |
394 | O.AddComment(T: "Begin XRay typed event" ); |
395 | EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9)); |
396 | EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre) |
397 | .addReg(AArch64::SP) |
398 | .addReg(AArch64::X0) |
399 | .addReg(AArch64::X1) |
400 | .addReg(AArch64::SP) |
401 | .addImm(-4)); |
402 | EmitToStreamer(O, MCInstBuilder(AArch64::STRXui) |
403 | .addReg(AArch64::X2) |
404 | .addReg(AArch64::SP) |
405 | .addImm(2)); |
406 | EmitToStreamer(S&: O, Inst: MovX0Op0); |
407 | EmitToStreamer(S&: O, Inst: MovX1Op1); |
408 | EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs) |
409 | .addReg(AArch64::X2) |
410 | .addReg(AArch64::XZR) |
411 | .addReg(MI.getOperand(2).getReg()) |
412 | .addImm(0)); |
413 | EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym)); |
414 | EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui) |
415 | .addReg(AArch64::X2) |
416 | .addReg(AArch64::SP) |
417 | .addImm(2)); |
418 | O.AddComment(T: "End XRay typed event" ); |
419 | EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost) |
420 | .addReg(AArch64::SP) |
421 | .addReg(AArch64::X0) |
422 | .addReg(AArch64::X1) |
423 | .addReg(AArch64::SP) |
424 | .addImm(4)); |
425 | |
426 | recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2); |
427 | } else { |
428 | O.AddComment(T: "Begin XRay custom event" ); |
429 | EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6)); |
430 | EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre) |
431 | .addReg(AArch64::SP) |
432 | .addReg(AArch64::X0) |
433 | .addReg(AArch64::X1) |
434 | .addReg(AArch64::SP) |
435 | .addImm(-2)); |
436 | EmitToStreamer(S&: O, Inst: MovX0Op0); |
437 | EmitToStreamer(S&: O, Inst: MovX1Op1); |
438 | EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym)); |
439 | O.AddComment(T: "End XRay custom event" ); |
440 | EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost) |
441 | .addReg(AArch64::SP) |
442 | .addReg(AArch64::X0) |
443 | .addReg(AArch64::X1) |
444 | .addReg(AArch64::SP) |
445 | .addImm(2)); |
446 | |
447 | recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2); |
448 | } |
449 | } |
450 | |
451 | void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) { |
452 | Register AddrReg = MI.getOperand(i: 0).getReg(); |
453 | assert(std::next(MI.getIterator())->isCall() && |
454 | "KCFI_CHECK not followed by a call instruction" ); |
455 | assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg && |
456 | "KCFI_CHECK call target doesn't match call operand" ); |
457 | |
458 | // Default to using the intra-procedure-call temporary registers for |
459 | // comparing the hashes. |
460 | unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17}; |
461 | if (AddrReg == AArch64::XZR) { |
462 | // Checking XZR makes no sense. Instead of emitting a load, zero |
463 | // ScratchRegs[0] and use it for the ESR AddrIndex below. |
464 | AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]); |
465 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs) |
466 | .addReg(AddrReg) |
467 | .addReg(AArch64::XZR) |
468 | .addReg(AArch64::XZR) |
469 | .addImm(0)); |
470 | } else { |
471 | // If one of the scratch registers is used for the call target (e.g. |
472 | // with AArch64::TCRETURNriBTI), we can clobber another caller-saved |
473 | // temporary register instead (in this case, AArch64::W9) as the check |
474 | // is immediately followed by the call instruction. |
475 | for (auto &Reg : ScratchRegs) { |
476 | if (Reg == getWRegFromXReg(AddrReg)) { |
477 | Reg = AArch64::W9; |
478 | break; |
479 | } |
480 | } |
481 | assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg && |
482 | "Invalid scratch registers for KCFI_CHECK" ); |
483 | |
484 | // Adjust the offset for patchable-function-prefix. This assumes that |
485 | // patchable-function-prefix is the same for all functions. |
486 | int64_t PrefixNops = 0; |
487 | (void)MI.getMF() |
488 | ->getFunction() |
489 | .getFnAttribute(Kind: "patchable-function-prefix" ) |
490 | .getValueAsString() |
491 | .getAsInteger(Radix: 10, Result&: PrefixNops); |
492 | |
493 | // Load the target function type hash. |
494 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi) |
495 | .addReg(ScratchRegs[0]) |
496 | .addReg(AddrReg) |
497 | .addImm(-(PrefixNops * 4 + 4))); |
498 | } |
499 | |
500 | // Load the expected type hash. |
501 | const int64_t Type = MI.getOperand(i: 1).getImm(); |
502 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) |
503 | .addReg(ScratchRegs[1]) |
504 | .addReg(ScratchRegs[1]) |
505 | .addImm(Type & 0xFFFF) |
506 | .addImm(0)); |
507 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi) |
508 | .addReg(ScratchRegs[1]) |
509 | .addReg(ScratchRegs[1]) |
510 | .addImm((Type >> 16) & 0xFFFF) |
511 | .addImm(16)); |
512 | |
513 | // Compare the hashes and trap if there's a mismatch. |
514 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs) |
515 | .addReg(AArch64::WZR) |
516 | .addReg(ScratchRegs[0]) |
517 | .addReg(ScratchRegs[1]) |
518 | .addImm(0)); |
519 | |
520 | MCSymbol *Pass = OutContext.createTempSymbol(); |
521 | EmitToStreamer(*OutStreamer, |
522 | MCInstBuilder(AArch64::Bcc) |
523 | .addImm(AArch64CC::EQ) |
524 | .addExpr(MCSymbolRefExpr::create(Pass, OutContext))); |
525 | |
526 | // The base ESR is 0x8000 and the register information is encoded in bits |
527 | // 0-9 as follows: |
528 | // - 0-4: n, where the register Xn contains the target address |
529 | // - 5-9: m, where the register Wm contains the expected type hash |
530 | // Where n, m are in [0, 30]. |
531 | unsigned TypeIndex = ScratchRegs[1] - AArch64::W0; |
532 | unsigned AddrIndex; |
533 | switch (AddrReg) { |
534 | default: |
535 | AddrIndex = AddrReg - AArch64::X0; |
536 | break; |
537 | case AArch64::FP: |
538 | AddrIndex = 29; |
539 | break; |
540 | case AArch64::LR: |
541 | AddrIndex = 30; |
542 | break; |
543 | } |
544 | |
545 | assert(AddrIndex < 31 && TypeIndex < 31); |
546 | |
547 | unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31); |
548 | EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR)); |
549 | OutStreamer->emitLabel(Symbol: Pass); |
550 | } |
551 | |
552 | void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) { |
553 | Register Reg = MI.getOperand(i: 0).getReg(); |
554 | bool IsShort = |
555 | ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) || |
556 | (MI.getOpcode() == |
557 | AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW)); |
558 | uint32_t AccessInfo = MI.getOperand(i: 1).getImm(); |
559 | bool IsFixedShadow = |
560 | ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) || |
561 | (MI.getOpcode() == |
562 | AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW)); |
563 | uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0; |
564 | |
565 | MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple( |
566 | Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)]; |
567 | if (!Sym) { |
568 | // FIXME: Make this work on non-ELF. |
569 | if (!TM.getTargetTriple().isOSBinFormatELF()) |
570 | report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF" ); |
571 | |
572 | std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" + |
573 | utostr(AccessInfo); |
574 | if (IsFixedShadow) |
575 | SymName += "_fixed_" + utostr(X: FixedShadowOffset); |
576 | if (IsShort) |
577 | SymName += "_short_v2" ; |
578 | Sym = OutContext.getOrCreateSymbol(Name: SymName); |
579 | } |
580 | |
581 | EmitToStreamer(*OutStreamer, |
582 | MCInstBuilder(AArch64::BL) |
583 | .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); |
584 | } |
585 | |
586 | void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) { |
587 | if (HwasanMemaccessSymbols.empty()) |
588 | return; |
589 | |
590 | const Triple &TT = TM.getTargetTriple(); |
591 | assert(TT.isOSBinFormatELF()); |
592 | std::unique_ptr<MCSubtargetInfo> STI( |
593 | TM.getTarget().createMCSubtargetInfo(TheTriple: TT.str(), CPU: "" , Features: "" )); |
594 | assert(STI && "Unable to create subtarget info" ); |
595 | |
596 | MCSymbol *HwasanTagMismatchV1Sym = |
597 | OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch" ); |
598 | MCSymbol *HwasanTagMismatchV2Sym = |
599 | OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2" ); |
600 | |
601 | const MCSymbolRefExpr *HwasanTagMismatchV1Ref = |
602 | MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext); |
603 | const MCSymbolRefExpr *HwasanTagMismatchV2Ref = |
604 | MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext); |
605 | |
606 | for (auto &P : HwasanMemaccessSymbols) { |
607 | unsigned Reg = std::get<0>(t: P.first); |
608 | bool IsShort = std::get<1>(t: P.first); |
609 | uint32_t AccessInfo = std::get<2>(t: P.first); |
610 | bool IsFixedShadow = std::get<3>(t: P.first); |
611 | uint64_t FixedShadowOffset = std::get<4>(t: P.first); |
612 | const MCSymbolRefExpr *HwasanTagMismatchRef = |
613 | IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref; |
614 | MCSymbol *Sym = P.second; |
615 | |
616 | bool HasMatchAllTag = |
617 | (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1; |
618 | uint8_t MatchAllTag = |
619 | (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff; |
620 | unsigned Size = |
621 | 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf); |
622 | bool CompileKernel = |
623 | (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1; |
624 | |
625 | OutStreamer->switchSection(Section: OutContext.getELFSection( |
626 | Section: ".text.hot" , Type: ELF::SHT_PROGBITS, |
627 | Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(), |
628 | /*IsComdat=*/true)); |
629 | |
630 | OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction); |
631 | OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak); |
632 | OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden); |
633 | OutStreamer->emitLabel(Symbol: Sym); |
634 | |
635 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri) |
636 | .addReg(AArch64::X16) |
637 | .addReg(Reg) |
638 | .addImm(4) |
639 | .addImm(55), |
640 | *STI); |
641 | |
642 | if (IsFixedShadow) { |
643 | // Aarch64 makes it difficult to embed large constants in the code. |
644 | // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit |
645 | // left-shift option in the MOV instruction. Combined with the 16-bit |
646 | // immediate, this is enough to represent any offset up to 2**48. |
647 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi) |
648 | .addReg(AArch64::X17) |
649 | .addImm(FixedShadowOffset >> 32) |
650 | .addImm(32), |
651 | *STI); |
652 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX) |
653 | .addReg(AArch64::W16) |
654 | .addReg(AArch64::X17) |
655 | .addReg(AArch64::X16) |
656 | .addImm(0) |
657 | .addImm(0), |
658 | *STI); |
659 | } else { |
660 | OutStreamer->emitInstruction( |
661 | MCInstBuilder(AArch64::LDRBBroX) |
662 | .addReg(AArch64::W16) |
663 | .addReg(IsShort ? AArch64::X20 : AArch64::X9) |
664 | .addReg(AArch64::X16) |
665 | .addImm(0) |
666 | .addImm(0), |
667 | *STI); |
668 | } |
669 | |
670 | OutStreamer->emitInstruction( |
671 | MCInstBuilder(AArch64::SUBSXrs) |
672 | .addReg(AArch64::XZR) |
673 | .addReg(AArch64::X16) |
674 | .addReg(Reg) |
675 | .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), |
676 | *STI); |
677 | MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol(); |
678 | OutStreamer->emitInstruction( |
679 | MCInstBuilder(AArch64::Bcc) |
680 | .addImm(AArch64CC::NE) |
681 | .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym, |
682 | OutContext)), |
683 | *STI); |
684 | MCSymbol *ReturnSym = OutContext.createTempSymbol(); |
685 | OutStreamer->emitLabel(Symbol: ReturnSym); |
686 | OutStreamer->emitInstruction( |
687 | MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI); |
688 | OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym); |
689 | |
690 | if (HasMatchAllTag) { |
691 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri) |
692 | .addReg(AArch64::X17) |
693 | .addReg(Reg) |
694 | .addImm(56) |
695 | .addImm(63), |
696 | *STI); |
697 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri) |
698 | .addReg(AArch64::XZR) |
699 | .addReg(AArch64::X17) |
700 | .addImm(MatchAllTag) |
701 | .addImm(0), |
702 | *STI); |
703 | OutStreamer->emitInstruction( |
704 | MCInstBuilder(AArch64::Bcc) |
705 | .addImm(AArch64CC::EQ) |
706 | .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), |
707 | *STI); |
708 | } |
709 | |
710 | if (IsShort) { |
711 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri) |
712 | .addReg(AArch64::WZR) |
713 | .addReg(AArch64::W16) |
714 | .addImm(15) |
715 | .addImm(0), |
716 | *STI); |
717 | MCSymbol *HandleMismatchSym = OutContext.createTempSymbol(); |
718 | OutStreamer->emitInstruction( |
719 | MCInstBuilder(AArch64::Bcc) |
720 | .addImm(AArch64CC::HI) |
721 | .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), |
722 | *STI); |
723 | |
724 | OutStreamer->emitInstruction( |
725 | MCInstBuilder(AArch64::ANDXri) |
726 | .addReg(AArch64::X17) |
727 | .addReg(Reg) |
728 | .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), |
729 | *STI); |
730 | if (Size != 1) |
731 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) |
732 | .addReg(AArch64::X17) |
733 | .addReg(AArch64::X17) |
734 | .addImm(Size - 1) |
735 | .addImm(0), |
736 | *STI); |
737 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs) |
738 | .addReg(AArch64::WZR) |
739 | .addReg(AArch64::W16) |
740 | .addReg(AArch64::W17) |
741 | .addImm(0), |
742 | *STI); |
743 | OutStreamer->emitInstruction( |
744 | MCInstBuilder(AArch64::Bcc) |
745 | .addImm(AArch64CC::LS) |
746 | .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)), |
747 | *STI); |
748 | |
749 | OutStreamer->emitInstruction( |
750 | MCInstBuilder(AArch64::ORRXri) |
751 | .addReg(AArch64::X16) |
752 | .addReg(Reg) |
753 | .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)), |
754 | *STI); |
755 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui) |
756 | .addReg(AArch64::W16) |
757 | .addReg(AArch64::X16) |
758 | .addImm(0), |
759 | *STI); |
760 | OutStreamer->emitInstruction( |
761 | MCInstBuilder(AArch64::SUBSXrs) |
762 | .addReg(AArch64::XZR) |
763 | .addReg(AArch64::X16) |
764 | .addReg(Reg) |
765 | .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)), |
766 | *STI); |
767 | OutStreamer->emitInstruction( |
768 | MCInstBuilder(AArch64::Bcc) |
769 | .addImm(AArch64CC::EQ) |
770 | .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)), |
771 | *STI); |
772 | |
773 | OutStreamer->emitLabel(Symbol: HandleMismatchSym); |
774 | } |
775 | |
776 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) |
777 | .addReg(AArch64::SP) |
778 | .addReg(AArch64::X0) |
779 | .addReg(AArch64::X1) |
780 | .addReg(AArch64::SP) |
781 | .addImm(-32), |
782 | *STI); |
783 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi) |
784 | .addReg(AArch64::FP) |
785 | .addReg(AArch64::LR) |
786 | .addReg(AArch64::SP) |
787 | .addImm(29), |
788 | *STI); |
789 | |
790 | if (Reg != AArch64::X0) |
791 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs) |
792 | .addReg(AArch64::X0) |
793 | .addReg(AArch64::XZR) |
794 | .addReg(Reg) |
795 | .addImm(0), |
796 | *STI); |
797 | OutStreamer->emitInstruction( |
798 | MCInstBuilder(AArch64::MOVZXi) |
799 | .addReg(AArch64::X1) |
800 | .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask) |
801 | .addImm(0), |
802 | *STI); |
803 | |
804 | if (CompileKernel) { |
805 | // The Linux kernel's dynamic loader doesn't support GOT relative |
806 | // relocations, but it doesn't support late binding either, so just call |
807 | // the function directly. |
808 | OutStreamer->emitInstruction( |
809 | MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI); |
810 | } else { |
811 | // Intentionally load the GOT entry and branch to it, rather than possibly |
812 | // late binding the function, which may clobber the registers before we |
813 | // have a chance to save them. |
814 | OutStreamer->emitInstruction( |
815 | MCInstBuilder(AArch64::ADRP) |
816 | .addReg(AArch64::X16) |
817 | .addExpr(AArch64MCExpr::create( |
818 | HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE, |
819 | OutContext)), |
820 | *STI); |
821 | OutStreamer->emitInstruction( |
822 | MCInstBuilder(AArch64::LDRXui) |
823 | .addReg(AArch64::X16) |
824 | .addReg(AArch64::X16) |
825 | .addExpr(AArch64MCExpr::create( |
826 | HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12, |
827 | OutContext)), |
828 | *STI); |
829 | OutStreamer->emitInstruction( |
830 | MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI); |
831 | } |
832 | } |
833 | } |
834 | |
835 | void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) { |
836 | emitHwasanMemaccessSymbols(M); |
837 | |
838 | const Triple &TT = TM.getTargetTriple(); |
839 | if (TT.isOSBinFormatMachO()) { |
840 | // Funny Darwin hack: This flag tells the linker that no global symbols |
841 | // contain code that falls through to other global symbols (e.g. the obvious |
842 | // implementation of multiple entry points). If this doesn't occur, the |
843 | // linker can safely perform dead code stripping. Since LLVM never |
844 | // generates code that does this, it is always safe to set. |
845 | OutStreamer->emitAssemblerFlag(Flag: MCAF_SubsectionsViaSymbols); |
846 | } |
847 | |
848 | // Emit stack and fault map information. |
849 | FM.serializeToFaultMapSection(); |
850 | |
851 | } |
852 | |
853 | void AArch64AsmPrinter::emitLOHs() { |
854 | SmallVector<MCSymbol *, 3> MCArgs; |
855 | |
856 | for (const auto &D : AArch64FI->getLOHContainer()) { |
857 | for (const MachineInstr *MI : D.getArgs()) { |
858 | MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI); |
859 | assert(LabelIt != LOHInstToLabel.end() && |
860 | "Label hasn't been inserted for LOH related instruction" ); |
861 | MCArgs.push_back(Elt: LabelIt->second); |
862 | } |
863 | OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs); |
864 | MCArgs.clear(); |
865 | } |
866 | } |
867 | |
868 | void AArch64AsmPrinter::emitFunctionBodyEnd() { |
869 | if (!AArch64FI->getLOHRelated().empty()) |
870 | emitLOHs(); |
871 | } |
872 | |
873 | /// GetCPISymbol - Return the symbol for the specified constant pool entry. |
874 | MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const { |
875 | // Darwin uses a linker-private symbol name for constant-pools (to |
876 | // avoid addends on the relocation?), ELF has no such concept and |
877 | // uses a normal private symbol. |
878 | if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty()) |
879 | return OutContext.getOrCreateSymbol( |
880 | Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" + |
881 | Twine(getFunctionNumber()) + "_" + Twine(CPID)); |
882 | |
883 | return AsmPrinter::GetCPISymbol(CPID); |
884 | } |
885 | |
886 | void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum, |
887 | raw_ostream &O) { |
888 | const MachineOperand &MO = MI->getOperand(i: OpNum); |
889 | switch (MO.getType()) { |
890 | default: |
891 | llvm_unreachable("<unknown operand type>" ); |
892 | case MachineOperand::MO_Register: { |
893 | Register Reg = MO.getReg(); |
894 | assert(Reg.isPhysical()); |
895 | assert(!MO.getSubReg() && "Subregs should be eliminated!" ); |
896 | O << AArch64InstPrinter::getRegisterName(Reg); |
897 | break; |
898 | } |
899 | case MachineOperand::MO_Immediate: { |
900 | O << MO.getImm(); |
901 | break; |
902 | } |
903 | case MachineOperand::MO_GlobalAddress: { |
904 | PrintSymbolOperand(MO, OS&: O); |
905 | break; |
906 | } |
907 | case MachineOperand::MO_BlockAddress: { |
908 | MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress()); |
909 | Sym->print(OS&: O, MAI); |
910 | break; |
911 | } |
912 | } |
913 | } |
914 | |
915 | bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode, |
916 | raw_ostream &O) { |
917 | Register Reg = MO.getReg(); |
918 | switch (Mode) { |
919 | default: |
920 | return true; // Unknown mode. |
921 | case 'w': |
922 | Reg = getWRegFromXReg(Reg); |
923 | break; |
924 | case 'x': |
925 | Reg = getXRegFromWReg(Reg); |
926 | break; |
927 | case 't': |
928 | Reg = getXRegFromXRegTuple(RegTuple: Reg); |
929 | break; |
930 | } |
931 | |
932 | O << AArch64InstPrinter::getRegisterName(Reg); |
933 | return false; |
934 | } |
935 | |
936 | // Prints the register in MO using class RC using the offset in the |
937 | // new register class. This should not be used for cross class |
938 | // printing. |
939 | bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO, |
940 | const TargetRegisterClass *RC, |
941 | unsigned AltName, raw_ostream &O) { |
942 | assert(MO.isReg() && "Should only get here with a register!" ); |
943 | const TargetRegisterInfo *RI = STI->getRegisterInfo(); |
944 | Register Reg = MO.getReg(); |
945 | unsigned RegToPrint = RC->getRegister(i: RI->getEncodingValue(RegNo: Reg)); |
946 | if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg)) |
947 | return true; |
948 | O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName); |
949 | return false; |
950 | } |
951 | |
952 | bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum, |
953 | const char *, raw_ostream &O) { |
954 | const MachineOperand &MO = MI->getOperand(i: OpNum); |
955 | |
956 | // First try the generic code, which knows about modifiers like 'c' and 'n'. |
957 | if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O)) |
958 | return false; |
959 | |
960 | // Does this asm operand have a single letter operand modifier? |
961 | if (ExtraCode && ExtraCode[0]) { |
962 | if (ExtraCode[1] != 0) |
963 | return true; // Unknown modifier. |
964 | |
965 | switch (ExtraCode[0]) { |
966 | default: |
967 | return true; // Unknown modifier. |
968 | case 'w': // Print W register |
969 | case 'x': // Print X register |
970 | if (MO.isReg()) |
971 | return printAsmMRegister(MO, Mode: ExtraCode[0], O); |
972 | if (MO.isImm() && MO.getImm() == 0) { |
973 | unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR; |
974 | O << AArch64InstPrinter::getRegisterName(Reg); |
975 | return false; |
976 | } |
977 | printOperand(MI, OpNum, O); |
978 | return false; |
979 | case 'b': // Print B register. |
980 | case 'h': // Print H register. |
981 | case 's': // Print S register. |
982 | case 'd': // Print D register. |
983 | case 'q': // Print Q register. |
984 | case 'z': // Print Z register. |
985 | if (MO.isReg()) { |
986 | const TargetRegisterClass *RC; |
987 | switch (ExtraCode[0]) { |
988 | case 'b': |
989 | RC = &AArch64::FPR8RegClass; |
990 | break; |
991 | case 'h': |
992 | RC = &AArch64::FPR16RegClass; |
993 | break; |
994 | case 's': |
995 | RC = &AArch64::FPR32RegClass; |
996 | break; |
997 | case 'd': |
998 | RC = &AArch64::FPR64RegClass; |
999 | break; |
1000 | case 'q': |
1001 | RC = &AArch64::FPR128RegClass; |
1002 | break; |
1003 | case 'z': |
1004 | RC = &AArch64::ZPRRegClass; |
1005 | break; |
1006 | default: |
1007 | return true; |
1008 | } |
1009 | return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O); |
1010 | } |
1011 | printOperand(MI, OpNum, O); |
1012 | return false; |
1013 | } |
1014 | } |
1015 | |
1016 | // According to ARM, we should emit x and v registers unless we have a |
1017 | // modifier. |
1018 | if (MO.isReg()) { |
1019 | Register Reg = MO.getReg(); |
1020 | |
1021 | // If this is a w or x register, print an x register. |
1022 | if (AArch64::GPR32allRegClass.contains(Reg) || |
1023 | AArch64::GPR64allRegClass.contains(Reg)) |
1024 | return printAsmMRegister(MO, Mode: 'x', O); |
1025 | |
1026 | // If this is an x register tuple, print an x register. |
1027 | if (AArch64::GPR64x8ClassRegClass.contains(Reg)) |
1028 | return printAsmMRegister(MO, Mode: 't', O); |
1029 | |
1030 | unsigned AltName = AArch64::NoRegAltName; |
1031 | const TargetRegisterClass *RegClass; |
1032 | if (AArch64::ZPRRegClass.contains(Reg)) { |
1033 | RegClass = &AArch64::ZPRRegClass; |
1034 | } else if (AArch64::PPRRegClass.contains(Reg)) { |
1035 | RegClass = &AArch64::PPRRegClass; |
1036 | } else if (AArch64::PNRRegClass.contains(Reg)) { |
1037 | RegClass = &AArch64::PNRRegClass; |
1038 | } else { |
1039 | RegClass = &AArch64::FPR128RegClass; |
1040 | AltName = AArch64::vreg; |
1041 | } |
1042 | |
1043 | // If this is a b, h, s, d, or q register, print it as a v register. |
1044 | return printAsmRegInClass(MO, RC: RegClass, AltName, O); |
1045 | } |
1046 | |
1047 | printOperand(MI, OpNum, O); |
1048 | return false; |
1049 | } |
1050 | |
1051 | bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, |
1052 | unsigned OpNum, |
1053 | const char *, |
1054 | raw_ostream &O) { |
1055 | if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a') |
1056 | return true; // Unknown modifier. |
1057 | |
1058 | const MachineOperand &MO = MI->getOperand(i: OpNum); |
1059 | assert(MO.isReg() && "unexpected inline asm memory operand" ); |
1060 | O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]" ; |
1061 | return false; |
1062 | } |
1063 | |
1064 | void AArch64AsmPrinter::(const MachineInstr *MI, |
1065 | raw_ostream &OS) { |
1066 | unsigned NOps = MI->getNumOperands(); |
1067 | assert(NOps == 4); |
1068 | OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: " ; |
1069 | // cast away const; DIetc do not take const operands for some reason. |
1070 | OS << MI->getDebugVariable()->getName(); |
1071 | OS << " <- " ; |
1072 | // Frame address. Currently handles register +- offset only. |
1073 | assert(MI->isIndirectDebugValue()); |
1074 | OS << '['; |
1075 | for (unsigned I = 0, E = std::distance(first: MI->debug_operands().begin(), |
1076 | last: MI->debug_operands().end()); |
1077 | I < E; ++I) { |
1078 | if (I != 0) |
1079 | OS << ", " ; |
1080 | printOperand(MI, OpNum: I, O&: OS); |
1081 | } |
1082 | OS << ']'; |
1083 | OS << "+" ; |
1084 | printOperand(MI, OpNum: NOps - 2, O&: OS); |
1085 | } |
1086 | |
1087 | void AArch64AsmPrinter::emitJumpTableInfo() { |
1088 | const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo(); |
1089 | if (!MJTI) return; |
1090 | |
1091 | const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); |
1092 | if (JT.empty()) return; |
1093 | |
1094 | const TargetLoweringObjectFile &TLOF = getObjFileLowering(); |
1095 | MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F: MF->getFunction(), TM); |
1096 | OutStreamer->switchSection(Section: ReadOnlySec); |
1097 | |
1098 | auto AFI = MF->getInfo<AArch64FunctionInfo>(); |
1099 | for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) { |
1100 | const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs; |
1101 | |
1102 | // If this jump table was deleted, ignore it. |
1103 | if (JTBBs.empty()) continue; |
1104 | |
1105 | unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI); |
1106 | emitAlignment(Alignment: Align(Size)); |
1107 | OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI)); |
1108 | |
1109 | const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI); |
1110 | const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext); |
1111 | |
1112 | for (auto *JTBB : JTBBs) { |
1113 | const MCExpr *Value = |
1114 | MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext); |
1115 | |
1116 | // Each entry is: |
1117 | // .byte/.hword (LBB - Lbase)>>2 |
1118 | // or plain: |
1119 | // .word LBB - Lbase |
1120 | Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext); |
1121 | if (Size != 4) |
1122 | Value = MCBinaryExpr::createLShr( |
1123 | LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext); |
1124 | |
1125 | OutStreamer->emitValue(Value, Size); |
1126 | } |
1127 | } |
1128 | } |
1129 | |
1130 | std::tuple<const MCSymbol *, uint64_t, const MCSymbol *, |
1131 | codeview::JumpTableEntrySize> |
1132 | AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI, |
1133 | const MachineInstr *BranchInstr, |
1134 | const MCSymbol *BranchLabel) const { |
1135 | const auto AFI = MF->getInfo<AArch64FunctionInfo>(); |
1136 | const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI); |
1137 | codeview::JumpTableEntrySize EntrySize; |
1138 | switch (AFI->getJumpTableEntrySize(Idx: JTI)) { |
1139 | case 1: |
1140 | EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft; |
1141 | break; |
1142 | case 2: |
1143 | EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft; |
1144 | break; |
1145 | case 4: |
1146 | EntrySize = codeview::JumpTableEntrySize::Int32; |
1147 | break; |
1148 | default: |
1149 | llvm_unreachable("Unexpected jump table entry size" ); |
1150 | } |
1151 | return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize); |
1152 | } |
1153 | |
1154 | void AArch64AsmPrinter::emitFunctionEntryLabel() { |
1155 | if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall || |
1156 | MF->getFunction().getCallingConv() == |
1157 | CallingConv::AArch64_SVE_VectorCall || |
1158 | MF->getInfo<AArch64FunctionInfo>()->isSVECC()) { |
1159 | auto *TS = |
1160 | static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); |
1161 | TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym); |
1162 | } |
1163 | |
1164 | if (TM.getTargetTriple().isWindowsArm64EC() && |
1165 | !MF->getFunction().hasLocalLinkage()) { |
1166 | // For ARM64EC targets, a function definition's name is mangled differently |
1167 | // from the normal symbol. We emit the alias from the unmangled symbol to |
1168 | // mangled symbol name here. |
1169 | if (MDNode *Unmangled = |
1170 | MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name" )) { |
1171 | AsmPrinter::emitFunctionEntryLabel(); |
1172 | |
1173 | if (MDNode *ECMangled = |
1174 | MF->getFunction().getMetadata(Kind: "arm64ec_ecmangled_name" )) { |
1175 | StringRef UnmangledStr = |
1176 | cast<MDString>(Val: Unmangled->getOperand(I: 0))->getString(); |
1177 | MCSymbol *UnmangledSym = |
1178 | MMI->getContext().getOrCreateSymbol(Name: UnmangledStr); |
1179 | StringRef ECMangledStr = |
1180 | cast<MDString>(Val: ECMangled->getOperand(I: 0))->getString(); |
1181 | MCSymbol *ECMangledSym = |
1182 | MMI->getContext().getOrCreateSymbol(Name: ECMangledStr); |
1183 | OutStreamer->emitSymbolAttribute(Symbol: UnmangledSym, Attribute: MCSA_WeakAntiDep); |
1184 | OutStreamer->emitAssignment( |
1185 | Symbol: UnmangledSym, |
1186 | Value: MCSymbolRefExpr::create(Symbol: ECMangledSym, Kind: MCSymbolRefExpr::VK_WEAKREF, |
1187 | Ctx&: MMI->getContext())); |
1188 | OutStreamer->emitSymbolAttribute(Symbol: ECMangledSym, Attribute: MCSA_WeakAntiDep); |
1189 | OutStreamer->emitAssignment( |
1190 | Symbol: ECMangledSym, |
1191 | Value: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Kind: MCSymbolRefExpr::VK_WEAKREF, |
1192 | Ctx&: MMI->getContext())); |
1193 | return; |
1194 | } else { |
1195 | StringRef UnmangledStr = |
1196 | cast<MDString>(Val: Unmangled->getOperand(I: 0))->getString(); |
1197 | MCSymbol *UnmangledSym = |
1198 | MMI->getContext().getOrCreateSymbol(Name: UnmangledStr); |
1199 | OutStreamer->emitSymbolAttribute(Symbol: UnmangledSym, Attribute: MCSA_WeakAntiDep); |
1200 | OutStreamer->emitAssignment( |
1201 | Symbol: UnmangledSym, |
1202 | Value: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Kind: MCSymbolRefExpr::VK_WEAKREF, |
1203 | Ctx&: MMI->getContext())); |
1204 | return; |
1205 | } |
1206 | } |
1207 | } |
1208 | |
1209 | return AsmPrinter::emitFunctionEntryLabel(); |
1210 | } |
1211 | |
1212 | /// Small jump tables contain an unsigned byte or half, representing the offset |
1213 | /// from the lowest-addressed possible destination to the desired basic |
1214 | /// block. Since all instructions are 4-byte aligned, this is further compressed |
1215 | /// by counting in instructions rather than bytes (i.e. divided by 4). So, to |
1216 | /// materialize the correct destination we need: |
1217 | /// |
1218 | /// adr xDest, .LBB0_0 |
1219 | /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh). |
1220 | /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries) |
1221 | void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer, |
1222 | const llvm::MachineInstr &MI) { |
1223 | Register DestReg = MI.getOperand(i: 0).getReg(); |
1224 | Register ScratchReg = MI.getOperand(i: 1).getReg(); |
1225 | Register ScratchRegW = |
1226 | STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32); |
1227 | Register TableReg = MI.getOperand(i: 2).getReg(); |
1228 | Register EntryReg = MI.getOperand(i: 3).getReg(); |
1229 | int JTIdx = MI.getOperand(i: 4).getIndex(); |
1230 | int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx); |
1231 | |
1232 | // This has to be first because the compression pass based its reachability |
1233 | // calculations on the start of the JumpTableDest instruction. |
1234 | auto Label = |
1235 | MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx); |
1236 | |
1237 | // If we don't already have a symbol to use as the base, use the ADR |
1238 | // instruction itself. |
1239 | if (!Label) { |
1240 | Label = MF->getContext().createTempSymbol(); |
1241 | AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label); |
1242 | OutStreamer.emitLabel(Symbol: Label); |
1243 | } |
1244 | |
1245 | auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext()); |
1246 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR) |
1247 | .addReg(DestReg) |
1248 | .addExpr(LabelExpr)); |
1249 | |
1250 | // Load the number of instruction-steps to offset from the label. |
1251 | unsigned LdrOpcode; |
1252 | switch (Size) { |
1253 | case 1: LdrOpcode = AArch64::LDRBBroX; break; |
1254 | case 2: LdrOpcode = AArch64::LDRHHroX; break; |
1255 | case 4: LdrOpcode = AArch64::LDRSWroX; break; |
1256 | default: |
1257 | llvm_unreachable("Unknown jump table size" ); |
1258 | } |
1259 | |
1260 | EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode) |
1261 | .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW) |
1262 | .addReg(Reg: TableReg) |
1263 | .addReg(Reg: EntryReg) |
1264 | .addImm(Val: 0) |
1265 | .addImm(Val: Size == 1 ? 0 : 1)); |
1266 | |
1267 | // Add to the already materialized base label address, multiplying by 4 if |
1268 | // compressed. |
1269 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs) |
1270 | .addReg(DestReg) |
1271 | .addReg(DestReg) |
1272 | .addReg(ScratchReg) |
1273 | .addImm(Size == 4 ? 0 : 2)); |
1274 | } |
1275 | |
1276 | void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer, |
1277 | const llvm::MachineInstr &MI) { |
1278 | unsigned Opcode = MI.getOpcode(); |
1279 | assert(STI->hasMOPS()); |
1280 | assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo); |
1281 | |
1282 | const auto Ops = [Opcode]() -> std::array<unsigned, 3> { |
1283 | if (Opcode == AArch64::MOPSMemoryCopyPseudo) |
1284 | return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE}; |
1285 | if (Opcode == AArch64::MOPSMemoryMovePseudo) |
1286 | return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE}; |
1287 | if (Opcode == AArch64::MOPSMemorySetPseudo) |
1288 | return {AArch64::SETP, AArch64::SETM, AArch64::SETE}; |
1289 | if (Opcode == AArch64::MOPSMemorySetTaggingPseudo) |
1290 | return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE}; |
1291 | llvm_unreachable("Unhandled memory operation pseudo" ); |
1292 | }(); |
1293 | const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo || |
1294 | Opcode == AArch64::MOPSMemorySetTaggingPseudo; |
1295 | |
1296 | for (auto Op : Ops) { |
1297 | int i = 0; |
1298 | auto MCIB = MCInstBuilder(Op); |
1299 | // Destination registers |
1300 | MCIB.addReg(Reg: MI.getOperand(i: i++).getReg()); |
1301 | MCIB.addReg(Reg: MI.getOperand(i: i++).getReg()); |
1302 | if (!IsSet) |
1303 | MCIB.addReg(Reg: MI.getOperand(i: i++).getReg()); |
1304 | // Input registers |
1305 | MCIB.addReg(Reg: MI.getOperand(i: i++).getReg()); |
1306 | MCIB.addReg(Reg: MI.getOperand(i: i++).getReg()); |
1307 | MCIB.addReg(Reg: MI.getOperand(i: i++).getReg()); |
1308 | |
1309 | EmitToStreamer(S&: OutStreamer, Inst: MCIB); |
1310 | } |
1311 | } |
1312 | |
1313 | void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, |
1314 | const MachineInstr &MI) { |
1315 | unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); |
1316 | |
1317 | auto &Ctx = OutStreamer.getContext(); |
1318 | MCSymbol *MILabel = Ctx.createTempSymbol(); |
1319 | OutStreamer.emitLabel(Symbol: MILabel); |
1320 | |
1321 | SM.recordStackMap(L: *MILabel, MI); |
1322 | assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!" ); |
1323 | |
1324 | // Scan ahead to trim the shadow. |
1325 | const MachineBasicBlock &MBB = *MI.getParent(); |
1326 | MachineBasicBlock::const_iterator MII(MI); |
1327 | ++MII; |
1328 | while (NumNOPBytes > 0) { |
1329 | if (MII == MBB.end() || MII->isCall() || |
1330 | MII->getOpcode() == AArch64::DBG_VALUE || |
1331 | MII->getOpcode() == TargetOpcode::PATCHPOINT || |
1332 | MII->getOpcode() == TargetOpcode::STACKMAP) |
1333 | break; |
1334 | ++MII; |
1335 | NumNOPBytes -= 4; |
1336 | } |
1337 | |
1338 | // Emit nops. |
1339 | for (unsigned i = 0; i < NumNOPBytes; i += 4) |
1340 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
1341 | } |
1342 | |
1343 | // Lower a patchpoint of the form: |
1344 | // [<def>], <id>, <numBytes>, <target>, <numArgs> |
1345 | void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, |
1346 | const MachineInstr &MI) { |
1347 | auto &Ctx = OutStreamer.getContext(); |
1348 | MCSymbol *MILabel = Ctx.createTempSymbol(); |
1349 | OutStreamer.emitLabel(Symbol: MILabel); |
1350 | SM.recordPatchPoint(L: *MILabel, MI); |
1351 | |
1352 | PatchPointOpers Opers(&MI); |
1353 | |
1354 | int64_t CallTarget = Opers.getCallTarget().getImm(); |
1355 | unsigned EncodedBytes = 0; |
1356 | if (CallTarget) { |
1357 | assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && |
1358 | "High 16 bits of call target should be zero." ); |
1359 | Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg(); |
1360 | EncodedBytes = 16; |
1361 | // Materialize the jump address: |
1362 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi) |
1363 | .addReg(ScratchReg) |
1364 | .addImm((CallTarget >> 32) & 0xFFFF) |
1365 | .addImm(32)); |
1366 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) |
1367 | .addReg(ScratchReg) |
1368 | .addReg(ScratchReg) |
1369 | .addImm((CallTarget >> 16) & 0xFFFF) |
1370 | .addImm(16)); |
1371 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi) |
1372 | .addReg(ScratchReg) |
1373 | .addReg(ScratchReg) |
1374 | .addImm(CallTarget & 0xFFFF) |
1375 | .addImm(0)); |
1376 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg)); |
1377 | } |
1378 | // Emit padding. |
1379 | unsigned NumBytes = Opers.getNumPatchBytes(); |
1380 | assert(NumBytes >= EncodedBytes && |
1381 | "Patchpoint can't request size less than the length of a call." ); |
1382 | assert((NumBytes - EncodedBytes) % 4 == 0 && |
1383 | "Invalid number of NOP bytes requested!" ); |
1384 | for (unsigned i = EncodedBytes; i < NumBytes; i += 4) |
1385 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
1386 | } |
1387 | |
1388 | void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM, |
1389 | const MachineInstr &MI) { |
1390 | StatepointOpers SOpers(&MI); |
1391 | if (unsigned PatchBytes = SOpers.getNumPatchBytes()) { |
1392 | assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!" ); |
1393 | for (unsigned i = 0; i < PatchBytes; i += 4) |
1394 | EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); |
1395 | } else { |
1396 | // Lower call target and choose correct opcode |
1397 | const MachineOperand &CallTarget = SOpers.getCallTarget(); |
1398 | MCOperand CallTargetMCOp; |
1399 | unsigned CallOpcode; |
1400 | switch (CallTarget.getType()) { |
1401 | case MachineOperand::MO_GlobalAddress: |
1402 | case MachineOperand::MO_ExternalSymbol: |
1403 | MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp); |
1404 | CallOpcode = AArch64::BL; |
1405 | break; |
1406 | case MachineOperand::MO_Immediate: |
1407 | CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm()); |
1408 | CallOpcode = AArch64::BL; |
1409 | break; |
1410 | case MachineOperand::MO_Register: |
1411 | CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg()); |
1412 | CallOpcode = AArch64::BLR; |
1413 | break; |
1414 | default: |
1415 | llvm_unreachable("Unsupported operand type in statepoint call target" ); |
1416 | break; |
1417 | } |
1418 | |
1419 | EmitToStreamer(S&: OutStreamer, |
1420 | Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp)); |
1421 | } |
1422 | |
1423 | auto &Ctx = OutStreamer.getContext(); |
1424 | MCSymbol *MILabel = Ctx.createTempSymbol(); |
1425 | OutStreamer.emitLabel(Symbol: MILabel); |
1426 | SM.recordStatepoint(L: *MILabel, MI); |
1427 | } |
1428 | |
1429 | void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) { |
1430 | // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>, |
1431 | // <opcode>, <operands> |
1432 | |
1433 | Register DefRegister = FaultingMI.getOperand(i: 0).getReg(); |
1434 | FaultMaps::FaultKind FK = |
1435 | static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm()); |
1436 | MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol(); |
1437 | unsigned Opcode = FaultingMI.getOperand(i: 3).getImm(); |
1438 | unsigned OperandsBeginIdx = 4; |
1439 | |
1440 | auto &Ctx = OutStreamer->getContext(); |
1441 | MCSymbol *FaultingLabel = Ctx.createTempSymbol(); |
1442 | OutStreamer->emitLabel(Symbol: FaultingLabel); |
1443 | |
1444 | assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!" ); |
1445 | FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel); |
1446 | |
1447 | MCInst MI; |
1448 | MI.setOpcode(Opcode); |
1449 | |
1450 | if (DefRegister != (Register)0) |
1451 | MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister)); |
1452 | |
1453 | for (const MachineOperand &MO : |
1454 | llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) { |
1455 | MCOperand Dest; |
1456 | lowerOperand(MO, MCOp&: Dest); |
1457 | MI.addOperand(Op: Dest); |
1458 | } |
1459 | |
1460 | OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName()); |
1461 | OutStreamer->emitInstruction(Inst: MI, STI: getSubtargetInfo()); |
1462 | } |
1463 | |
1464 | void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) { |
1465 | Register DestReg = MI.getOperand(i: 0).getReg(); |
1466 | if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() && |
1467 | STI->isNeonAvailable()) { |
1468 | // Convert H/S register to corresponding D register |
1469 | if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31) |
1470 | DestReg = AArch64::D0 + (DestReg - AArch64::H0); |
1471 | else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31) |
1472 | DestReg = AArch64::D0 + (DestReg - AArch64::S0); |
1473 | else |
1474 | assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31); |
1475 | |
1476 | MCInst MOVI; |
1477 | MOVI.setOpcode(AArch64::MOVID); |
1478 | MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1479 | MOVI.addOperand(Op: MCOperand::createImm(Val: 0)); |
1480 | EmitToStreamer(S&: *OutStreamer, Inst: MOVI); |
1481 | } else { |
1482 | MCInst FMov; |
1483 | switch (MI.getOpcode()) { |
1484 | default: llvm_unreachable("Unexpected opcode" ); |
1485 | case AArch64::FMOVH0: |
1486 | FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr); |
1487 | if (!STI->hasFullFP16()) |
1488 | DestReg = (AArch64::S0 + (DestReg - AArch64::H0)); |
1489 | FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1490 | FMov.addOperand(MCOperand::createReg(AArch64::WZR)); |
1491 | break; |
1492 | case AArch64::FMOVS0: |
1493 | FMov.setOpcode(AArch64::FMOVWSr); |
1494 | FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1495 | FMov.addOperand(MCOperand::createReg(AArch64::WZR)); |
1496 | break; |
1497 | case AArch64::FMOVD0: |
1498 | FMov.setOpcode(AArch64::FMOVXDr); |
1499 | FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1500 | FMov.addOperand(MCOperand::createReg(AArch64::XZR)); |
1501 | break; |
1502 | } |
1503 | EmitToStreamer(S&: *OutStreamer, Inst: FMov); |
1504 | } |
1505 | } |
1506 | |
1507 | // Simple pseudo-instructions have their lowering (with expansion to real |
1508 | // instructions) auto-generated. |
1509 | #include "AArch64GenMCPseudoLowering.inc" |
1510 | |
1511 | void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { |
1512 | AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits()); |
1513 | |
1514 | // Do any auto-generated pseudo lowerings. |
1515 | if (emitPseudoExpansionLowering(OutStreamer&: *OutStreamer, MI)) |
1516 | return; |
1517 | |
1518 | if (MI->getOpcode() == AArch64::ADRP) { |
1519 | for (auto &Opd : MI->operands()) { |
1520 | if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) == |
1521 | "swift_async_extendedFramePointerFlags" ) { |
1522 | ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true; |
1523 | } |
1524 | } |
1525 | } |
1526 | |
1527 | if (AArch64FI->getLOHRelated().count(Ptr: MI)) { |
1528 | // Generate a label for LOH related instruction |
1529 | MCSymbol *LOHLabel = createTempSymbol(Name: "loh" ); |
1530 | // Associate the instruction with the label |
1531 | LOHInstToLabel[MI] = LOHLabel; |
1532 | OutStreamer->emitLabel(Symbol: LOHLabel); |
1533 | } |
1534 | |
1535 | AArch64TargetStreamer *TS = |
1536 | static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer()); |
1537 | // Do any manual lowerings. |
1538 | switch (MI->getOpcode()) { |
1539 | default: |
1540 | break; |
1541 | case AArch64::HINT: { |
1542 | // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for |
1543 | // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be |
1544 | // non-empty. If MI is the initial BTI, place the |
1545 | // __patchable_function_entries label after BTI. |
1546 | if (CurrentPatchableFunctionEntrySym && |
1547 | CurrentPatchableFunctionEntrySym == CurrentFnBegin && |
1548 | MI == &MF->front().front()) { |
1549 | int64_t Imm = MI->getOperand(i: 0).getImm(); |
1550 | if ((Imm & 32) && (Imm & 6)) { |
1551 | MCInst Inst; |
1552 | MCInstLowering.Lower(MI, OutMI&: Inst); |
1553 | EmitToStreamer(S&: *OutStreamer, Inst); |
1554 | CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch" ); |
1555 | OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym); |
1556 | return; |
1557 | } |
1558 | } |
1559 | break; |
1560 | } |
1561 | case AArch64::MOVMCSym: { |
1562 | Register DestReg = MI->getOperand(i: 0).getReg(); |
1563 | const MachineOperand &MO_Sym = MI->getOperand(i: 1); |
1564 | MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym); |
1565 | MCOperand Hi_MCSym, Lo_MCSym; |
1566 | |
1567 | Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S); |
1568 | Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC); |
1569 | |
1570 | MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym); |
1571 | MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym); |
1572 | |
1573 | MCInst MovZ; |
1574 | MovZ.setOpcode(AArch64::MOVZXi); |
1575 | MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1576 | MovZ.addOperand(Op: Hi_MCSym); |
1577 | MovZ.addOperand(Op: MCOperand::createImm(Val: 16)); |
1578 | EmitToStreamer(S&: *OutStreamer, Inst: MovZ); |
1579 | |
1580 | MCInst MovK; |
1581 | MovK.setOpcode(AArch64::MOVKXi); |
1582 | MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1583 | MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg)); |
1584 | MovK.addOperand(Op: Lo_MCSym); |
1585 | MovK.addOperand(Op: MCOperand::createImm(Val: 0)); |
1586 | EmitToStreamer(S&: *OutStreamer, Inst: MovK); |
1587 | return; |
1588 | } |
1589 | case AArch64::MOVIv2d_ns: |
1590 | // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0". |
1591 | // as movi is more efficient across all cores. Newer cores can eliminate |
1592 | // fmovs early and there is no difference with movi, but this not true for |
1593 | // all implementations. |
1594 | // |
1595 | // The floating-point version doesn't quite work in rare cases on older |
1596 | // CPUs, so on those targets we lower this instruction to movi.16b instead. |
1597 | if (STI->hasZeroCycleZeroingFPWorkaround() && |
1598 | MI->getOperand(i: 1).getImm() == 0) { |
1599 | MCInst TmpInst; |
1600 | TmpInst.setOpcode(AArch64::MOVIv16b_ns); |
1601 | TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); |
1602 | TmpInst.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 1).getImm())); |
1603 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInst); |
1604 | return; |
1605 | } |
1606 | break; |
1607 | |
1608 | case AArch64::DBG_VALUE: |
1609 | case AArch64::DBG_VALUE_LIST: |
1610 | if (isVerbose() && OutStreamer->hasRawTextSupport()) { |
1611 | SmallString<128> TmpStr; |
1612 | raw_svector_ostream OS(TmpStr); |
1613 | PrintDebugValueComment(MI, OS); |
1614 | OutStreamer->emitRawText(String: StringRef(OS.str())); |
1615 | } |
1616 | return; |
1617 | |
1618 | case AArch64::EMITBKEY: { |
1619 | ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); |
1620 | if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && |
1621 | ExceptionHandlingType != ExceptionHandling::ARM) |
1622 | return; |
1623 | |
1624 | if (getFunctionCFISectionType(MF: *MF) == CFISection::None) |
1625 | return; |
1626 | |
1627 | OutStreamer->emitCFIBKeyFrame(); |
1628 | return; |
1629 | } |
1630 | |
1631 | case AArch64::EMITMTETAGGED: { |
1632 | ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType(); |
1633 | if (ExceptionHandlingType != ExceptionHandling::DwarfCFI && |
1634 | ExceptionHandlingType != ExceptionHandling::ARM) |
1635 | return; |
1636 | |
1637 | if (getFunctionCFISectionType(MF: *MF) != CFISection::None) |
1638 | OutStreamer->emitCFIMTETaggedFrame(); |
1639 | return; |
1640 | } |
1641 | |
1642 | // Tail calls use pseudo instructions so they have the proper code-gen |
1643 | // attributes (isCall, isReturn, etc.). We lower them to the real |
1644 | // instruction here. |
1645 | case AArch64::TCRETURNri: |
1646 | case AArch64::TCRETURNrix16x17: |
1647 | case AArch64::TCRETURNrix17: |
1648 | case AArch64::TCRETURNrinotx16: |
1649 | case AArch64::TCRETURNriALL: { |
1650 | MCInst TmpInst; |
1651 | TmpInst.setOpcode(AArch64::BR); |
1652 | TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg())); |
1653 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInst); |
1654 | return; |
1655 | } |
1656 | case AArch64::TCRETURNdi: { |
1657 | MCOperand Dest; |
1658 | MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest); |
1659 | MCInst TmpInst; |
1660 | TmpInst.setOpcode(AArch64::B); |
1661 | TmpInst.addOperand(Op: Dest); |
1662 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInst); |
1663 | return; |
1664 | } |
1665 | case AArch64::SpeculationBarrierISBDSBEndBB: { |
1666 | // Print DSB SYS + ISB |
1667 | MCInst TmpInstDSB; |
1668 | TmpInstDSB.setOpcode(AArch64::DSB); |
1669 | TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf)); |
1670 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB); |
1671 | MCInst TmpInstISB; |
1672 | TmpInstISB.setOpcode(AArch64::ISB); |
1673 | TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf)); |
1674 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB); |
1675 | return; |
1676 | } |
1677 | case AArch64::SpeculationBarrierSBEndBB: { |
1678 | // Print SB |
1679 | MCInst TmpInstSB; |
1680 | TmpInstSB.setOpcode(AArch64::SB); |
1681 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB); |
1682 | return; |
1683 | } |
1684 | case AArch64::TLSDESC_CALLSEQ: { |
1685 | /// lower this to: |
1686 | /// adrp x0, :tlsdesc:var |
1687 | /// ldr x1, [x0, #:tlsdesc_lo12:var] |
1688 | /// add x0, x0, #:tlsdesc_lo12:var |
1689 | /// .tlsdesccall var |
1690 | /// blr x1 |
1691 | /// (TPIDR_EL0 offset now in x0) |
1692 | const MachineOperand &MO_Sym = MI->getOperand(i: 0); |
1693 | MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym); |
1694 | MCOperand Sym, SymTLSDescLo12, SymTLSDesc; |
1695 | MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF); |
1696 | MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE); |
1697 | MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym); |
1698 | MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12); |
1699 | MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc); |
1700 | |
1701 | MCInst Adrp; |
1702 | Adrp.setOpcode(AArch64::ADRP); |
1703 | Adrp.addOperand(MCOperand::createReg(AArch64::X0)); |
1704 | Adrp.addOperand(Op: SymTLSDesc); |
1705 | EmitToStreamer(S&: *OutStreamer, Inst: Adrp); |
1706 | |
1707 | MCInst Ldr; |
1708 | if (STI->isTargetILP32()) { |
1709 | Ldr.setOpcode(AArch64::LDRWui); |
1710 | Ldr.addOperand(MCOperand::createReg(AArch64::W1)); |
1711 | } else { |
1712 | Ldr.setOpcode(AArch64::LDRXui); |
1713 | Ldr.addOperand(MCOperand::createReg(AArch64::X1)); |
1714 | } |
1715 | Ldr.addOperand(MCOperand::createReg(AArch64::X0)); |
1716 | Ldr.addOperand(Op: SymTLSDescLo12); |
1717 | Ldr.addOperand(Op: MCOperand::createImm(Val: 0)); |
1718 | EmitToStreamer(S&: *OutStreamer, Inst: Ldr); |
1719 | |
1720 | MCInst Add; |
1721 | if (STI->isTargetILP32()) { |
1722 | Add.setOpcode(AArch64::ADDWri); |
1723 | Add.addOperand(MCOperand::createReg(AArch64::W0)); |
1724 | Add.addOperand(MCOperand::createReg(AArch64::W0)); |
1725 | } else { |
1726 | Add.setOpcode(AArch64::ADDXri); |
1727 | Add.addOperand(MCOperand::createReg(AArch64::X0)); |
1728 | Add.addOperand(MCOperand::createReg(AArch64::X0)); |
1729 | } |
1730 | Add.addOperand(Op: SymTLSDescLo12); |
1731 | Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0))); |
1732 | EmitToStreamer(S&: *OutStreamer, Inst: Add); |
1733 | |
1734 | // Emit a relocation-annotation. This expands to no code, but requests |
1735 | // the following instruction gets an R_AARCH64_TLSDESC_CALL. |
1736 | MCInst TLSDescCall; |
1737 | TLSDescCall.setOpcode(AArch64::TLSDESCCALL); |
1738 | TLSDescCall.addOperand(Op: Sym); |
1739 | EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall); |
1740 | |
1741 | MCInst Blr; |
1742 | Blr.setOpcode(AArch64::BLR); |
1743 | Blr.addOperand(MCOperand::createReg(AArch64::X1)); |
1744 | EmitToStreamer(S&: *OutStreamer, Inst: Blr); |
1745 | |
1746 | return; |
1747 | } |
1748 | |
1749 | case AArch64::JumpTableDest32: |
1750 | case AArch64::JumpTableDest16: |
1751 | case AArch64::JumpTableDest8: |
1752 | LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI); |
1753 | return; |
1754 | |
1755 | case AArch64::FMOVH0: |
1756 | case AArch64::FMOVS0: |
1757 | case AArch64::FMOVD0: |
1758 | emitFMov0(MI: *MI); |
1759 | return; |
1760 | |
1761 | case AArch64::MOPSMemoryCopyPseudo: |
1762 | case AArch64::MOPSMemoryMovePseudo: |
1763 | case AArch64::MOPSMemorySetPseudo: |
1764 | case AArch64::MOPSMemorySetTaggingPseudo: |
1765 | LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI); |
1766 | return; |
1767 | |
1768 | case TargetOpcode::STACKMAP: |
1769 | return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI); |
1770 | |
1771 | case TargetOpcode::PATCHPOINT: |
1772 | return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI); |
1773 | |
1774 | case TargetOpcode::STATEPOINT: |
1775 | return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI); |
1776 | |
1777 | case TargetOpcode::FAULTING_OP: |
1778 | return LowerFAULTING_OP(FaultingMI: *MI); |
1779 | |
1780 | case TargetOpcode::PATCHABLE_FUNCTION_ENTER: |
1781 | LowerPATCHABLE_FUNCTION_ENTER(MI: *MI); |
1782 | return; |
1783 | |
1784 | case TargetOpcode::PATCHABLE_FUNCTION_EXIT: |
1785 | LowerPATCHABLE_FUNCTION_EXIT(MI: *MI); |
1786 | return; |
1787 | |
1788 | case TargetOpcode::PATCHABLE_TAIL_CALL: |
1789 | LowerPATCHABLE_TAIL_CALL(MI: *MI); |
1790 | return; |
1791 | case TargetOpcode::PATCHABLE_EVENT_CALL: |
1792 | return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false); |
1793 | case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL: |
1794 | return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true); |
1795 | |
1796 | case AArch64::KCFI_CHECK: |
1797 | LowerKCFI_CHECK(MI: *MI); |
1798 | return; |
1799 | |
1800 | case AArch64::HWASAN_CHECK_MEMACCESS: |
1801 | case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES: |
1802 | case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW: |
1803 | case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW: |
1804 | LowerHWASAN_CHECK_MEMACCESS(MI: *MI); |
1805 | return; |
1806 | |
1807 | case AArch64::SEH_StackAlloc: |
1808 | TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm()); |
1809 | return; |
1810 | |
1811 | case AArch64::SEH_SaveFPLR: |
1812 | TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm()); |
1813 | return; |
1814 | |
1815 | case AArch64::SEH_SaveFPLR_X: |
1816 | assert(MI->getOperand(0).getImm() < 0 && |
1817 | "Pre increment SEH opcode must have a negative offset" ); |
1818 | TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm()); |
1819 | return; |
1820 | |
1821 | case AArch64::SEH_SaveReg: |
1822 | TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(), |
1823 | Offset: MI->getOperand(i: 1).getImm()); |
1824 | return; |
1825 | |
1826 | case AArch64::SEH_SaveReg_X: |
1827 | assert(MI->getOperand(1).getImm() < 0 && |
1828 | "Pre increment SEH opcode must have a negative offset" ); |
1829 | TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(), |
1830 | Offset: -MI->getOperand(i: 1).getImm()); |
1831 | return; |
1832 | |
1833 | case AArch64::SEH_SaveRegP: |
1834 | if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 && |
1835 | MI->getOperand(i: 0).getImm() <= 28) { |
1836 | assert((MI->getOperand(0).getImm() - 19) % 2 == 0 && |
1837 | "Register paired with LR must be odd" ); |
1838 | TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(), |
1839 | Offset: MI->getOperand(i: 2).getImm()); |
1840 | return; |
1841 | } |
1842 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1843 | "Non-consecutive registers not allowed for save_regp" ); |
1844 | TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(), |
1845 | Offset: MI->getOperand(i: 2).getImm()); |
1846 | return; |
1847 | |
1848 | case AArch64::SEH_SaveRegP_X: |
1849 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1850 | "Non-consecutive registers not allowed for save_regp_x" ); |
1851 | assert(MI->getOperand(2).getImm() < 0 && |
1852 | "Pre increment SEH opcode must have a negative offset" ); |
1853 | TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(), |
1854 | Offset: -MI->getOperand(i: 2).getImm()); |
1855 | return; |
1856 | |
1857 | case AArch64::SEH_SaveFReg: |
1858 | TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(), |
1859 | Offset: MI->getOperand(i: 1).getImm()); |
1860 | return; |
1861 | |
1862 | case AArch64::SEH_SaveFReg_X: |
1863 | assert(MI->getOperand(1).getImm() < 0 && |
1864 | "Pre increment SEH opcode must have a negative offset" ); |
1865 | TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(), |
1866 | Offset: -MI->getOperand(i: 1).getImm()); |
1867 | return; |
1868 | |
1869 | case AArch64::SEH_SaveFRegP: |
1870 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1871 | "Non-consecutive registers not allowed for save_regp" ); |
1872 | TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(), |
1873 | Offset: MI->getOperand(i: 2).getImm()); |
1874 | return; |
1875 | |
1876 | case AArch64::SEH_SaveFRegP_X: |
1877 | assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && |
1878 | "Non-consecutive registers not allowed for save_regp_x" ); |
1879 | assert(MI->getOperand(2).getImm() < 0 && |
1880 | "Pre increment SEH opcode must have a negative offset" ); |
1881 | TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(), |
1882 | Offset: -MI->getOperand(i: 2).getImm()); |
1883 | return; |
1884 | |
1885 | case AArch64::SEH_SetFP: |
1886 | TS->emitARM64WinCFISetFP(); |
1887 | return; |
1888 | |
1889 | case AArch64::SEH_AddFP: |
1890 | TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm()); |
1891 | return; |
1892 | |
1893 | case AArch64::SEH_Nop: |
1894 | TS->emitARM64WinCFINop(); |
1895 | return; |
1896 | |
1897 | case AArch64::SEH_PrologEnd: |
1898 | TS->emitARM64WinCFIPrologEnd(); |
1899 | return; |
1900 | |
1901 | case AArch64::SEH_EpilogStart: |
1902 | TS->emitARM64WinCFIEpilogStart(); |
1903 | return; |
1904 | |
1905 | case AArch64::SEH_EpilogEnd: |
1906 | TS->emitARM64WinCFIEpilogEnd(); |
1907 | return; |
1908 | |
1909 | case AArch64::SEH_PACSignLR: |
1910 | TS->emitARM64WinCFIPACSignLR(); |
1911 | return; |
1912 | |
1913 | case AArch64::SEH_SaveAnyRegQP: |
1914 | assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 && |
1915 | "Non-consecutive registers not allowed for save_any_reg" ); |
1916 | assert(MI->getOperand(2).getImm() >= 0 && |
1917 | "SaveAnyRegQP SEH opcode offset must be non-negative" ); |
1918 | assert(MI->getOperand(2).getImm() <= 1008 && |
1919 | "SaveAnyRegQP SEH opcode offset must fit into 6 bits" ); |
1920 | TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(), |
1921 | Offset: MI->getOperand(i: 2).getImm()); |
1922 | return; |
1923 | |
1924 | case AArch64::SEH_SaveAnyRegQPX: |
1925 | assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 && |
1926 | "Non-consecutive registers not allowed for save_any_reg" ); |
1927 | assert(MI->getOperand(2).getImm() < 0 && |
1928 | "SaveAnyRegQPX SEH opcode offset must be negative" ); |
1929 | assert(MI->getOperand(2).getImm() >= -1008 && |
1930 | "SaveAnyRegQPX SEH opcode offset must fit into 6 bits" ); |
1931 | TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(), |
1932 | Offset: -MI->getOperand(i: 2).getImm()); |
1933 | return; |
1934 | } |
1935 | |
1936 | // Finally, do the automated lowerings for everything else. |
1937 | MCInst TmpInst; |
1938 | MCInstLowering.Lower(MI, OutMI&: TmpInst); |
1939 | EmitToStreamer(S&: *OutStreamer, Inst: TmpInst); |
1940 | } |
1941 | |
1942 | void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, |
1943 | MCSymbol *LazyPointer) { |
1944 | // _ifunc: |
1945 | // adrp x16, lazy_pointer@GOTPAGE |
1946 | // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] |
1947 | // ldr x16, [x16] |
1948 | // br x16 |
1949 | |
1950 | { |
1951 | MCInst Adrp; |
1952 | Adrp.setOpcode(AArch64::ADRP); |
1953 | Adrp.addOperand(MCOperand::createReg(AArch64::X16)); |
1954 | MCOperand SymPage; |
1955 | MCInstLowering.lowerOperand( |
1956 | MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, |
1957 | TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE), |
1958 | MCOp&: SymPage); |
1959 | Adrp.addOperand(Op: SymPage); |
1960 | OutStreamer->emitInstruction(Adrp, *STI); |
1961 | } |
1962 | |
1963 | { |
1964 | MCInst Ldr; |
1965 | Ldr.setOpcode(AArch64::LDRXui); |
1966 | Ldr.addOperand(MCOperand::createReg(AArch64::X16)); |
1967 | Ldr.addOperand(MCOperand::createReg(AArch64::X16)); |
1968 | MCOperand SymPageOff; |
1969 | MCInstLowering.lowerOperand( |
1970 | MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT | |
1971 | AArch64II::MO_PAGEOFF), |
1972 | MCOp&: SymPageOff); |
1973 | Ldr.addOperand(Op: SymPageOff); |
1974 | Ldr.addOperand(Op: MCOperand::createImm(Val: 0)); |
1975 | OutStreamer->emitInstruction(Ldr, *STI); |
1976 | } |
1977 | |
1978 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui) |
1979 | .addReg(AArch64::X16) |
1980 | .addReg(AArch64::X16) |
1981 | .addImm(0), |
1982 | *STI); |
1983 | |
1984 | OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() |
1985 | ? AArch64::BRAAZ |
1986 | : AArch64::BR) |
1987 | .addReg(AArch64::X16), |
1988 | *STI); |
1989 | } |
1990 | |
1991 | void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M, |
1992 | const GlobalIFunc &GI, |
1993 | MCSymbol *LazyPointer) { |
1994 | // These stub helpers are only ever called once, so here we're optimizing for |
1995 | // minimum size by using the pre-indexed store variants, which saves a few |
1996 | // bytes of instructions to bump & restore sp. |
1997 | |
1998 | // _ifunc.stub_helper: |
1999 | // stp fp, lr, [sp, #-16]! |
2000 | // mov fp, sp |
2001 | // stp x1, x0, [sp, #-16]! |
2002 | // stp x3, x2, [sp, #-16]! |
2003 | // stp x5, x4, [sp, #-16]! |
2004 | // stp x7, x6, [sp, #-16]! |
2005 | // stp d1, d0, [sp, #-16]! |
2006 | // stp d3, d2, [sp, #-16]! |
2007 | // stp d5, d4, [sp, #-16]! |
2008 | // stp d7, d6, [sp, #-16]! |
2009 | // bl _resolver |
2010 | // adrp x16, lazy_pointer@GOTPAGE |
2011 | // ldr x16, [x16, lazy_pointer@GOTPAGEOFF] |
2012 | // str x0, [x16] |
2013 | // mov x16, x0 |
2014 | // ldp d7, d6, [sp], #16 |
2015 | // ldp d5, d4, [sp], #16 |
2016 | // ldp d3, d2, [sp], #16 |
2017 | // ldp d1, d0, [sp], #16 |
2018 | // ldp x7, x6, [sp], #16 |
2019 | // ldp x5, x4, [sp], #16 |
2020 | // ldp x3, x2, [sp], #16 |
2021 | // ldp x1, x0, [sp], #16 |
2022 | // ldp fp, lr, [sp], #16 |
2023 | // br x16 |
2024 | |
2025 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) |
2026 | .addReg(AArch64::SP) |
2027 | .addReg(AArch64::FP) |
2028 | .addReg(AArch64::LR) |
2029 | .addReg(AArch64::SP) |
2030 | .addImm(-2), |
2031 | *STI); |
2032 | |
2033 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) |
2034 | .addReg(AArch64::FP) |
2035 | .addReg(AArch64::SP) |
2036 | .addImm(0) |
2037 | .addImm(0), |
2038 | *STI); |
2039 | |
2040 | for (int I = 0; I != 4; ++I) |
2041 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre) |
2042 | .addReg(AArch64::SP) |
2043 | .addReg(AArch64::X1 + 2 * I) |
2044 | .addReg(AArch64::X0 + 2 * I) |
2045 | .addReg(AArch64::SP) |
2046 | .addImm(-2), |
2047 | *STI); |
2048 | |
2049 | for (int I = 0; I != 4; ++I) |
2050 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre) |
2051 | .addReg(AArch64::SP) |
2052 | .addReg(AArch64::D1 + 2 * I) |
2053 | .addReg(AArch64::D0 + 2 * I) |
2054 | .addReg(AArch64::SP) |
2055 | .addImm(-2), |
2056 | *STI); |
2057 | |
2058 | OutStreamer->emitInstruction( |
2059 | MCInstBuilder(AArch64::BL) |
2060 | .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))), |
2061 | *STI); |
2062 | |
2063 | { |
2064 | MCInst Adrp; |
2065 | Adrp.setOpcode(AArch64::ADRP); |
2066 | Adrp.addOperand(MCOperand::createReg(AArch64::X16)); |
2067 | MCOperand SymPage; |
2068 | MCInstLowering.lowerOperand( |
2069 | MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1, |
2070 | TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE), |
2071 | MCOp&: SymPage); |
2072 | Adrp.addOperand(Op: SymPage); |
2073 | OutStreamer->emitInstruction(Adrp, *STI); |
2074 | } |
2075 | |
2076 | { |
2077 | MCInst Ldr; |
2078 | Ldr.setOpcode(AArch64::LDRXui); |
2079 | Ldr.addOperand(MCOperand::createReg(AArch64::X16)); |
2080 | Ldr.addOperand(MCOperand::createReg(AArch64::X16)); |
2081 | MCOperand SymPageOff; |
2082 | MCInstLowering.lowerOperand( |
2083 | MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1, |
2084 | TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF), |
2085 | MCOp&: SymPageOff); |
2086 | Ldr.addOperand(Op: SymPageOff); |
2087 | Ldr.addOperand(Op: MCOperand::createImm(Val: 0)); |
2088 | OutStreamer->emitInstruction(Ldr, *STI); |
2089 | } |
2090 | |
2091 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui) |
2092 | .addReg(AArch64::X0) |
2093 | .addReg(AArch64::X16) |
2094 | .addImm(0), |
2095 | *STI); |
2096 | |
2097 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri) |
2098 | .addReg(AArch64::X16) |
2099 | .addReg(AArch64::X0) |
2100 | .addImm(0) |
2101 | .addImm(0), |
2102 | *STI); |
2103 | |
2104 | for (int I = 3; I != -1; --I) |
2105 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost) |
2106 | .addReg(AArch64::SP) |
2107 | .addReg(AArch64::D1 + 2 * I) |
2108 | .addReg(AArch64::D0 + 2 * I) |
2109 | .addReg(AArch64::SP) |
2110 | .addImm(2), |
2111 | *STI); |
2112 | |
2113 | for (int I = 3; I != -1; --I) |
2114 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) |
2115 | .addReg(AArch64::SP) |
2116 | .addReg(AArch64::X1 + 2 * I) |
2117 | .addReg(AArch64::X0 + 2 * I) |
2118 | .addReg(AArch64::SP) |
2119 | .addImm(2), |
2120 | *STI); |
2121 | |
2122 | OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost) |
2123 | .addReg(AArch64::SP) |
2124 | .addReg(AArch64::FP) |
2125 | .addReg(AArch64::LR) |
2126 | .addReg(AArch64::SP) |
2127 | .addImm(2), |
2128 | *STI); |
2129 | |
2130 | OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e() |
2131 | ? AArch64::BRAAZ |
2132 | : AArch64::BR) |
2133 | .addReg(AArch64::X16), |
2134 | *STI); |
2135 | } |
2136 | |
2137 | const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV) { |
2138 | if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) { |
2139 | return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0), |
2140 | Ctx&: OutContext); |
2141 | } |
2142 | |
2143 | return AsmPrinter::lowerConstant(CV); |
2144 | } |
2145 | |
2146 | // Force static initialization. |
2147 | extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() { |
2148 | RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget()); |
2149 | RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget()); |
2150 | RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target()); |
2151 | RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target()); |
2152 | RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target()); |
2153 | } |
2154 | |