1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetObjectFile.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64InstPrinter.h"
22#include "MCTargetDesc/AArch64MCExpr.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "MCTargetDesc/AArch64TargetStreamer.h"
25#include "TargetInfo/AArch64TargetInfo.h"
26#include "Utils/AArch64BaseInfo.h"
27#include "llvm/ADT/SmallString.h"
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/Twine.h"
31#include "llvm/BinaryFormat/COFF.h"
32#include "llvm/BinaryFormat/ELF.h"
33#include "llvm/BinaryFormat/MachO.h"
34#include "llvm/CodeGen/AsmPrinter.h"
35#include "llvm/CodeGen/FaultMaps.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineInstr.h"
39#include "llvm/CodeGen/MachineJumpTableInfo.h"
40#include "llvm/CodeGen/MachineModuleInfoImpls.h"
41#include "llvm/CodeGen/MachineOperand.h"
42#include "llvm/CodeGen/StackMaps.h"
43#include "llvm/CodeGen/TargetRegisterInfo.h"
44#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/DebugInfoMetadata.h"
46#include "llvm/MC/MCAsmInfo.h"
47#include "llvm/MC/MCContext.h"
48#include "llvm/MC/MCInst.h"
49#include "llvm/MC/MCInstBuilder.h"
50#include "llvm/MC/MCSectionELF.h"
51#include "llvm/MC/MCSectionMachO.h"
52#include "llvm/MC/MCStreamer.h"
53#include "llvm/MC/MCSymbol.h"
54#include "llvm/MC/TargetRegistry.h"
55#include "llvm/Support/Casting.h"
56#include "llvm/Support/CommandLine.h"
57#include "llvm/Support/ErrorHandling.h"
58#include "llvm/Support/raw_ostream.h"
59#include "llvm/Target/TargetMachine.h"
60#include "llvm/TargetParser/Triple.h"
61#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
62#include <algorithm>
63#include <cassert>
64#include <cstdint>
65#include <map>
66#include <memory>
67
68using namespace llvm;
69
70#define DEBUG_TYPE "asm-printer"
71
72namespace {
73
74class AArch64AsmPrinter : public AsmPrinter {
75 AArch64MCInstLower MCInstLowering;
76 FaultMaps FM;
77 const AArch64Subtarget *STI;
78 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
79
80public:
81 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
82 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
83 FM(*this) {}
84
85 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
86
87 /// Wrapper for MCInstLowering.lowerOperand() for the
88 /// tblgen'erated pseudo lowering.
89 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
90 return MCInstLowering.lowerOperand(MO, MCOp);
91 }
92
93 void emitStartOfAsmFile(Module &M) override;
94 void emitJumpTableInfo() override;
95 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
96 codeview::JumpTableEntrySize>
97 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
98 const MCSymbol *BranchLabel) const override;
99
100 void emitFunctionEntryLabel() override;
101
102 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
103
104 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
105
106 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
107 const MachineInstr &MI);
108 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
109 const MachineInstr &MI);
110 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
111 const MachineInstr &MI);
112 void LowerFAULTING_OP(const MachineInstr &MI);
113
114 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
115 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
116 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
117 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
118
119 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
120 HwasanMemaccessTuple;
121 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
122 void LowerKCFI_CHECK(const MachineInstr &MI);
123 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
124 void emitHwasanMemaccessSymbols(Module &M);
125
126 void emitSled(const MachineInstr &MI, SledKind Kind);
127
128 /// tblgen'erated driver function for lowering simple MI->MC
129 /// pseudo instructions.
130 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
131 const MachineInstr *MI);
132
133 void emitInstruction(const MachineInstr *MI) override;
134
135 void emitFunctionHeaderComment() override;
136
137 void getAnalysisUsage(AnalysisUsage &AU) const override {
138 AsmPrinter::getAnalysisUsage(AU);
139 AU.setPreservesAll();
140 }
141
142 bool runOnMachineFunction(MachineFunction &MF) override {
143 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
144 STI = &MF.getSubtarget<AArch64Subtarget>();
145
146 SetupMachineFunction(MF);
147
148 if (STI->isTargetCOFF()) {
149 bool Local = MF.getFunction().hasLocalLinkage();
150 COFF::SymbolStorageClass Scl =
151 Local ? COFF::IMAGE_SYM_CLASS_STATIC : COFF::IMAGE_SYM_CLASS_EXTERNAL;
152 int Type =
153 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
154
155 OutStreamer->beginCOFFSymbolDef(Symbol: CurrentFnSym);
156 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: Scl);
157 OutStreamer->emitCOFFSymbolType(Type);
158 OutStreamer->endCOFFSymbolDef();
159 }
160
161 // Emit the rest of the function body.
162 emitFunctionBody();
163
164 // Emit the XRay table for this function.
165 emitXRayTable();
166
167 // We didn't modify anything.
168 return false;
169 }
170
171 const MCExpr *lowerConstant(const Constant *CV) override;
172
173private:
174 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
175 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
176 bool printAsmRegInClass(const MachineOperand &MO,
177 const TargetRegisterClass *RC, unsigned AltName,
178 raw_ostream &O);
179
180 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
181 const char *ExtraCode, raw_ostream &O) override;
182 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
183 const char *ExtraCode, raw_ostream &O) override;
184
185 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
186
187 void emitFunctionBodyEnd() override;
188
189 MCSymbol *GetCPISymbol(unsigned CPID) const override;
190 void emitEndOfAsmFile(Module &M) override;
191
192 AArch64FunctionInfo *AArch64FI = nullptr;
193
194 /// Emit the LOHs contained in AArch64FI.
195 void emitLOHs();
196
197 /// Emit instruction to set float register to zero.
198 void emitFMov0(const MachineInstr &MI);
199
200 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
201
202 MInstToMCSymbol LOHInstToLabel;
203
204 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
205 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
206 }
207
208 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
209 assert(STI);
210 return STI;
211 }
212 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
213 MCSymbol *LazyPointer) override;
214 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
215 MCSymbol *LazyPointer) override;
216};
217
218} // end anonymous namespace
219
220void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
221 const Triple &TT = TM.getTargetTriple();
222
223 if (TT.isOSBinFormatCOFF()) {
224 // Emit an absolute @feat.00 symbol
225 MCSymbol *S = MMI->getContext().getOrCreateSymbol(Name: StringRef("@feat.00"));
226 OutStreamer->beginCOFFSymbolDef(Symbol: S);
227 OutStreamer->emitCOFFSymbolStorageClass(StorageClass: COFF::IMAGE_SYM_CLASS_STATIC);
228 OutStreamer->emitCOFFSymbolType(Type: COFF::IMAGE_SYM_DTYPE_NULL);
229 OutStreamer->endCOFFSymbolDef();
230 int64_t Feat00Value = 0;
231
232 if (M.getModuleFlag(Key: "cfguard")) {
233 // Object is CFG-aware.
234 Feat00Value |= COFF::Feat00Flags::GuardCF;
235 }
236
237 if (M.getModuleFlag(Key: "ehcontguard")) {
238 // Object also has EHCont.
239 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
240 }
241
242 if (M.getModuleFlag(Key: "ms-kernel")) {
243 // Object is compiled with /kernel.
244 Feat00Value |= COFF::Feat00Flags::Kernel;
245 }
246
247 OutStreamer->emitSymbolAttribute(Symbol: S, Attribute: MCSA_Global);
248 OutStreamer->emitAssignment(
249 Symbol: S, Value: MCConstantExpr::create(Value: Feat00Value, Ctx&: MMI->getContext()));
250 }
251
252 if (!TT.isOSBinFormatELF())
253 return;
254
255 // Assemble feature flags that may require creation of a note section.
256 unsigned Flags = 0;
257 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
258 MD: M.getModuleFlag(Key: "branch-target-enforcement")))
259 if (BTE->getZExtValue())
260 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
261
262 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
263 MD: M.getModuleFlag(Key: "guarded-control-stack")))
264 if (GCS->getZExtValue())
265 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_GCS;
266
267 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
268 MD: M.getModuleFlag(Key: "sign-return-address")))
269 if (Sign->getZExtValue())
270 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
271
272 uint64_t PAuthABIPlatform = -1;
273 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
274 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-platform")))
275 PAuthABIPlatform = PAP->getZExtValue();
276 uint64_t PAuthABIVersion = -1;
277 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
278 MD: M.getModuleFlag(Key: "aarch64-elf-pauthabi-version")))
279 PAuthABIVersion = PAV->getZExtValue();
280
281 // Emit a .note.gnu.property section with the flags.
282 auto *TS =
283 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
284 TS->emitNoteSection(Flags, PAuthABIPlatform, PAuthABIVersion);
285}
286
287void AArch64AsmPrinter::emitFunctionHeaderComment() {
288 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
289 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
290 if (OutlinerString != std::nullopt)
291 OutStreamer->getCommentOS() << ' ' << OutlinerString;
292}
293
294void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
295{
296 const Function &F = MF->getFunction();
297 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
298 unsigned Num;
299 if (F.getFnAttribute(Kind: "patchable-function-entry")
300 .getValueAsString()
301 .getAsInteger(Radix: 10, Result&: Num))
302 return;
303 emitNops(N: Num);
304 return;
305 }
306
307 emitSled(MI, Kind: SledKind::FUNCTION_ENTER);
308}
309
310void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
311 emitSled(MI, Kind: SledKind::FUNCTION_EXIT);
312}
313
314void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
315 emitSled(MI, Kind: SledKind::TAIL_CALL);
316}
317
318void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
319 static const int8_t NoopsInSledCount = 7;
320 // We want to emit the following pattern:
321 //
322 // .Lxray_sled_N:
323 // ALIGN
324 // B #32
325 // ; 7 NOP instructions (28 bytes)
326 // .tmpN
327 //
328 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
329 // over the full 32 bytes (8 instructions) with the following pattern:
330 //
331 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
332 // LDR W17, #12 ; W17 := function ID
333 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
334 // BLR X16 ; call the tracing trampoline
335 // ;DATA: 32 bits of function ID
336 // ;DATA: lower 32 bits of the address of the trampoline
337 // ;DATA: higher 32 bits of the address of the trampoline
338 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
339 //
340 OutStreamer->emitCodeAlignment(Alignment: Align(4), STI: &getSubtargetInfo());
341 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
342 OutStreamer->emitLabel(Symbol: CurSled);
343 auto Target = OutContext.createTempSymbol();
344
345 // Emit "B #32" instruction, which jumps over the next 28 bytes.
346 // The operand has to be the number of 4-byte instructions to jump over,
347 // including the current instruction.
348 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
349
350 for (int8_t I = 0; I < NoopsInSledCount; I++)
351 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
352
353 OutStreamer->emitLabel(Symbol: Target);
354 recordSled(Sled: CurSled, MI, Kind, Version: 2);
355}
356
357// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
358// (built-in functions __xray_customevent/__xray_typedevent).
359//
360// .Lxray_event_sled_N:
361// b 1f
362// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
363// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
364// bl __xray_CustomEvent or __xray_TypedEvent
365// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
366// 1:
367//
368// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
369//
370// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
371// After patching, b .+N will become a nop.
372void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
373 bool Typed) {
374 auto &O = *OutStreamer;
375 MCSymbol *CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
376 O.emitLabel(Symbol: CurSled);
377 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs)
378 .addReg(AArch64::X0)
379 .addReg(AArch64::XZR)
380 .addReg(MI.getOperand(0).getReg())
381 .addImm(0);
382 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs)
383 .addReg(AArch64::X1)
384 .addReg(AArch64::XZR)
385 .addReg(MI.getOperand(1).getReg())
386 .addImm(0);
387 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
388 auto *Sym = MCSymbolRefExpr::create(
389 Symbol: OutContext.getOrCreateSymbol(
390 Name: Twine(MachO ? "_" : "") +
391 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
392 Ctx&: OutContext);
393 if (Typed) {
394 O.AddComment(T: "Begin XRay typed event");
395 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9));
396 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
397 .addReg(AArch64::SP)
398 .addReg(AArch64::X0)
399 .addReg(AArch64::X1)
400 .addReg(AArch64::SP)
401 .addImm(-4));
402 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui)
403 .addReg(AArch64::X2)
404 .addReg(AArch64::SP)
405 .addImm(2));
406 EmitToStreamer(S&: O, Inst: MovX0Op0);
407 EmitToStreamer(S&: O, Inst: MovX1Op1);
408 EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs)
409 .addReg(AArch64::X2)
410 .addReg(AArch64::XZR)
411 .addReg(MI.getOperand(2).getReg())
412 .addImm(0));
413 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
414 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
415 .addReg(AArch64::X2)
416 .addReg(AArch64::SP)
417 .addImm(2));
418 O.AddComment(T: "End XRay typed event");
419 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
420 .addReg(AArch64::SP)
421 .addReg(AArch64::X0)
422 .addReg(AArch64::X1)
423 .addReg(AArch64::SP)
424 .addImm(4));
425
426 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
427 } else {
428 O.AddComment(T: "Begin XRay custom event");
429 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6));
430 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
431 .addReg(AArch64::SP)
432 .addReg(AArch64::X0)
433 .addReg(AArch64::X1)
434 .addReg(AArch64::SP)
435 .addImm(-2));
436 EmitToStreamer(S&: O, Inst: MovX0Op0);
437 EmitToStreamer(S&: O, Inst: MovX1Op1);
438 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
439 O.AddComment(T: "End XRay custom event");
440 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
441 .addReg(AArch64::SP)
442 .addReg(AArch64::X0)
443 .addReg(AArch64::X1)
444 .addReg(AArch64::SP)
445 .addImm(2));
446
447 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
448 }
449}
450
451void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
452 Register AddrReg = MI.getOperand(i: 0).getReg();
453 assert(std::next(MI.getIterator())->isCall() &&
454 "KCFI_CHECK not followed by a call instruction");
455 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
456 "KCFI_CHECK call target doesn't match call operand");
457
458 // Default to using the intra-procedure-call temporary registers for
459 // comparing the hashes.
460 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
461 if (AddrReg == AArch64::XZR) {
462 // Checking XZR makes no sense. Instead of emitting a load, zero
463 // ScratchRegs[0] and use it for the ESR AddrIndex below.
464 AddrReg = getXRegFromWReg(Reg: ScratchRegs[0]);
465 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
466 .addReg(AddrReg)
467 .addReg(AArch64::XZR)
468 .addReg(AArch64::XZR)
469 .addImm(0));
470 } else {
471 // If one of the scratch registers is used for the call target (e.g.
472 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
473 // temporary register instead (in this case, AArch64::W9) as the check
474 // is immediately followed by the call instruction.
475 for (auto &Reg : ScratchRegs) {
476 if (Reg == getWRegFromXReg(AddrReg)) {
477 Reg = AArch64::W9;
478 break;
479 }
480 }
481 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
482 "Invalid scratch registers for KCFI_CHECK");
483
484 // Adjust the offset for patchable-function-prefix. This assumes that
485 // patchable-function-prefix is the same for all functions.
486 int64_t PrefixNops = 0;
487 (void)MI.getMF()
488 ->getFunction()
489 .getFnAttribute(Kind: "patchable-function-prefix")
490 .getValueAsString()
491 .getAsInteger(Radix: 10, Result&: PrefixNops);
492
493 // Load the target function type hash.
494 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
495 .addReg(ScratchRegs[0])
496 .addReg(AddrReg)
497 .addImm(-(PrefixNops * 4 + 4)));
498 }
499
500 // Load the expected type hash.
501 const int64_t Type = MI.getOperand(i: 1).getImm();
502 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
503 .addReg(ScratchRegs[1])
504 .addReg(ScratchRegs[1])
505 .addImm(Type & 0xFFFF)
506 .addImm(0));
507 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
508 .addReg(ScratchRegs[1])
509 .addReg(ScratchRegs[1])
510 .addImm((Type >> 16) & 0xFFFF)
511 .addImm(16));
512
513 // Compare the hashes and trap if there's a mismatch.
514 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
515 .addReg(AArch64::WZR)
516 .addReg(ScratchRegs[0])
517 .addReg(ScratchRegs[1])
518 .addImm(0));
519
520 MCSymbol *Pass = OutContext.createTempSymbol();
521 EmitToStreamer(*OutStreamer,
522 MCInstBuilder(AArch64::Bcc)
523 .addImm(AArch64CC::EQ)
524 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
525
526 // The base ESR is 0x8000 and the register information is encoded in bits
527 // 0-9 as follows:
528 // - 0-4: n, where the register Xn contains the target address
529 // - 5-9: m, where the register Wm contains the expected type hash
530 // Where n, m are in [0, 30].
531 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
532 unsigned AddrIndex;
533 switch (AddrReg) {
534 default:
535 AddrIndex = AddrReg - AArch64::X0;
536 break;
537 case AArch64::FP:
538 AddrIndex = 29;
539 break;
540 case AArch64::LR:
541 AddrIndex = 30;
542 break;
543 }
544
545 assert(AddrIndex < 31 && TypeIndex < 31);
546
547 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
548 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
549 OutStreamer->emitLabel(Symbol: Pass);
550}
551
552void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
553 Register Reg = MI.getOperand(i: 0).getReg();
554 bool IsShort =
555 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
556 (MI.getOpcode() ==
557 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
558 uint32_t AccessInfo = MI.getOperand(i: 1).getImm();
559 bool IsFixedShadow =
560 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
561 (MI.getOpcode() ==
562 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
563 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(i: 2).getImm() : 0;
564
565 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
566 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
567 if (!Sym) {
568 // FIXME: Make this work on non-ELF.
569 if (!TM.getTargetTriple().isOSBinFormatELF())
570 report_fatal_error(reason: "llvm.hwasan.check.memaccess only supported on ELF");
571
572 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
573 utostr(AccessInfo);
574 if (IsFixedShadow)
575 SymName += "_fixed_" + utostr(X: FixedShadowOffset);
576 if (IsShort)
577 SymName += "_short_v2";
578 Sym = OutContext.getOrCreateSymbol(Name: SymName);
579 }
580
581 EmitToStreamer(*OutStreamer,
582 MCInstBuilder(AArch64::BL)
583 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
584}
585
586void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
587 if (HwasanMemaccessSymbols.empty())
588 return;
589
590 const Triple &TT = TM.getTargetTriple();
591 assert(TT.isOSBinFormatELF());
592 std::unique_ptr<MCSubtargetInfo> STI(
593 TM.getTarget().createMCSubtargetInfo(TheTriple: TT.str(), CPU: "", Features: ""));
594 assert(STI && "Unable to create subtarget info");
595
596 MCSymbol *HwasanTagMismatchV1Sym =
597 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch");
598 MCSymbol *HwasanTagMismatchV2Sym =
599 OutContext.getOrCreateSymbol(Name: "__hwasan_tag_mismatch_v2");
600
601 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
602 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV1Sym, Ctx&: OutContext);
603 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
604 MCSymbolRefExpr::create(Symbol: HwasanTagMismatchV2Sym, Ctx&: OutContext);
605
606 for (auto &P : HwasanMemaccessSymbols) {
607 unsigned Reg = std::get<0>(t: P.first);
608 bool IsShort = std::get<1>(t: P.first);
609 uint32_t AccessInfo = std::get<2>(t: P.first);
610 bool IsFixedShadow = std::get<3>(t: P.first);
611 uint64_t FixedShadowOffset = std::get<4>(t: P.first);
612 const MCSymbolRefExpr *HwasanTagMismatchRef =
613 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
614 MCSymbol *Sym = P.second;
615
616 bool HasMatchAllTag =
617 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
618 uint8_t MatchAllTag =
619 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
620 unsigned Size =
621 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
622 bool CompileKernel =
623 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
624
625 OutStreamer->switchSection(Section: OutContext.getELFSection(
626 Section: ".text.hot", Type: ELF::SHT_PROGBITS,
627 Flags: ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, EntrySize: 0, Group: Sym->getName(),
628 /*IsComdat=*/true));
629
630 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_ELF_TypeFunction);
631 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Weak);
632 OutStreamer->emitSymbolAttribute(Symbol: Sym, Attribute: MCSA_Hidden);
633 OutStreamer->emitLabel(Symbol: Sym);
634
635 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
636 .addReg(AArch64::X16)
637 .addReg(Reg)
638 .addImm(4)
639 .addImm(55),
640 *STI);
641
642 if (IsFixedShadow) {
643 // Aarch64 makes it difficult to embed large constants in the code.
644 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
645 // left-shift option in the MOV instruction. Combined with the 16-bit
646 // immediate, this is enough to represent any offset up to 2**48.
647 OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
648 .addReg(AArch64::X17)
649 .addImm(FixedShadowOffset >> 32)
650 .addImm(32),
651 *STI);
652 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
653 .addReg(AArch64::W16)
654 .addReg(AArch64::X17)
655 .addReg(AArch64::X16)
656 .addImm(0)
657 .addImm(0),
658 *STI);
659 } else {
660 OutStreamer->emitInstruction(
661 MCInstBuilder(AArch64::LDRBBroX)
662 .addReg(AArch64::W16)
663 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
664 .addReg(AArch64::X16)
665 .addImm(0)
666 .addImm(0),
667 *STI);
668 }
669
670 OutStreamer->emitInstruction(
671 MCInstBuilder(AArch64::SUBSXrs)
672 .addReg(AArch64::XZR)
673 .addReg(AArch64::X16)
674 .addReg(Reg)
675 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
676 *STI);
677 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
678 OutStreamer->emitInstruction(
679 MCInstBuilder(AArch64::Bcc)
680 .addImm(AArch64CC::NE)
681 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
682 OutContext)),
683 *STI);
684 MCSymbol *ReturnSym = OutContext.createTempSymbol();
685 OutStreamer->emitLabel(Symbol: ReturnSym);
686 OutStreamer->emitInstruction(
687 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
688 OutStreamer->emitLabel(Symbol: HandleMismatchOrPartialSym);
689
690 if (HasMatchAllTag) {
691 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
692 .addReg(AArch64::X17)
693 .addReg(Reg)
694 .addImm(56)
695 .addImm(63),
696 *STI);
697 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
698 .addReg(AArch64::XZR)
699 .addReg(AArch64::X17)
700 .addImm(MatchAllTag)
701 .addImm(0),
702 *STI);
703 OutStreamer->emitInstruction(
704 MCInstBuilder(AArch64::Bcc)
705 .addImm(AArch64CC::EQ)
706 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
707 *STI);
708 }
709
710 if (IsShort) {
711 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
712 .addReg(AArch64::WZR)
713 .addReg(AArch64::W16)
714 .addImm(15)
715 .addImm(0),
716 *STI);
717 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
718 OutStreamer->emitInstruction(
719 MCInstBuilder(AArch64::Bcc)
720 .addImm(AArch64CC::HI)
721 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
722 *STI);
723
724 OutStreamer->emitInstruction(
725 MCInstBuilder(AArch64::ANDXri)
726 .addReg(AArch64::X17)
727 .addReg(Reg)
728 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
729 *STI);
730 if (Size != 1)
731 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
732 .addReg(AArch64::X17)
733 .addReg(AArch64::X17)
734 .addImm(Size - 1)
735 .addImm(0),
736 *STI);
737 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
738 .addReg(AArch64::WZR)
739 .addReg(AArch64::W16)
740 .addReg(AArch64::W17)
741 .addImm(0),
742 *STI);
743 OutStreamer->emitInstruction(
744 MCInstBuilder(AArch64::Bcc)
745 .addImm(AArch64CC::LS)
746 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
747 *STI);
748
749 OutStreamer->emitInstruction(
750 MCInstBuilder(AArch64::ORRXri)
751 .addReg(AArch64::X16)
752 .addReg(Reg)
753 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
754 *STI);
755 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
756 .addReg(AArch64::W16)
757 .addReg(AArch64::X16)
758 .addImm(0),
759 *STI);
760 OutStreamer->emitInstruction(
761 MCInstBuilder(AArch64::SUBSXrs)
762 .addReg(AArch64::XZR)
763 .addReg(AArch64::X16)
764 .addReg(Reg)
765 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
766 *STI);
767 OutStreamer->emitInstruction(
768 MCInstBuilder(AArch64::Bcc)
769 .addImm(AArch64CC::EQ)
770 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
771 *STI);
772
773 OutStreamer->emitLabel(Symbol: HandleMismatchSym);
774 }
775
776 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
777 .addReg(AArch64::SP)
778 .addReg(AArch64::X0)
779 .addReg(AArch64::X1)
780 .addReg(AArch64::SP)
781 .addImm(-32),
782 *STI);
783 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
784 .addReg(AArch64::FP)
785 .addReg(AArch64::LR)
786 .addReg(AArch64::SP)
787 .addImm(29),
788 *STI);
789
790 if (Reg != AArch64::X0)
791 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
792 .addReg(AArch64::X0)
793 .addReg(AArch64::XZR)
794 .addReg(Reg)
795 .addImm(0),
796 *STI);
797 OutStreamer->emitInstruction(
798 MCInstBuilder(AArch64::MOVZXi)
799 .addReg(AArch64::X1)
800 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
801 .addImm(0),
802 *STI);
803
804 if (CompileKernel) {
805 // The Linux kernel's dynamic loader doesn't support GOT relative
806 // relocations, but it doesn't support late binding either, so just call
807 // the function directly.
808 OutStreamer->emitInstruction(
809 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
810 } else {
811 // Intentionally load the GOT entry and branch to it, rather than possibly
812 // late binding the function, which may clobber the registers before we
813 // have a chance to save them.
814 OutStreamer->emitInstruction(
815 MCInstBuilder(AArch64::ADRP)
816 .addReg(AArch64::X16)
817 .addExpr(AArch64MCExpr::create(
818 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
819 OutContext)),
820 *STI);
821 OutStreamer->emitInstruction(
822 MCInstBuilder(AArch64::LDRXui)
823 .addReg(AArch64::X16)
824 .addReg(AArch64::X16)
825 .addExpr(AArch64MCExpr::create(
826 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
827 OutContext)),
828 *STI);
829 OutStreamer->emitInstruction(
830 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
831 }
832 }
833}
834
835void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
836 emitHwasanMemaccessSymbols(M);
837
838 const Triple &TT = TM.getTargetTriple();
839 if (TT.isOSBinFormatMachO()) {
840 // Funny Darwin hack: This flag tells the linker that no global symbols
841 // contain code that falls through to other global symbols (e.g. the obvious
842 // implementation of multiple entry points). If this doesn't occur, the
843 // linker can safely perform dead code stripping. Since LLVM never
844 // generates code that does this, it is always safe to set.
845 OutStreamer->emitAssemblerFlag(Flag: MCAF_SubsectionsViaSymbols);
846 }
847
848 // Emit stack and fault map information.
849 FM.serializeToFaultMapSection();
850
851}
852
853void AArch64AsmPrinter::emitLOHs() {
854 SmallVector<MCSymbol *, 3> MCArgs;
855
856 for (const auto &D : AArch64FI->getLOHContainer()) {
857 for (const MachineInstr *MI : D.getArgs()) {
858 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(x: MI);
859 assert(LabelIt != LOHInstToLabel.end() &&
860 "Label hasn't been inserted for LOH related instruction");
861 MCArgs.push_back(Elt: LabelIt->second);
862 }
863 OutStreamer->emitLOHDirective(Kind: D.getKind(), Args: MCArgs);
864 MCArgs.clear();
865 }
866}
867
868void AArch64AsmPrinter::emitFunctionBodyEnd() {
869 if (!AArch64FI->getLOHRelated().empty())
870 emitLOHs();
871}
872
873/// GetCPISymbol - Return the symbol for the specified constant pool entry.
874MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
875 // Darwin uses a linker-private symbol name for constant-pools (to
876 // avoid addends on the relocation?), ELF has no such concept and
877 // uses a normal private symbol.
878 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
879 return OutContext.getOrCreateSymbol(
880 Name: Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
881 Twine(getFunctionNumber()) + "_" + Twine(CPID));
882
883 return AsmPrinter::GetCPISymbol(CPID);
884}
885
886void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
887 raw_ostream &O) {
888 const MachineOperand &MO = MI->getOperand(i: OpNum);
889 switch (MO.getType()) {
890 default:
891 llvm_unreachable("<unknown operand type>");
892 case MachineOperand::MO_Register: {
893 Register Reg = MO.getReg();
894 assert(Reg.isPhysical());
895 assert(!MO.getSubReg() && "Subregs should be eliminated!");
896 O << AArch64InstPrinter::getRegisterName(Reg);
897 break;
898 }
899 case MachineOperand::MO_Immediate: {
900 O << MO.getImm();
901 break;
902 }
903 case MachineOperand::MO_GlobalAddress: {
904 PrintSymbolOperand(MO, OS&: O);
905 break;
906 }
907 case MachineOperand::MO_BlockAddress: {
908 MCSymbol *Sym = GetBlockAddressSymbol(BA: MO.getBlockAddress());
909 Sym->print(OS&: O, MAI);
910 break;
911 }
912 }
913}
914
915bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
916 raw_ostream &O) {
917 Register Reg = MO.getReg();
918 switch (Mode) {
919 default:
920 return true; // Unknown mode.
921 case 'w':
922 Reg = getWRegFromXReg(Reg);
923 break;
924 case 'x':
925 Reg = getXRegFromWReg(Reg);
926 break;
927 case 't':
928 Reg = getXRegFromXRegTuple(RegTuple: Reg);
929 break;
930 }
931
932 O << AArch64InstPrinter::getRegisterName(Reg);
933 return false;
934}
935
936// Prints the register in MO using class RC using the offset in the
937// new register class. This should not be used for cross class
938// printing.
939bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
940 const TargetRegisterClass *RC,
941 unsigned AltName, raw_ostream &O) {
942 assert(MO.isReg() && "Should only get here with a register!");
943 const TargetRegisterInfo *RI = STI->getRegisterInfo();
944 Register Reg = MO.getReg();
945 unsigned RegToPrint = RC->getRegister(i: RI->getEncodingValue(RegNo: Reg));
946 if (!RI->regsOverlap(RegA: RegToPrint, RegB: Reg))
947 return true;
948 O << AArch64InstPrinter::getRegisterName(Reg: RegToPrint, AltIdx: AltName);
949 return false;
950}
951
952bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
953 const char *ExtraCode, raw_ostream &O) {
954 const MachineOperand &MO = MI->getOperand(i: OpNum);
955
956 // First try the generic code, which knows about modifiers like 'c' and 'n'.
957 if (!AsmPrinter::PrintAsmOperand(MI, OpNo: OpNum, ExtraCode, OS&: O))
958 return false;
959
960 // Does this asm operand have a single letter operand modifier?
961 if (ExtraCode && ExtraCode[0]) {
962 if (ExtraCode[1] != 0)
963 return true; // Unknown modifier.
964
965 switch (ExtraCode[0]) {
966 default:
967 return true; // Unknown modifier.
968 case 'w': // Print W register
969 case 'x': // Print X register
970 if (MO.isReg())
971 return printAsmMRegister(MO, Mode: ExtraCode[0], O);
972 if (MO.isImm() && MO.getImm() == 0) {
973 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
974 O << AArch64InstPrinter::getRegisterName(Reg);
975 return false;
976 }
977 printOperand(MI, OpNum, O);
978 return false;
979 case 'b': // Print B register.
980 case 'h': // Print H register.
981 case 's': // Print S register.
982 case 'd': // Print D register.
983 case 'q': // Print Q register.
984 case 'z': // Print Z register.
985 if (MO.isReg()) {
986 const TargetRegisterClass *RC;
987 switch (ExtraCode[0]) {
988 case 'b':
989 RC = &AArch64::FPR8RegClass;
990 break;
991 case 'h':
992 RC = &AArch64::FPR16RegClass;
993 break;
994 case 's':
995 RC = &AArch64::FPR32RegClass;
996 break;
997 case 'd':
998 RC = &AArch64::FPR64RegClass;
999 break;
1000 case 'q':
1001 RC = &AArch64::FPR128RegClass;
1002 break;
1003 case 'z':
1004 RC = &AArch64::ZPRRegClass;
1005 break;
1006 default:
1007 return true;
1008 }
1009 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
1010 }
1011 printOperand(MI, OpNum, O);
1012 return false;
1013 }
1014 }
1015
1016 // According to ARM, we should emit x and v registers unless we have a
1017 // modifier.
1018 if (MO.isReg()) {
1019 Register Reg = MO.getReg();
1020
1021 // If this is a w or x register, print an x register.
1022 if (AArch64::GPR32allRegClass.contains(Reg) ||
1023 AArch64::GPR64allRegClass.contains(Reg))
1024 return printAsmMRegister(MO, Mode: 'x', O);
1025
1026 // If this is an x register tuple, print an x register.
1027 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1028 return printAsmMRegister(MO, Mode: 't', O);
1029
1030 unsigned AltName = AArch64::NoRegAltName;
1031 const TargetRegisterClass *RegClass;
1032 if (AArch64::ZPRRegClass.contains(Reg)) {
1033 RegClass = &AArch64::ZPRRegClass;
1034 } else if (AArch64::PPRRegClass.contains(Reg)) {
1035 RegClass = &AArch64::PPRRegClass;
1036 } else if (AArch64::PNRRegClass.contains(Reg)) {
1037 RegClass = &AArch64::PNRRegClass;
1038 } else {
1039 RegClass = &AArch64::FPR128RegClass;
1040 AltName = AArch64::vreg;
1041 }
1042
1043 // If this is a b, h, s, d, or q register, print it as a v register.
1044 return printAsmRegInClass(MO, RC: RegClass, AltName, O);
1045 }
1046
1047 printOperand(MI, OpNum, O);
1048 return false;
1049}
1050
1051bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1052 unsigned OpNum,
1053 const char *ExtraCode,
1054 raw_ostream &O) {
1055 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1056 return true; // Unknown modifier.
1057
1058 const MachineOperand &MO = MI->getOperand(i: OpNum);
1059 assert(MO.isReg() && "unexpected inline asm memory operand");
1060 O << "[" << AArch64InstPrinter::getRegisterName(Reg: MO.getReg()) << "]";
1061 return false;
1062}
1063
1064void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1065 raw_ostream &OS) {
1066 unsigned NOps = MI->getNumOperands();
1067 assert(NOps == 4);
1068 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1069 // cast away const; DIetc do not take const operands for some reason.
1070 OS << MI->getDebugVariable()->getName();
1071 OS << " <- ";
1072 // Frame address. Currently handles register +- offset only.
1073 assert(MI->isIndirectDebugValue());
1074 OS << '[';
1075 for (unsigned I = 0, E = std::distance(first: MI->debug_operands().begin(),
1076 last: MI->debug_operands().end());
1077 I < E; ++I) {
1078 if (I != 0)
1079 OS << ", ";
1080 printOperand(MI, OpNum: I, O&: OS);
1081 }
1082 OS << ']';
1083 OS << "+";
1084 printOperand(MI, OpNum: NOps - 2, O&: OS);
1085}
1086
1087void AArch64AsmPrinter::emitJumpTableInfo() {
1088 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1089 if (!MJTI) return;
1090
1091 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
1092 if (JT.empty()) return;
1093
1094 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1095 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F: MF->getFunction(), TM);
1096 OutStreamer->switchSection(Section: ReadOnlySec);
1097
1098 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1099 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
1100 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1101
1102 // If this jump table was deleted, ignore it.
1103 if (JTBBs.empty()) continue;
1104
1105 unsigned Size = AFI->getJumpTableEntrySize(Idx: JTI);
1106 emitAlignment(Alignment: Align(Size));
1107 OutStreamer->emitLabel(Symbol: GetJTISymbol(JTID: JTI));
1108
1109 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1110 const MCExpr *Base = MCSymbolRefExpr::create(Symbol: BaseSym, Ctx&: OutContext);
1111
1112 for (auto *JTBB : JTBBs) {
1113 const MCExpr *Value =
1114 MCSymbolRefExpr::create(Symbol: JTBB->getSymbol(), Ctx&: OutContext);
1115
1116 // Each entry is:
1117 // .byte/.hword (LBB - Lbase)>>2
1118 // or plain:
1119 // .word LBB - Lbase
1120 Value = MCBinaryExpr::createSub(LHS: Value, RHS: Base, Ctx&: OutContext);
1121 if (Size != 4)
1122 Value = MCBinaryExpr::createLShr(
1123 LHS: Value, RHS: MCConstantExpr::create(Value: 2, Ctx&: OutContext), Ctx&: OutContext);
1124
1125 OutStreamer->emitValue(Value, Size);
1126 }
1127 }
1128}
1129
1130std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1131 codeview::JumpTableEntrySize>
1132AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1133 const MachineInstr *BranchInstr,
1134 const MCSymbol *BranchLabel) const {
1135 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1136 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(Idx: JTI);
1137 codeview::JumpTableEntrySize EntrySize;
1138 switch (AFI->getJumpTableEntrySize(Idx: JTI)) {
1139 case 1:
1140 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1141 break;
1142 case 2:
1143 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1144 break;
1145 case 4:
1146 EntrySize = codeview::JumpTableEntrySize::Int32;
1147 break;
1148 default:
1149 llvm_unreachable("Unexpected jump table entry size");
1150 }
1151 return std::make_tuple(args: Base, args: 0, args&: BranchLabel, args&: EntrySize);
1152}
1153
1154void AArch64AsmPrinter::emitFunctionEntryLabel() {
1155 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1156 MF->getFunction().getCallingConv() ==
1157 CallingConv::AArch64_SVE_VectorCall ||
1158 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
1159 auto *TS =
1160 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1161 TS->emitDirectiveVariantPCS(Symbol: CurrentFnSym);
1162 }
1163
1164 if (TM.getTargetTriple().isWindowsArm64EC() &&
1165 !MF->getFunction().hasLocalLinkage()) {
1166 // For ARM64EC targets, a function definition's name is mangled differently
1167 // from the normal symbol. We emit the alias from the unmangled symbol to
1168 // mangled symbol name here.
1169 if (MDNode *Unmangled =
1170 MF->getFunction().getMetadata(Kind: "arm64ec_unmangled_name")) {
1171 AsmPrinter::emitFunctionEntryLabel();
1172
1173 if (MDNode *ECMangled =
1174 MF->getFunction().getMetadata(Kind: "arm64ec_ecmangled_name")) {
1175 StringRef UnmangledStr =
1176 cast<MDString>(Val: Unmangled->getOperand(I: 0))->getString();
1177 MCSymbol *UnmangledSym =
1178 MMI->getContext().getOrCreateSymbol(Name: UnmangledStr);
1179 StringRef ECMangledStr =
1180 cast<MDString>(Val: ECMangled->getOperand(I: 0))->getString();
1181 MCSymbol *ECMangledSym =
1182 MMI->getContext().getOrCreateSymbol(Name: ECMangledStr);
1183 OutStreamer->emitSymbolAttribute(Symbol: UnmangledSym, Attribute: MCSA_WeakAntiDep);
1184 OutStreamer->emitAssignment(
1185 Symbol: UnmangledSym,
1186 Value: MCSymbolRefExpr::create(Symbol: ECMangledSym, Kind: MCSymbolRefExpr::VK_WEAKREF,
1187 Ctx&: MMI->getContext()));
1188 OutStreamer->emitSymbolAttribute(Symbol: ECMangledSym, Attribute: MCSA_WeakAntiDep);
1189 OutStreamer->emitAssignment(
1190 Symbol: ECMangledSym,
1191 Value: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Kind: MCSymbolRefExpr::VK_WEAKREF,
1192 Ctx&: MMI->getContext()));
1193 return;
1194 } else {
1195 StringRef UnmangledStr =
1196 cast<MDString>(Val: Unmangled->getOperand(I: 0))->getString();
1197 MCSymbol *UnmangledSym =
1198 MMI->getContext().getOrCreateSymbol(Name: UnmangledStr);
1199 OutStreamer->emitSymbolAttribute(Symbol: UnmangledSym, Attribute: MCSA_WeakAntiDep);
1200 OutStreamer->emitAssignment(
1201 Symbol: UnmangledSym,
1202 Value: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Kind: MCSymbolRefExpr::VK_WEAKREF,
1203 Ctx&: MMI->getContext()));
1204 return;
1205 }
1206 }
1207 }
1208
1209 return AsmPrinter::emitFunctionEntryLabel();
1210}
1211
1212/// Small jump tables contain an unsigned byte or half, representing the offset
1213/// from the lowest-addressed possible destination to the desired basic
1214/// block. Since all instructions are 4-byte aligned, this is further compressed
1215/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1216/// materialize the correct destination we need:
1217///
1218/// adr xDest, .LBB0_0
1219/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1220/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1221void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1222 const llvm::MachineInstr &MI) {
1223 Register DestReg = MI.getOperand(i: 0).getReg();
1224 Register ScratchReg = MI.getOperand(i: 1).getReg();
1225 Register ScratchRegW =
1226 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1227 Register TableReg = MI.getOperand(i: 2).getReg();
1228 Register EntryReg = MI.getOperand(i: 3).getReg();
1229 int JTIdx = MI.getOperand(i: 4).getIndex();
1230 int Size = AArch64FI->getJumpTableEntrySize(Idx: JTIdx);
1231
1232 // This has to be first because the compression pass based its reachability
1233 // calculations on the start of the JumpTableDest instruction.
1234 auto Label =
1235 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(Idx: JTIdx);
1236
1237 // If we don't already have a symbol to use as the base, use the ADR
1238 // instruction itself.
1239 if (!Label) {
1240 Label = MF->getContext().createTempSymbol();
1241 AArch64FI->setJumpTableEntryInfo(Idx: JTIdx, Size, PCRelSym: Label);
1242 OutStreamer.emitLabel(Symbol: Label);
1243 }
1244
1245 auto LabelExpr = MCSymbolRefExpr::create(Symbol: Label, Ctx&: MF->getContext());
1246 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
1247 .addReg(DestReg)
1248 .addExpr(LabelExpr));
1249
1250 // Load the number of instruction-steps to offset from the label.
1251 unsigned LdrOpcode;
1252 switch (Size) {
1253 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1254 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1255 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1256 default:
1257 llvm_unreachable("Unknown jump table size");
1258 }
1259
1260 EmitToStreamer(S&: OutStreamer, Inst: MCInstBuilder(LdrOpcode)
1261 .addReg(Reg: Size == 4 ? ScratchReg : ScratchRegW)
1262 .addReg(Reg: TableReg)
1263 .addReg(Reg: EntryReg)
1264 .addImm(Val: 0)
1265 .addImm(Val: Size == 1 ? 0 : 1));
1266
1267 // Add to the already materialized base label address, multiplying by 4 if
1268 // compressed.
1269 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1270 .addReg(DestReg)
1271 .addReg(DestReg)
1272 .addReg(ScratchReg)
1273 .addImm(Size == 4 ? 0 : 2));
1274}
1275
1276void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1277 const llvm::MachineInstr &MI) {
1278 unsigned Opcode = MI.getOpcode();
1279 assert(STI->hasMOPS());
1280 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1281
1282 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1283 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1284 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1285 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1286 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1287 if (Opcode == AArch64::MOPSMemorySetPseudo)
1288 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1289 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1290 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1291 llvm_unreachable("Unhandled memory operation pseudo");
1292 }();
1293 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1294 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1295
1296 for (auto Op : Ops) {
1297 int i = 0;
1298 auto MCIB = MCInstBuilder(Op);
1299 // Destination registers
1300 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1301 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1302 if (!IsSet)
1303 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1304 // Input registers
1305 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1306 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1307 MCIB.addReg(Reg: MI.getOperand(i: i++).getReg());
1308
1309 EmitToStreamer(S&: OutStreamer, Inst: MCIB);
1310 }
1311}
1312
1313void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1314 const MachineInstr &MI) {
1315 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1316
1317 auto &Ctx = OutStreamer.getContext();
1318 MCSymbol *MILabel = Ctx.createTempSymbol();
1319 OutStreamer.emitLabel(Symbol: MILabel);
1320
1321 SM.recordStackMap(L: *MILabel, MI);
1322 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1323
1324 // Scan ahead to trim the shadow.
1325 const MachineBasicBlock &MBB = *MI.getParent();
1326 MachineBasicBlock::const_iterator MII(MI);
1327 ++MII;
1328 while (NumNOPBytes > 0) {
1329 if (MII == MBB.end() || MII->isCall() ||
1330 MII->getOpcode() == AArch64::DBG_VALUE ||
1331 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1332 MII->getOpcode() == TargetOpcode::STACKMAP)
1333 break;
1334 ++MII;
1335 NumNOPBytes -= 4;
1336 }
1337
1338 // Emit nops.
1339 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1340 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1341}
1342
1343// Lower a patchpoint of the form:
1344// [<def>], <id>, <numBytes>, <target>, <numArgs>
1345void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1346 const MachineInstr &MI) {
1347 auto &Ctx = OutStreamer.getContext();
1348 MCSymbol *MILabel = Ctx.createTempSymbol();
1349 OutStreamer.emitLabel(Symbol: MILabel);
1350 SM.recordPatchPoint(L: *MILabel, MI);
1351
1352 PatchPointOpers Opers(&MI);
1353
1354 int64_t CallTarget = Opers.getCallTarget().getImm();
1355 unsigned EncodedBytes = 0;
1356 if (CallTarget) {
1357 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1358 "High 16 bits of call target should be zero.");
1359 Register ScratchReg = MI.getOperand(i: Opers.getNextScratchIdx()).getReg();
1360 EncodedBytes = 16;
1361 // Materialize the jump address:
1362 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
1363 .addReg(ScratchReg)
1364 .addImm((CallTarget >> 32) & 0xFFFF)
1365 .addImm(32));
1366 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1367 .addReg(ScratchReg)
1368 .addReg(ScratchReg)
1369 .addImm((CallTarget >> 16) & 0xFFFF)
1370 .addImm(16));
1371 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1372 .addReg(ScratchReg)
1373 .addReg(ScratchReg)
1374 .addImm(CallTarget & 0xFFFF)
1375 .addImm(0));
1376 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1377 }
1378 // Emit padding.
1379 unsigned NumBytes = Opers.getNumPatchBytes();
1380 assert(NumBytes >= EncodedBytes &&
1381 "Patchpoint can't request size less than the length of a call.");
1382 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1383 "Invalid number of NOP bytes requested!");
1384 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1385 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1386}
1387
1388void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1389 const MachineInstr &MI) {
1390 StatepointOpers SOpers(&MI);
1391 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1392 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1393 for (unsigned i = 0; i < PatchBytes; i += 4)
1394 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1395 } else {
1396 // Lower call target and choose correct opcode
1397 const MachineOperand &CallTarget = SOpers.getCallTarget();
1398 MCOperand CallTargetMCOp;
1399 unsigned CallOpcode;
1400 switch (CallTarget.getType()) {
1401 case MachineOperand::MO_GlobalAddress:
1402 case MachineOperand::MO_ExternalSymbol:
1403 MCInstLowering.lowerOperand(MO: CallTarget, MCOp&: CallTargetMCOp);
1404 CallOpcode = AArch64::BL;
1405 break;
1406 case MachineOperand::MO_Immediate:
1407 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
1408 CallOpcode = AArch64::BL;
1409 break;
1410 case MachineOperand::MO_Register:
1411 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
1412 CallOpcode = AArch64::BLR;
1413 break;
1414 default:
1415 llvm_unreachable("Unsupported operand type in statepoint call target");
1416 break;
1417 }
1418
1419 EmitToStreamer(S&: OutStreamer,
1420 Inst: MCInstBuilder(CallOpcode).addOperand(Op: CallTargetMCOp));
1421 }
1422
1423 auto &Ctx = OutStreamer.getContext();
1424 MCSymbol *MILabel = Ctx.createTempSymbol();
1425 OutStreamer.emitLabel(Symbol: MILabel);
1426 SM.recordStatepoint(L: *MILabel, MI);
1427}
1428
1429void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1430 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1431 // <opcode>, <operands>
1432
1433 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
1434 FaultMaps::FaultKind FK =
1435 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
1436 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
1437 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
1438 unsigned OperandsBeginIdx = 4;
1439
1440 auto &Ctx = OutStreamer->getContext();
1441 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1442 OutStreamer->emitLabel(Symbol: FaultingLabel);
1443
1444 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1445 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
1446
1447 MCInst MI;
1448 MI.setOpcode(Opcode);
1449
1450 if (DefRegister != (Register)0)
1451 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
1452
1453 for (const MachineOperand &MO :
1454 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx)) {
1455 MCOperand Dest;
1456 lowerOperand(MO, MCOp&: Dest);
1457 MI.addOperand(Op: Dest);
1458 }
1459
1460 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
1461 OutStreamer->emitInstruction(Inst: MI, STI: getSubtargetInfo());
1462}
1463
1464void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1465 Register DestReg = MI.getOperand(i: 0).getReg();
1466 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1467 STI->isNeonAvailable()) {
1468 // Convert H/S register to corresponding D register
1469 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1470 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1471 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1472 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1473 else
1474 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1475
1476 MCInst MOVI;
1477 MOVI.setOpcode(AArch64::MOVID);
1478 MOVI.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1479 MOVI.addOperand(Op: MCOperand::createImm(Val: 0));
1480 EmitToStreamer(S&: *OutStreamer, Inst: MOVI);
1481 } else {
1482 MCInst FMov;
1483 switch (MI.getOpcode()) {
1484 default: llvm_unreachable("Unexpected opcode");
1485 case AArch64::FMOVH0:
1486 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1487 if (!STI->hasFullFP16())
1488 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1489 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1490 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1491 break;
1492 case AArch64::FMOVS0:
1493 FMov.setOpcode(AArch64::FMOVWSr);
1494 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1495 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1496 break;
1497 case AArch64::FMOVD0:
1498 FMov.setOpcode(AArch64::FMOVXDr);
1499 FMov.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1500 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1501 break;
1502 }
1503 EmitToStreamer(S&: *OutStreamer, Inst: FMov);
1504 }
1505}
1506
1507// Simple pseudo-instructions have their lowering (with expansion to real
1508// instructions) auto-generated.
1509#include "AArch64GenMCPseudoLowering.inc"
1510
1511void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1512 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
1513
1514 // Do any auto-generated pseudo lowerings.
1515 if (emitPseudoExpansionLowering(OutStreamer&: *OutStreamer, MI))
1516 return;
1517
1518 if (MI->getOpcode() == AArch64::ADRP) {
1519 for (auto &Opd : MI->operands()) {
1520 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
1521 "swift_async_extendedFramePointerFlags") {
1522 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
1523 }
1524 }
1525 }
1526
1527 if (AArch64FI->getLOHRelated().count(Ptr: MI)) {
1528 // Generate a label for LOH related instruction
1529 MCSymbol *LOHLabel = createTempSymbol(Name: "loh");
1530 // Associate the instruction with the label
1531 LOHInstToLabel[MI] = LOHLabel;
1532 OutStreamer->emitLabel(Symbol: LOHLabel);
1533 }
1534
1535 AArch64TargetStreamer *TS =
1536 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1537 // Do any manual lowerings.
1538 switch (MI->getOpcode()) {
1539 default:
1540 break;
1541 case AArch64::HINT: {
1542 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1543 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1544 // non-empty. If MI is the initial BTI, place the
1545 // __patchable_function_entries label after BTI.
1546 if (CurrentPatchableFunctionEntrySym &&
1547 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1548 MI == &MF->front().front()) {
1549 int64_t Imm = MI->getOperand(i: 0).getImm();
1550 if ((Imm & 32) && (Imm & 6)) {
1551 MCInst Inst;
1552 MCInstLowering.Lower(MI, OutMI&: Inst);
1553 EmitToStreamer(S&: *OutStreamer, Inst);
1554 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
1555 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
1556 return;
1557 }
1558 }
1559 break;
1560 }
1561 case AArch64::MOVMCSym: {
1562 Register DestReg = MI->getOperand(i: 0).getReg();
1563 const MachineOperand &MO_Sym = MI->getOperand(i: 1);
1564 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1565 MCOperand Hi_MCSym, Lo_MCSym;
1566
1567 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1568 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1569
1570 MCInstLowering.lowerOperand(MO: Hi_MOSym, MCOp&: Hi_MCSym);
1571 MCInstLowering.lowerOperand(MO: Lo_MOSym, MCOp&: Lo_MCSym);
1572
1573 MCInst MovZ;
1574 MovZ.setOpcode(AArch64::MOVZXi);
1575 MovZ.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1576 MovZ.addOperand(Op: Hi_MCSym);
1577 MovZ.addOperand(Op: MCOperand::createImm(Val: 16));
1578 EmitToStreamer(S&: *OutStreamer, Inst: MovZ);
1579
1580 MCInst MovK;
1581 MovK.setOpcode(AArch64::MOVKXi);
1582 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1583 MovK.addOperand(Op: MCOperand::createReg(Reg: DestReg));
1584 MovK.addOperand(Op: Lo_MCSym);
1585 MovK.addOperand(Op: MCOperand::createImm(Val: 0));
1586 EmitToStreamer(S&: *OutStreamer, Inst: MovK);
1587 return;
1588 }
1589 case AArch64::MOVIv2d_ns:
1590 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
1591 // as movi is more efficient across all cores. Newer cores can eliminate
1592 // fmovs early and there is no difference with movi, but this not true for
1593 // all implementations.
1594 //
1595 // The floating-point version doesn't quite work in rare cases on older
1596 // CPUs, so on those targets we lower this instruction to movi.16b instead.
1597 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1598 MI->getOperand(i: 1).getImm() == 0) {
1599 MCInst TmpInst;
1600 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1601 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
1602 TmpInst.addOperand(Op: MCOperand::createImm(Val: MI->getOperand(i: 1).getImm()));
1603 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
1604 return;
1605 }
1606 break;
1607
1608 case AArch64::DBG_VALUE:
1609 case AArch64::DBG_VALUE_LIST:
1610 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1611 SmallString<128> TmpStr;
1612 raw_svector_ostream OS(TmpStr);
1613 PrintDebugValueComment(MI, OS);
1614 OutStreamer->emitRawText(String: StringRef(OS.str()));
1615 }
1616 return;
1617
1618 case AArch64::EMITBKEY: {
1619 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1620 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1621 ExceptionHandlingType != ExceptionHandling::ARM)
1622 return;
1623
1624 if (getFunctionCFISectionType(MF: *MF) == CFISection::None)
1625 return;
1626
1627 OutStreamer->emitCFIBKeyFrame();
1628 return;
1629 }
1630
1631 case AArch64::EMITMTETAGGED: {
1632 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1633 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1634 ExceptionHandlingType != ExceptionHandling::ARM)
1635 return;
1636
1637 if (getFunctionCFISectionType(MF: *MF) != CFISection::None)
1638 OutStreamer->emitCFIMTETaggedFrame();
1639 return;
1640 }
1641
1642 // Tail calls use pseudo instructions so they have the proper code-gen
1643 // attributes (isCall, isReturn, etc.). We lower them to the real
1644 // instruction here.
1645 case AArch64::TCRETURNri:
1646 case AArch64::TCRETURNrix16x17:
1647 case AArch64::TCRETURNrix17:
1648 case AArch64::TCRETURNrinotx16:
1649 case AArch64::TCRETURNriALL: {
1650 MCInst TmpInst;
1651 TmpInst.setOpcode(AArch64::BR);
1652 TmpInst.addOperand(Op: MCOperand::createReg(Reg: MI->getOperand(i: 0).getReg()));
1653 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
1654 return;
1655 }
1656 case AArch64::TCRETURNdi: {
1657 MCOperand Dest;
1658 MCInstLowering.lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
1659 MCInst TmpInst;
1660 TmpInst.setOpcode(AArch64::B);
1661 TmpInst.addOperand(Op: Dest);
1662 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
1663 return;
1664 }
1665 case AArch64::SpeculationBarrierISBDSBEndBB: {
1666 // Print DSB SYS + ISB
1667 MCInst TmpInstDSB;
1668 TmpInstDSB.setOpcode(AArch64::DSB);
1669 TmpInstDSB.addOperand(Op: MCOperand::createImm(Val: 0xf));
1670 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstDSB);
1671 MCInst TmpInstISB;
1672 TmpInstISB.setOpcode(AArch64::ISB);
1673 TmpInstISB.addOperand(Op: MCOperand::createImm(Val: 0xf));
1674 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstISB);
1675 return;
1676 }
1677 case AArch64::SpeculationBarrierSBEndBB: {
1678 // Print SB
1679 MCInst TmpInstSB;
1680 TmpInstSB.setOpcode(AArch64::SB);
1681 EmitToStreamer(S&: *OutStreamer, Inst: TmpInstSB);
1682 return;
1683 }
1684 case AArch64::TLSDESC_CALLSEQ: {
1685 /// lower this to:
1686 /// adrp x0, :tlsdesc:var
1687 /// ldr x1, [x0, #:tlsdesc_lo12:var]
1688 /// add x0, x0, #:tlsdesc_lo12:var
1689 /// .tlsdesccall var
1690 /// blr x1
1691 /// (TPIDR_EL0 offset now in x0)
1692 const MachineOperand &MO_Sym = MI->getOperand(i: 0);
1693 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1694 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1695 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1696 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1697 MCInstLowering.lowerOperand(MO: MO_Sym, MCOp&: Sym);
1698 MCInstLowering.lowerOperand(MO: MO_TLSDESC_LO12, MCOp&: SymTLSDescLo12);
1699 MCInstLowering.lowerOperand(MO: MO_TLSDESC, MCOp&: SymTLSDesc);
1700
1701 MCInst Adrp;
1702 Adrp.setOpcode(AArch64::ADRP);
1703 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1704 Adrp.addOperand(Op: SymTLSDesc);
1705 EmitToStreamer(S&: *OutStreamer, Inst: Adrp);
1706
1707 MCInst Ldr;
1708 if (STI->isTargetILP32()) {
1709 Ldr.setOpcode(AArch64::LDRWui);
1710 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1711 } else {
1712 Ldr.setOpcode(AArch64::LDRXui);
1713 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1714 }
1715 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1716 Ldr.addOperand(Op: SymTLSDescLo12);
1717 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
1718 EmitToStreamer(S&: *OutStreamer, Inst: Ldr);
1719
1720 MCInst Add;
1721 if (STI->isTargetILP32()) {
1722 Add.setOpcode(AArch64::ADDWri);
1723 Add.addOperand(MCOperand::createReg(AArch64::W0));
1724 Add.addOperand(MCOperand::createReg(AArch64::W0));
1725 } else {
1726 Add.setOpcode(AArch64::ADDXri);
1727 Add.addOperand(MCOperand::createReg(AArch64::X0));
1728 Add.addOperand(MCOperand::createReg(AArch64::X0));
1729 }
1730 Add.addOperand(Op: SymTLSDescLo12);
1731 Add.addOperand(Op: MCOperand::createImm(Val: AArch64_AM::getShiftValue(Imm: 0)));
1732 EmitToStreamer(S&: *OutStreamer, Inst: Add);
1733
1734 // Emit a relocation-annotation. This expands to no code, but requests
1735 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1736 MCInst TLSDescCall;
1737 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1738 TLSDescCall.addOperand(Op: Sym);
1739 EmitToStreamer(S&: *OutStreamer, Inst: TLSDescCall);
1740
1741 MCInst Blr;
1742 Blr.setOpcode(AArch64::BLR);
1743 Blr.addOperand(MCOperand::createReg(AArch64::X1));
1744 EmitToStreamer(S&: *OutStreamer, Inst: Blr);
1745
1746 return;
1747 }
1748
1749 case AArch64::JumpTableDest32:
1750 case AArch64::JumpTableDest16:
1751 case AArch64::JumpTableDest8:
1752 LowerJumpTableDest(OutStreamer&: *OutStreamer, MI: *MI);
1753 return;
1754
1755 case AArch64::FMOVH0:
1756 case AArch64::FMOVS0:
1757 case AArch64::FMOVD0:
1758 emitFMov0(MI: *MI);
1759 return;
1760
1761 case AArch64::MOPSMemoryCopyPseudo:
1762 case AArch64::MOPSMemoryMovePseudo:
1763 case AArch64::MOPSMemorySetPseudo:
1764 case AArch64::MOPSMemorySetTaggingPseudo:
1765 LowerMOPS(OutStreamer&: *OutStreamer, MI: *MI);
1766 return;
1767
1768 case TargetOpcode::STACKMAP:
1769 return LowerSTACKMAP(OutStreamer&: *OutStreamer, SM, MI: *MI);
1770
1771 case TargetOpcode::PATCHPOINT:
1772 return LowerPATCHPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
1773
1774 case TargetOpcode::STATEPOINT:
1775 return LowerSTATEPOINT(OutStreamer&: *OutStreamer, SM, MI: *MI);
1776
1777 case TargetOpcode::FAULTING_OP:
1778 return LowerFAULTING_OP(FaultingMI: *MI);
1779
1780 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1781 LowerPATCHABLE_FUNCTION_ENTER(MI: *MI);
1782 return;
1783
1784 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1785 LowerPATCHABLE_FUNCTION_EXIT(MI: *MI);
1786 return;
1787
1788 case TargetOpcode::PATCHABLE_TAIL_CALL:
1789 LowerPATCHABLE_TAIL_CALL(MI: *MI);
1790 return;
1791 case TargetOpcode::PATCHABLE_EVENT_CALL:
1792 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: false);
1793 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
1794 return LowerPATCHABLE_EVENT_CALL(MI: *MI, Typed: true);
1795
1796 case AArch64::KCFI_CHECK:
1797 LowerKCFI_CHECK(MI: *MI);
1798 return;
1799
1800 case AArch64::HWASAN_CHECK_MEMACCESS:
1801 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1802 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
1803 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
1804 LowerHWASAN_CHECK_MEMACCESS(MI: *MI);
1805 return;
1806
1807 case AArch64::SEH_StackAlloc:
1808 TS->emitARM64WinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
1809 return;
1810
1811 case AArch64::SEH_SaveFPLR:
1812 TS->emitARM64WinCFISaveFPLR(Offset: MI->getOperand(i: 0).getImm());
1813 return;
1814
1815 case AArch64::SEH_SaveFPLR_X:
1816 assert(MI->getOperand(0).getImm() < 0 &&
1817 "Pre increment SEH opcode must have a negative offset");
1818 TS->emitARM64WinCFISaveFPLRX(Offset: -MI->getOperand(i: 0).getImm());
1819 return;
1820
1821 case AArch64::SEH_SaveReg:
1822 TS->emitARM64WinCFISaveReg(Reg: MI->getOperand(i: 0).getImm(),
1823 Offset: MI->getOperand(i: 1).getImm());
1824 return;
1825
1826 case AArch64::SEH_SaveReg_X:
1827 assert(MI->getOperand(1).getImm() < 0 &&
1828 "Pre increment SEH opcode must have a negative offset");
1829 TS->emitARM64WinCFISaveRegX(Reg: MI->getOperand(i: 0).getImm(),
1830 Offset: -MI->getOperand(i: 1).getImm());
1831 return;
1832
1833 case AArch64::SEH_SaveRegP:
1834 if (MI->getOperand(i: 1).getImm() == 30 && MI->getOperand(i: 0).getImm() >= 19 &&
1835 MI->getOperand(i: 0).getImm() <= 28) {
1836 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1837 "Register paired with LR must be odd");
1838 TS->emitARM64WinCFISaveLRPair(Reg: MI->getOperand(i: 0).getImm(),
1839 Offset: MI->getOperand(i: 2).getImm());
1840 return;
1841 }
1842 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1843 "Non-consecutive registers not allowed for save_regp");
1844 TS->emitARM64WinCFISaveRegP(Reg: MI->getOperand(i: 0).getImm(),
1845 Offset: MI->getOperand(i: 2).getImm());
1846 return;
1847
1848 case AArch64::SEH_SaveRegP_X:
1849 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1850 "Non-consecutive registers not allowed for save_regp_x");
1851 assert(MI->getOperand(2).getImm() < 0 &&
1852 "Pre increment SEH opcode must have a negative offset");
1853 TS->emitARM64WinCFISaveRegPX(Reg: MI->getOperand(i: 0).getImm(),
1854 Offset: -MI->getOperand(i: 2).getImm());
1855 return;
1856
1857 case AArch64::SEH_SaveFReg:
1858 TS->emitARM64WinCFISaveFReg(Reg: MI->getOperand(i: 0).getImm(),
1859 Offset: MI->getOperand(i: 1).getImm());
1860 return;
1861
1862 case AArch64::SEH_SaveFReg_X:
1863 assert(MI->getOperand(1).getImm() < 0 &&
1864 "Pre increment SEH opcode must have a negative offset");
1865 TS->emitARM64WinCFISaveFRegX(Reg: MI->getOperand(i: 0).getImm(),
1866 Offset: -MI->getOperand(i: 1).getImm());
1867 return;
1868
1869 case AArch64::SEH_SaveFRegP:
1870 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1871 "Non-consecutive registers not allowed for save_regp");
1872 TS->emitARM64WinCFISaveFRegP(Reg: MI->getOperand(i: 0).getImm(),
1873 Offset: MI->getOperand(i: 2).getImm());
1874 return;
1875
1876 case AArch64::SEH_SaveFRegP_X:
1877 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1878 "Non-consecutive registers not allowed for save_regp_x");
1879 assert(MI->getOperand(2).getImm() < 0 &&
1880 "Pre increment SEH opcode must have a negative offset");
1881 TS->emitARM64WinCFISaveFRegPX(Reg: MI->getOperand(i: 0).getImm(),
1882 Offset: -MI->getOperand(i: 2).getImm());
1883 return;
1884
1885 case AArch64::SEH_SetFP:
1886 TS->emitARM64WinCFISetFP();
1887 return;
1888
1889 case AArch64::SEH_AddFP:
1890 TS->emitARM64WinCFIAddFP(Size: MI->getOperand(i: 0).getImm());
1891 return;
1892
1893 case AArch64::SEH_Nop:
1894 TS->emitARM64WinCFINop();
1895 return;
1896
1897 case AArch64::SEH_PrologEnd:
1898 TS->emitARM64WinCFIPrologEnd();
1899 return;
1900
1901 case AArch64::SEH_EpilogStart:
1902 TS->emitARM64WinCFIEpilogStart();
1903 return;
1904
1905 case AArch64::SEH_EpilogEnd:
1906 TS->emitARM64WinCFIEpilogEnd();
1907 return;
1908
1909 case AArch64::SEH_PACSignLR:
1910 TS->emitARM64WinCFIPACSignLR();
1911 return;
1912
1913 case AArch64::SEH_SaveAnyRegQP:
1914 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
1915 "Non-consecutive registers not allowed for save_any_reg");
1916 assert(MI->getOperand(2).getImm() >= 0 &&
1917 "SaveAnyRegQP SEH opcode offset must be non-negative");
1918 assert(MI->getOperand(2).getImm() <= 1008 &&
1919 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
1920 TS->emitARM64WinCFISaveAnyRegQP(Reg: MI->getOperand(i: 0).getImm(),
1921 Offset: MI->getOperand(i: 2).getImm());
1922 return;
1923
1924 case AArch64::SEH_SaveAnyRegQPX:
1925 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
1926 "Non-consecutive registers not allowed for save_any_reg");
1927 assert(MI->getOperand(2).getImm() < 0 &&
1928 "SaveAnyRegQPX SEH opcode offset must be negative");
1929 assert(MI->getOperand(2).getImm() >= -1008 &&
1930 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
1931 TS->emitARM64WinCFISaveAnyRegQPX(Reg: MI->getOperand(i: 0).getImm(),
1932 Offset: -MI->getOperand(i: 2).getImm());
1933 return;
1934 }
1935
1936 // Finally, do the automated lowerings for everything else.
1937 MCInst TmpInst;
1938 MCInstLowering.Lower(MI, OutMI&: TmpInst);
1939 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
1940}
1941
1942void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
1943 MCSymbol *LazyPointer) {
1944 // _ifunc:
1945 // adrp x16, lazy_pointer@GOTPAGE
1946 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
1947 // ldr x16, [x16]
1948 // br x16
1949
1950 {
1951 MCInst Adrp;
1952 Adrp.setOpcode(AArch64::ADRP);
1953 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
1954 MCOperand SymPage;
1955 MCInstLowering.lowerOperand(
1956 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer,
1957 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
1958 MCOp&: SymPage);
1959 Adrp.addOperand(Op: SymPage);
1960 OutStreamer->emitInstruction(Adrp, *STI);
1961 }
1962
1963 {
1964 MCInst Ldr;
1965 Ldr.setOpcode(AArch64::LDRXui);
1966 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
1967 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
1968 MCOperand SymPageOff;
1969 MCInstLowering.lowerOperand(
1970 MO: MachineOperand::CreateMCSymbol(Sym: LazyPointer, TargetFlags: AArch64II::MO_GOT |
1971 AArch64II::MO_PAGEOFF),
1972 MCOp&: SymPageOff);
1973 Ldr.addOperand(Op: SymPageOff);
1974 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
1975 OutStreamer->emitInstruction(Ldr, *STI);
1976 }
1977
1978 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui)
1979 .addReg(AArch64::X16)
1980 .addReg(AArch64::X16)
1981 .addImm(0),
1982 *STI);
1983
1984 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
1985 ? AArch64::BRAAZ
1986 : AArch64::BR)
1987 .addReg(AArch64::X16),
1988 *STI);
1989}
1990
1991void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
1992 const GlobalIFunc &GI,
1993 MCSymbol *LazyPointer) {
1994 // These stub helpers are only ever called once, so here we're optimizing for
1995 // minimum size by using the pre-indexed store variants, which saves a few
1996 // bytes of instructions to bump & restore sp.
1997
1998 // _ifunc.stub_helper:
1999 // stp fp, lr, [sp, #-16]!
2000 // mov fp, sp
2001 // stp x1, x0, [sp, #-16]!
2002 // stp x3, x2, [sp, #-16]!
2003 // stp x5, x4, [sp, #-16]!
2004 // stp x7, x6, [sp, #-16]!
2005 // stp d1, d0, [sp, #-16]!
2006 // stp d3, d2, [sp, #-16]!
2007 // stp d5, d4, [sp, #-16]!
2008 // stp d7, d6, [sp, #-16]!
2009 // bl _resolver
2010 // adrp x16, lazy_pointer@GOTPAGE
2011 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
2012 // str x0, [x16]
2013 // mov x16, x0
2014 // ldp d7, d6, [sp], #16
2015 // ldp d5, d4, [sp], #16
2016 // ldp d3, d2, [sp], #16
2017 // ldp d1, d0, [sp], #16
2018 // ldp x7, x6, [sp], #16
2019 // ldp x5, x4, [sp], #16
2020 // ldp x3, x2, [sp], #16
2021 // ldp x1, x0, [sp], #16
2022 // ldp fp, lr, [sp], #16
2023 // br x16
2024
2025 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
2026 .addReg(AArch64::SP)
2027 .addReg(AArch64::FP)
2028 .addReg(AArch64::LR)
2029 .addReg(AArch64::SP)
2030 .addImm(-2),
2031 *STI);
2032
2033 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
2034 .addReg(AArch64::FP)
2035 .addReg(AArch64::SP)
2036 .addImm(0)
2037 .addImm(0),
2038 *STI);
2039
2040 for (int I = 0; I != 4; ++I)
2041 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
2042 .addReg(AArch64::SP)
2043 .addReg(AArch64::X1 + 2 * I)
2044 .addReg(AArch64::X0 + 2 * I)
2045 .addReg(AArch64::SP)
2046 .addImm(-2),
2047 *STI);
2048
2049 for (int I = 0; I != 4; ++I)
2050 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre)
2051 .addReg(AArch64::SP)
2052 .addReg(AArch64::D1 + 2 * I)
2053 .addReg(AArch64::D0 + 2 * I)
2054 .addReg(AArch64::SP)
2055 .addImm(-2),
2056 *STI);
2057
2058 OutStreamer->emitInstruction(
2059 MCInstBuilder(AArch64::BL)
2060 .addOperand(MCOperand::createExpr(lowerConstant(GI.getResolver()))),
2061 *STI);
2062
2063 {
2064 MCInst Adrp;
2065 Adrp.setOpcode(AArch64::ADRP);
2066 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
2067 MCOperand SymPage;
2068 MCInstLowering.lowerOperand(
2069 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
2070 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGE),
2071 MCOp&: SymPage);
2072 Adrp.addOperand(Op: SymPage);
2073 OutStreamer->emitInstruction(Adrp, *STI);
2074 }
2075
2076 {
2077 MCInst Ldr;
2078 Ldr.setOpcode(AArch64::LDRXui);
2079 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2080 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2081 MCOperand SymPageOff;
2082 MCInstLowering.lowerOperand(
2083 MO: MachineOperand::CreateES(SymName: LazyPointer->getName().data() + 1,
2084 TargetFlags: AArch64II::MO_GOT | AArch64II::MO_PAGEOFF),
2085 MCOp&: SymPageOff);
2086 Ldr.addOperand(Op: SymPageOff);
2087 Ldr.addOperand(Op: MCOperand::createImm(Val: 0));
2088 OutStreamer->emitInstruction(Ldr, *STI);
2089 }
2090
2091 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui)
2092 .addReg(AArch64::X0)
2093 .addReg(AArch64::X16)
2094 .addImm(0),
2095 *STI);
2096
2097 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
2098 .addReg(AArch64::X16)
2099 .addReg(AArch64::X0)
2100 .addImm(0)
2101 .addImm(0),
2102 *STI);
2103
2104 for (int I = 3; I != -1; --I)
2105 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost)
2106 .addReg(AArch64::SP)
2107 .addReg(AArch64::D1 + 2 * I)
2108 .addReg(AArch64::D0 + 2 * I)
2109 .addReg(AArch64::SP)
2110 .addImm(2),
2111 *STI);
2112
2113 for (int I = 3; I != -1; --I)
2114 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
2115 .addReg(AArch64::SP)
2116 .addReg(AArch64::X1 + 2 * I)
2117 .addReg(AArch64::X0 + 2 * I)
2118 .addReg(AArch64::SP)
2119 .addImm(2),
2120 *STI);
2121
2122 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
2123 .addReg(AArch64::SP)
2124 .addReg(AArch64::FP)
2125 .addReg(AArch64::LR)
2126 .addReg(AArch64::SP)
2127 .addImm(2),
2128 *STI);
2129
2130 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
2131 ? AArch64::BRAAZ
2132 : AArch64::BR)
2133 .addReg(AArch64::X16),
2134 *STI);
2135}
2136
2137const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV) {
2138 if (const GlobalValue *GV = dyn_cast<GlobalValue>(Val: CV)) {
2139 return MCSymbolRefExpr::create(Symbol: MCInstLowering.GetGlobalValueSymbol(GV, TargetFlags: 0),
2140 Ctx&: OutContext);
2141 }
2142
2143 return AsmPrinter::lowerConstant(CV);
2144}
2145
2146// Force static initialization.
2147extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
2148 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
2149 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
2150 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
2151 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
2152 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
2153}
2154

source code of llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp