1//===- bolt/Target/AArch64/AArch64MCPlusBuilder.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides AArch64-specific MCPlus builder.
10//
11//===----------------------------------------------------------------------===//
12
13#include "MCTargetDesc/AArch64AddressingModes.h"
14#include "MCTargetDesc/AArch64FixupKinds.h"
15#include "MCTargetDesc/AArch64MCExpr.h"
16#include "MCTargetDesc/AArch64MCTargetDesc.h"
17#include "Utils/AArch64BaseInfo.h"
18#include "bolt/Core/MCPlusBuilder.h"
19#include "llvm/BinaryFormat/ELF.h"
20#include "llvm/MC/MCContext.h"
21#include "llvm/MC/MCFixupKindInfo.h"
22#include "llvm/MC/MCInstBuilder.h"
23#include "llvm/MC/MCInstrInfo.h"
24#include "llvm/MC/MCRegisterInfo.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/ErrorHandling.h"
27
28#define DEBUG_TYPE "mcplus"
29
30using namespace llvm;
31using namespace bolt;
32
33namespace {
34
35static void getSystemFlag(MCInst &Inst, MCPhysReg RegName) {
36 Inst.setOpcode(AArch64::MRS);
37 Inst.clear();
38 Inst.addOperand(Op: MCOperand::createReg(Reg: RegName));
39 Inst.addOperand(Op: MCOperand::createImm(AArch64SysReg::Val: NZCV));
40}
41
42static void setSystemFlag(MCInst &Inst, MCPhysReg RegName) {
43 Inst.setOpcode(AArch64::MSR);
44 Inst.clear();
45 Inst.addOperand(Op: MCOperand::createImm(AArch64SysReg::Val: NZCV));
46 Inst.addOperand(Op: MCOperand::createReg(Reg: RegName));
47}
48
49static void createPushRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {
50 Inst.clear();
51 unsigned NewOpcode = AArch64::STPXpre;
52 Inst.setOpcode(NewOpcode);
53 Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
54 Inst.addOperand(Op: MCOperand::createReg(Reg: Reg1));
55 Inst.addOperand(Op: MCOperand::createReg(Reg: Reg2));
56 Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
57 Inst.addOperand(Op: MCOperand::createImm(Val: -2));
58}
59
60static void createPopRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {
61 Inst.clear();
62 unsigned NewOpcode = AArch64::LDPXpost;
63 Inst.setOpcode(NewOpcode);
64 Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
65 Inst.addOperand(Op: MCOperand::createReg(Reg: Reg1));
66 Inst.addOperand(Op: MCOperand::createReg(Reg: Reg2));
67 Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
68 Inst.addOperand(Op: MCOperand::createImm(Val: 2));
69}
70
71static void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From) {
72 Inst.setOpcode(AArch64::LDRXui);
73 Inst.clear();
74 if (From == AArch64::SP) {
75 Inst.setOpcode(AArch64::LDRXpost);
76 Inst.addOperand(Op: MCOperand::createReg(Reg: From));
77 Inst.addOperand(Op: MCOperand::createReg(Reg: To));
78 Inst.addOperand(Op: MCOperand::createReg(Reg: From));
79 Inst.addOperand(Op: MCOperand::createImm(Val: 16));
80 } else {
81 Inst.addOperand(Op: MCOperand::createReg(Reg: To));
82 Inst.addOperand(Op: MCOperand::createReg(Reg: From));
83 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
84 }
85}
86
87static void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To) {
88 Inst.setOpcode(AArch64::STRXui);
89 Inst.clear();
90 if (To == AArch64::SP) {
91 Inst.setOpcode(AArch64::STRXpre);
92 Inst.addOperand(Op: MCOperand::createReg(Reg: To));
93 Inst.addOperand(Op: MCOperand::createReg(Reg: From));
94 Inst.addOperand(Op: MCOperand::createReg(Reg: To));
95 Inst.addOperand(Op: MCOperand::createImm(Val: -16));
96 } else {
97 Inst.addOperand(Op: MCOperand::createReg(Reg: From));
98 Inst.addOperand(Op: MCOperand::createReg(Reg: To));
99 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
100 }
101}
102
103static void atomicAdd(MCInst &Inst, MCPhysReg RegTo, MCPhysReg RegCnt) {
104 // NOTE: Supports only ARM with LSE extension
105 Inst.setOpcode(AArch64::LDADDX);
106 Inst.clear();
107 Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: XZR));
108 Inst.addOperand(Op: MCOperand::createReg(Reg: RegCnt));
109 Inst.addOperand(Op: MCOperand::createReg(Reg: RegTo));
110}
111
112static void createMovz(MCInst &Inst, MCPhysReg Reg, uint64_t Imm) {
113 assert(Imm <= UINT16_MAX && "Invalid Imm size");
114 Inst.clear();
115 Inst.setOpcode(AArch64::MOVZXi);
116 Inst.addOperand(Op: MCOperand::createReg(Reg));
117 Inst.addOperand(Op: MCOperand::createImm(Val: Imm & 0xFFFF));
118 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
119}
120
121static InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegTmp) {
122 InstructionListType Insts;
123 Insts.emplace_back();
124 createMovz(Inst&: Insts.back(), Reg: RegTmp, Imm: 1);
125 Insts.emplace_back();
126 atomicAdd(Inst&: Insts.back(), RegTo, RegCnt: RegTmp);
127 return Insts;
128}
129class AArch64MCPlusBuilder : public MCPlusBuilder {
130public:
131 using MCPlusBuilder::MCPlusBuilder;
132
133 bool equals(const MCTargetExpr &A, const MCTargetExpr &B,
134 CompFuncTy Comp) const override {
135 const auto &AArch64ExprA = cast<AArch64MCExpr>(Val: A);
136 const auto &AArch64ExprB = cast<AArch64MCExpr>(Val: B);
137 if (AArch64ExprA.getKind() != AArch64ExprB.getKind())
138 return false;
139
140 return MCPlusBuilder::equals(A: *AArch64ExprA.getSubExpr(),
141 B: *AArch64ExprB.getSubExpr(), Comp);
142 }
143
144 bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
145 return false;
146 }
147
148 bool shortenInstruction(MCInst &, const MCSubtargetInfo &) const override {
149 return false;
150 }
151
152 bool isADRP(const MCInst &Inst) const override {
153 return Inst.getOpcode() == AArch64::ADRP;
154 }
155
156 bool isADR(const MCInst &Inst) const override {
157 return Inst.getOpcode() == AArch64::ADR;
158 }
159
160 bool isAddXri(const MCInst &Inst) const {
161 return Inst.getOpcode() == AArch64::ADDXri;
162 }
163
164 void getADRReg(const MCInst &Inst, MCPhysReg &RegName) const override {
165 assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
166 assert(MCPlus::getNumPrimeOperands(Inst) != 0 &&
167 "No operands for ADR instruction");
168 assert(Inst.getOperand(0).isReg() &&
169 "Unexpected operand in ADR instruction");
170 RegName = Inst.getOperand(i: 0).getReg();
171 }
172
173 bool isTB(const MCInst &Inst) const {
174 return (Inst.getOpcode() == AArch64::TBNZW ||
175 Inst.getOpcode() == AArch64::TBNZX ||
176 Inst.getOpcode() == AArch64::TBZW ||
177 Inst.getOpcode() == AArch64::TBZX);
178 }
179
180 bool isCB(const MCInst &Inst) const {
181 return (Inst.getOpcode() == AArch64::CBNZW ||
182 Inst.getOpcode() == AArch64::CBNZX ||
183 Inst.getOpcode() == AArch64::CBZW ||
184 Inst.getOpcode() == AArch64::CBZX);
185 }
186
187 bool isMOVW(const MCInst &Inst) const {
188 return (Inst.getOpcode() == AArch64::MOVKWi ||
189 Inst.getOpcode() == AArch64::MOVKXi ||
190 Inst.getOpcode() == AArch64::MOVNWi ||
191 Inst.getOpcode() == AArch64::MOVNXi ||
192 Inst.getOpcode() == AArch64::MOVZXi ||
193 Inst.getOpcode() == AArch64::MOVZWi);
194 }
195
196 bool isADD(const MCInst &Inst) const {
197 return (Inst.getOpcode() == AArch64::ADDSWri ||
198 Inst.getOpcode() == AArch64::ADDSWrr ||
199 Inst.getOpcode() == AArch64::ADDSWrs ||
200 Inst.getOpcode() == AArch64::ADDSWrx ||
201 Inst.getOpcode() == AArch64::ADDSXri ||
202 Inst.getOpcode() == AArch64::ADDSXrr ||
203 Inst.getOpcode() == AArch64::ADDSXrs ||
204 Inst.getOpcode() == AArch64::ADDSXrx ||
205 Inst.getOpcode() == AArch64::ADDSXrx64 ||
206 Inst.getOpcode() == AArch64::ADDWri ||
207 Inst.getOpcode() == AArch64::ADDWrr ||
208 Inst.getOpcode() == AArch64::ADDWrs ||
209 Inst.getOpcode() == AArch64::ADDWrx ||
210 Inst.getOpcode() == AArch64::ADDXri ||
211 Inst.getOpcode() == AArch64::ADDXrr ||
212 Inst.getOpcode() == AArch64::ADDXrs ||
213 Inst.getOpcode() == AArch64::ADDXrx ||
214 Inst.getOpcode() == AArch64::ADDXrx64);
215 }
216
217 bool isLDRB(const MCInst &Inst) const {
218 return (Inst.getOpcode() == AArch64::LDRBBpost ||
219 Inst.getOpcode() == AArch64::LDRBBpre ||
220 Inst.getOpcode() == AArch64::LDRBBroW ||
221 Inst.getOpcode() == AArch64::LDRBBroX ||
222 Inst.getOpcode() == AArch64::LDRBBui ||
223 Inst.getOpcode() == AArch64::LDRSBWpost ||
224 Inst.getOpcode() == AArch64::LDRSBWpre ||
225 Inst.getOpcode() == AArch64::LDRSBWroW ||
226 Inst.getOpcode() == AArch64::LDRSBWroX ||
227 Inst.getOpcode() == AArch64::LDRSBWui ||
228 Inst.getOpcode() == AArch64::LDRSBXpost ||
229 Inst.getOpcode() == AArch64::LDRSBXpre ||
230 Inst.getOpcode() == AArch64::LDRSBXroW ||
231 Inst.getOpcode() == AArch64::LDRSBXroX ||
232 Inst.getOpcode() == AArch64::LDRSBXui);
233 }
234
235 bool isLDRH(const MCInst &Inst) const {
236 return (Inst.getOpcode() == AArch64::LDRHHpost ||
237 Inst.getOpcode() == AArch64::LDRHHpre ||
238 Inst.getOpcode() == AArch64::LDRHHroW ||
239 Inst.getOpcode() == AArch64::LDRHHroX ||
240 Inst.getOpcode() == AArch64::LDRHHui ||
241 Inst.getOpcode() == AArch64::LDRSHWpost ||
242 Inst.getOpcode() == AArch64::LDRSHWpre ||
243 Inst.getOpcode() == AArch64::LDRSHWroW ||
244 Inst.getOpcode() == AArch64::LDRSHWroX ||
245 Inst.getOpcode() == AArch64::LDRSHWui ||
246 Inst.getOpcode() == AArch64::LDRSHXpost ||
247 Inst.getOpcode() == AArch64::LDRSHXpre ||
248 Inst.getOpcode() == AArch64::LDRSHXroW ||
249 Inst.getOpcode() == AArch64::LDRSHXroX ||
250 Inst.getOpcode() == AArch64::LDRSHXui);
251 }
252
253 bool isLDRW(const MCInst &Inst) const {
254 return (Inst.getOpcode() == AArch64::LDRWpost ||
255 Inst.getOpcode() == AArch64::LDRWpre ||
256 Inst.getOpcode() == AArch64::LDRWroW ||
257 Inst.getOpcode() == AArch64::LDRWroX ||
258 Inst.getOpcode() == AArch64::LDRWui);
259 }
260
261 bool isLDRX(const MCInst &Inst) const {
262 return (Inst.getOpcode() == AArch64::LDRXpost ||
263 Inst.getOpcode() == AArch64::LDRXpre ||
264 Inst.getOpcode() == AArch64::LDRXroW ||
265 Inst.getOpcode() == AArch64::LDRXroX ||
266 Inst.getOpcode() == AArch64::LDRXui);
267 }
268
269 bool mayLoad(const MCInst &Inst) const override {
270 return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst);
271 }
272
273 bool isAArch64ExclusiveLoad(const MCInst &Inst) const override {
274 return (Inst.getOpcode() == AArch64::LDXPX ||
275 Inst.getOpcode() == AArch64::LDXPW ||
276 Inst.getOpcode() == AArch64::LDXRX ||
277 Inst.getOpcode() == AArch64::LDXRW ||
278 Inst.getOpcode() == AArch64::LDXRH ||
279 Inst.getOpcode() == AArch64::LDXRB ||
280 Inst.getOpcode() == AArch64::LDAXPX ||
281 Inst.getOpcode() == AArch64::LDAXPW ||
282 Inst.getOpcode() == AArch64::LDAXRX ||
283 Inst.getOpcode() == AArch64::LDAXRW ||
284 Inst.getOpcode() == AArch64::LDAXRH ||
285 Inst.getOpcode() == AArch64::LDAXRB);
286 }
287
288 bool isAArch64ExclusiveStore(const MCInst &Inst) const override {
289 return (Inst.getOpcode() == AArch64::STXPX ||
290 Inst.getOpcode() == AArch64::STXPW ||
291 Inst.getOpcode() == AArch64::STXRX ||
292 Inst.getOpcode() == AArch64::STXRW ||
293 Inst.getOpcode() == AArch64::STXRH ||
294 Inst.getOpcode() == AArch64::STXRB ||
295 Inst.getOpcode() == AArch64::STLXPX ||
296 Inst.getOpcode() == AArch64::STLXPW ||
297 Inst.getOpcode() == AArch64::STLXRX ||
298 Inst.getOpcode() == AArch64::STLXRW ||
299 Inst.getOpcode() == AArch64::STLXRH ||
300 Inst.getOpcode() == AArch64::STLXRB);
301 }
302
303 bool isAArch64ExclusiveClear(const MCInst &Inst) const override {
304 return (Inst.getOpcode() == AArch64::CLREX);
305 }
306
307 bool isLoadFromStack(const MCInst &Inst) const {
308 if (!mayLoad(Inst))
309 return false;
310 for (const MCOperand &Operand : useOperands(Inst)) {
311 if (!Operand.isReg())
312 continue;
313 unsigned Reg = Operand.getReg();
314 if (Reg == AArch64::SP || Reg == AArch64::WSP || Reg == AArch64::FP ||
315 Reg == AArch64::W29)
316 return true;
317 }
318 return false;
319 }
320
321 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
322 MCPhysReg &To) const override {
323 if (Inst.getOpcode() == AArch64::FMOVDXr) {
324 From = Inst.getOperand(i: 1).getReg();
325 To = Inst.getOperand(i: 0).getReg();
326 return true;
327 }
328
329 if (Inst.getOpcode() != AArch64::ORRXrs)
330 return false;
331 if (Inst.getOperand(1).getReg() != AArch64::XZR)
332 return false;
333 if (Inst.getOperand(i: 3).getImm() != 0)
334 return false;
335 From = Inst.getOperand(i: 2).getReg();
336 To = Inst.getOperand(i: 0).getReg();
337 return true;
338 }
339
340 bool isIndirectCall(const MCInst &Inst) const override {
341 return Inst.getOpcode() == AArch64::BLR;
342 }
343
344 MCPhysReg getSpRegister(int Size) const {
345 switch (Size) {
346 case 4:
347 return AArch64::WSP;
348 case 8:
349 return AArch64::SP;
350 default:
351 llvm_unreachable("Unexpected size");
352 }
353 }
354
355 MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
356 switch (ArgNo) {
357 case 0:
358 return AArch64::X0;
359 case 1:
360 return AArch64::X1;
361 case 2:
362 return AArch64::X2;
363 case 3:
364 return AArch64::X3;
365 case 4:
366 return AArch64::X4;
367 case 5:
368 return AArch64::X5;
369 case 6:
370 return AArch64::X6;
371 case 7:
372 return AArch64::X7;
373 default:
374 return getNoRegister();
375 }
376 }
377
378 bool hasPCRelOperand(const MCInst &Inst) const override {
379 // ADRP is blacklisted and is an exception. Even though it has a
380 // PC-relative operand, this operand is not a complete symbol reference
381 // and BOLT shouldn't try to process it in isolation.
382 if (isADRP(Inst))
383 return false;
384
385 if (isADR(Inst))
386 return true;
387
388 // Look for literal addressing mode (see C1-143 ARM DDI 0487B.a)
389 const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
390 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I)
391 if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)
392 return true;
393
394 return false;
395 }
396
397 bool evaluateADR(const MCInst &Inst, int64_t &Imm,
398 const MCExpr **DispExpr) const {
399 assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
400
401 const MCOperand &Label = Inst.getOperand(i: 1);
402 if (!Label.isImm()) {
403 assert(Label.isExpr() && "Unexpected ADR operand");
404 assert(DispExpr && "DispExpr must be set");
405 *DispExpr = Label.getExpr();
406 return false;
407 }
408
409 if (Inst.getOpcode() == AArch64::ADR) {
410 Imm = Label.getImm();
411 return true;
412 }
413 Imm = Label.getImm() << 12;
414 return true;
415 }
416
417 bool evaluateAArch64MemoryOperand(const MCInst &Inst, int64_t &DispImm,
418 const MCExpr **DispExpr = nullptr) const {
419 if (isADR(Inst) || isADRP(Inst))
420 return evaluateADR(Inst, Imm&: DispImm, DispExpr);
421
422 // Literal addressing mode
423 const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
424 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
425 if (MCII.operands()[I].OperandType != MCOI::OPERAND_PCREL)
426 continue;
427
428 if (!Inst.getOperand(i: I).isImm()) {
429 assert(Inst.getOperand(I).isExpr() && "Unexpected PCREL operand");
430 assert(DispExpr && "DispExpr must be set");
431 *DispExpr = Inst.getOperand(i: I).getExpr();
432 return true;
433 }
434
435 DispImm = Inst.getOperand(i: I).getImm() * 4;
436 return true;
437 }
438 return false;
439 }
440
441 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
442 uint64_t Address,
443 uint64_t Size) const override {
444 int64_t DispValue;
445 const MCExpr *DispExpr = nullptr;
446 if (!evaluateAArch64MemoryOperand(Inst, DispImm&: DispValue, DispExpr: &DispExpr))
447 return false;
448
449 // Make sure it's a well-formed addressing we can statically evaluate.
450 if (DispExpr)
451 return false;
452
453 Target = DispValue;
454 if (Inst.getOpcode() == AArch64::ADRP)
455 Target += Address & ~0xFFFULL;
456 else
457 Target += Address;
458 return true;
459 }
460
461 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
462 MCInst::iterator OI = Inst.begin();
463 if (isADR(Inst) || isADRP(Inst)) {
464 assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&
465 "Unexpected number of operands");
466 return ++OI;
467 }
468 const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
469 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
470 if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)
471 break;
472 ++OI;
473 }
474 assert(OI != Inst.end() && "Literal operand not found");
475 return OI;
476 }
477
478 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
479 MCInst::iterator OI = getMemOperandDisp(Inst);
480 *OI = Operand;
481 return true;
482 }
483
484 void getCalleeSavedRegs(BitVector &Regs) const override {
485 Regs |= getAliases(AArch64::X18);
486 Regs |= getAliases(AArch64::X19);
487 Regs |= getAliases(AArch64::X20);
488 Regs |= getAliases(AArch64::X21);
489 Regs |= getAliases(AArch64::X22);
490 Regs |= getAliases(AArch64::X23);
491 Regs |= getAliases(AArch64::X24);
492 Regs |= getAliases(AArch64::X25);
493 Regs |= getAliases(AArch64::X26);
494 Regs |= getAliases(AArch64::X27);
495 Regs |= getAliases(AArch64::X28);
496 Regs |= getAliases(AArch64::LR);
497 Regs |= getAliases(AArch64::FP);
498 }
499
500 const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr,
501 MCContext &Ctx,
502 uint64_t RelType) const override {
503
504 if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||
505 RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {
506 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_ABS, Ctx);
507 } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||
508 RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||
509 RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||
510 RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 ||
511 RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
512 // Never emit a GOT reloc, we handled this in
513 // RewriteInstance::readRelocations().
514 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_ABS_PAGE, Ctx);
515 } else {
516 switch (RelType) {
517 case ELF::R_AARCH64_ADD_ABS_LO12_NC:
518 case ELF::R_AARCH64_LD64_GOT_LO12_NC:
519 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
520 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
521 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
522 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
523 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
524 case ELF::R_AARCH64_TLSDESC_ADD_LO12:
525 case ELF::R_AARCH64_TLSDESC_LD64_LO12:
526 case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
527 case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
528 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_LO12, Ctx);
529 case ELF::R_AARCH64_MOVW_UABS_G3:
530 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_ABS_G3, Ctx);
531 case ELF::R_AARCH64_MOVW_UABS_G2:
532 case ELF::R_AARCH64_MOVW_UABS_G2_NC:
533 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_ABS_G2_NC, Ctx);
534 case ELF::R_AARCH64_MOVW_UABS_G1:
535 case ELF::R_AARCH64_MOVW_UABS_G1_NC:
536 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_ABS_G1_NC, Ctx);
537 case ELF::R_AARCH64_MOVW_UABS_G0:
538 case ELF::R_AARCH64_MOVW_UABS_G0_NC:
539 return AArch64MCExpr::create(Expr, Kind: AArch64MCExpr::VK_ABS_G0_NC, Ctx);
540 default:
541 break;
542 }
543 }
544 return Expr;
545 }
546
547 bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const {
548 if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
549 return false;
550
551 // Auto-select correct operand number
552 if (OpNum == 0) {
553 if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst) ||
554 isMOVW(Inst))
555 OpNum = 1;
556 if (isTB(Inst) || isAddXri(Inst))
557 OpNum = 2;
558 }
559
560 return true;
561 }
562
563 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
564 auto *AArchExpr = dyn_cast<AArch64MCExpr>(Val: Expr);
565 if (AArchExpr && AArchExpr->getSubExpr())
566 return getTargetSymbol(Expr: AArchExpr->getSubExpr());
567
568 auto *BinExpr = dyn_cast<MCBinaryExpr>(Val: Expr);
569 if (BinExpr)
570 return getTargetSymbol(Expr: BinExpr->getLHS());
571
572 auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Val: Expr);
573 if (SymExpr && SymExpr->getKind() == MCSymbolRefExpr::VK_None)
574 return &SymExpr->getSymbol();
575
576 return nullptr;
577 }
578
579 const MCSymbol *getTargetSymbol(const MCInst &Inst,
580 unsigned OpNum = 0) const override {
581 if (!getSymbolRefOperandNum(Inst, OpNum))
582 return nullptr;
583
584 const MCOperand &Op = Inst.getOperand(i: OpNum);
585 if (!Op.isExpr())
586 return nullptr;
587
588 return getTargetSymbol(Expr: Op.getExpr());
589 }
590
591 int64_t getTargetAddend(const MCExpr *Expr) const override {
592 auto *AArchExpr = dyn_cast<AArch64MCExpr>(Val: Expr);
593 if (AArchExpr && AArchExpr->getSubExpr())
594 return getTargetAddend(Expr: AArchExpr->getSubExpr());
595
596 auto *BinExpr = dyn_cast<MCBinaryExpr>(Val: Expr);
597 if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add)
598 return getTargetAddend(Expr: BinExpr->getRHS());
599
600 auto *ConstExpr = dyn_cast<MCConstantExpr>(Val: Expr);
601 if (ConstExpr)
602 return ConstExpr->getValue();
603
604 return 0;
605 }
606
607 int64_t getTargetAddend(const MCInst &Inst,
608 unsigned OpNum = 0) const override {
609 if (!getSymbolRefOperandNum(Inst, OpNum))
610 return 0;
611
612 const MCOperand &Op = Inst.getOperand(i: OpNum);
613 if (!Op.isExpr())
614 return 0;
615
616 return getTargetAddend(Expr: Op.getExpr());
617 }
618
619 bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
620 MCContext *Ctx) const override {
621 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
622 "Invalid instruction");
623 assert(MCPlus::getNumPrimeOperands(Inst) >= 1 &&
624 "Invalid number of operands");
625 MCInst::iterator OI = Inst.begin();
626
627 if (isConditionalBranch(Inst)) {
628 assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&
629 "Invalid number of operands");
630 ++OI;
631 }
632
633 if (isTB(Inst)) {
634 assert(MCPlus::getNumPrimeOperands(Inst) >= 3 &&
635 "Invalid number of operands");
636 OI = Inst.begin() + 2;
637 }
638
639 *OI = MCOperand::createExpr(
640 Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
641 return true;
642 }
643
644 /// Matches indirect branch patterns in AArch64 related to a jump table (JT),
645 /// helping us to build the complete CFG. A typical indirect branch to
646 /// a jump table entry in AArch64 looks like the following:
647 ///
648 /// adrp x1, #-7585792 # Get JT Page location
649 /// add x1, x1, #692 # Complement with JT Page offset
650 /// ldrh w0, [x1, w0, uxtw #1] # Loads JT entry
651 /// adr x1, #12 # Get PC + 12 (end of this BB) used next
652 /// add x0, x1, w0, sxth #2 # Finish building branch target
653 /// # (entries in JT are relative to the end
654 /// # of this BB)
655 /// br x0 # Indirect jump instruction
656 ///
657 bool analyzeIndirectBranchFragment(
658 const MCInst &Inst,
659 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain,
660 const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue,
661 MCInst *&PCRelBase) const {
662 // Expect AArch64 BR
663 assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");
664
665 // Match the indirect branch pattern for aarch64
666 SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];
667 if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr)
668 return false;
669
670 const MCInst *DefAdd = UsesRoot[0];
671
672 // Now we match an ADD
673 if (!isADD(Inst: *DefAdd)) {
674 // If the address is not broken up in two parts, this is not branching
675 // according to a jump table entry. Fail.
676 return false;
677 }
678 if (DefAdd->getOpcode() == AArch64::ADDXri) {
679 // This can happen when there is no offset, but a direct jump that was
680 // transformed into an indirect one (indirect tail call) :
681 // ADRP x2, Perl_re_compiler
682 // ADD x2, x2, :lo12:Perl_re_compiler
683 // BR x2
684 return false;
685 }
686 if (DefAdd->getOpcode() == AArch64::ADDXrs) {
687 // Covers the less common pattern where JT entries are relative to
688 // the JT itself (like x86). Seems less efficient since we can't
689 // assume the JT is aligned at 4B boundary and thus drop 2 bits from
690 // JT values.
691 // cde264:
692 // adrp x12, #21544960 ; 216a000
693 // add x12, x12, #1696 ; 216a6a0 (JT object in .rodata)
694 // ldrsw x8, [x12, x8, lsl #2] --> loads e.g. 0xfeb73bd8
695 // * add x8, x8, x12 --> = cde278, next block
696 // br x8
697 // cde278:
698 //
699 // Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:0
700 return false;
701 }
702 assert(DefAdd->getOpcode() == AArch64::ADDXrx &&
703 "Failed to match indirect branch!");
704
705 // Validate ADD operands
706 int64_t OperandExtension = DefAdd->getOperand(i: 3).getImm();
707 unsigned ShiftVal = AArch64_AM::getArithShiftValue(Imm: OperandExtension);
708 AArch64_AM::ShiftExtendType ExtendType =
709 AArch64_AM::getArithExtendType(Imm: OperandExtension);
710 if (ShiftVal != 2)
711 llvm_unreachable("Failed to match indirect branch! (fragment 2)");
712
713 if (ExtendType == AArch64_AM::SXTB)
714 ScaleValue = 1LL;
715 else if (ExtendType == AArch64_AM::SXTH)
716 ScaleValue = 2LL;
717 else if (ExtendType == AArch64_AM::SXTW)
718 ScaleValue = 4LL;
719 else
720 llvm_unreachable("Failed to match indirect branch! (fragment 3)");
721
722 // Match an ADR to load base address to be used when addressing JT targets
723 SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd];
724 if (UsesAdd.size() <= 1 || UsesAdd[1] == nullptr || UsesAdd[2] == nullptr) {
725 // This happens when we don't have enough context about this jump table
726 // because the jumping code sequence was split in multiple basic blocks.
727 // This was observed in the wild in HHVM code (dispatchImpl).
728 return false;
729 }
730 MCInst *DefBaseAddr = UsesAdd[1];
731 assert(DefBaseAddr->getOpcode() == AArch64::ADR &&
732 "Failed to match indirect branch pattern! (fragment 3)");
733
734 PCRelBase = DefBaseAddr;
735 // Match LOAD to load the jump table (relative) target
736 const MCInst *DefLoad = UsesAdd[2];
737 assert(mayLoad(*DefLoad) &&
738 "Failed to match indirect branch load pattern! (1)");
739 assert((ScaleValue != 1LL || isLDRB(*DefLoad)) &&
740 "Failed to match indirect branch load pattern! (2)");
741 assert((ScaleValue != 2LL || isLDRH(*DefLoad)) &&
742 "Failed to match indirect branch load pattern! (3)");
743
744 // Match ADD that calculates the JumpTable Base Address (not the offset)
745 SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];
746 const MCInst *DefJTBaseAdd = UsesLoad[1];
747 MCPhysReg From, To;
748 if (DefJTBaseAdd == nullptr || isLoadFromStack(Inst: *DefJTBaseAdd) ||
749 isRegToRegMove(Inst: *DefJTBaseAdd, From, To)) {
750 // Sometimes base address may have been defined in another basic block
751 // (hoisted). Return with no jump table info.
752 JumpTable = nullptr;
753 return true;
754 }
755
756 assert(DefJTBaseAdd->getOpcode() == AArch64::ADDXri &&
757 "Failed to match jump table base address pattern! (1)");
758
759 if (DefJTBaseAdd->getOperand(i: 2).isImm())
760 Offset = DefJTBaseAdd->getOperand(i: 2).getImm();
761 SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
762 const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
763 if (DefJTBasePage == nullptr || isLoadFromStack(Inst: *DefJTBasePage)) {
764 JumpTable = nullptr;
765 return true;
766 }
767 assert(DefJTBasePage->getOpcode() == AArch64::ADRP &&
768 "Failed to match jump table base page pattern! (2)");
769 if (DefJTBasePage->getOperand(i: 1).isExpr())
770 JumpTable = DefJTBasePage->getOperand(i: 1).getExpr();
771 return true;
772 }
773
774 DenseMap<const MCInst *, SmallVector<MCInst *, 4>>
775 computeLocalUDChain(const MCInst *CurInstr, InstructionIterator Begin,
776 InstructionIterator End) const {
777 DenseMap<int, MCInst *> RegAliasTable;
778 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> Uses;
779
780 auto addInstrOperands = [&](const MCInst &Instr) {
781 // Update Uses table
782 for (const MCOperand &Operand : MCPlus::primeOperands(Inst: Instr)) {
783 if (!Operand.isReg())
784 continue;
785 unsigned Reg = Operand.getReg();
786 MCInst *AliasInst = RegAliasTable[Reg];
787 Uses[&Instr].push_back(Elt: AliasInst);
788 LLVM_DEBUG({
789 dbgs() << "Adding reg operand " << Reg << " refs ";
790 if (AliasInst != nullptr)
791 AliasInst->dump();
792 else
793 dbgs() << "\n";
794 });
795 }
796 };
797
798 LLVM_DEBUG(dbgs() << "computeLocalUDChain\n");
799 bool TerminatorSeen = false;
800 for (auto II = Begin; II != End; ++II) {
801 MCInst &Instr = *II;
802 // Ignore nops and CFIs
803 if (isPseudo(Inst: Instr) || isNoop(Inst: Instr))
804 continue;
805 if (TerminatorSeen) {
806 RegAliasTable.clear();
807 Uses.clear();
808 }
809
810 LLVM_DEBUG(dbgs() << "Now updating for:\n ");
811 LLVM_DEBUG(Instr.dump());
812 addInstrOperands(Instr);
813
814 BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
815 getWrittenRegs(Inst: Instr, Regs);
816
817 // Update register definitions after this point
818 for (int Idx : Regs.set_bits()) {
819 RegAliasTable[Idx] = &Instr;
820 LLVM_DEBUG(dbgs() << "Setting reg " << Idx
821 << " def to current instr.\n");
822 }
823
824 TerminatorSeen = isTerminator(Inst: Instr);
825 }
826
827 // Process the last instruction, which is not currently added into the
828 // instruction stream
829 if (CurInstr)
830 addInstrOperands(*CurInstr);
831
832 return Uses;
833 }
834
835 IndirectBranchType analyzeIndirectBranch(
836 MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
837 const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
838 unsigned &IndexRegNumOut, int64_t &DispValueOut,
839 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
840 MemLocInstrOut = nullptr;
841 BaseRegNumOut = AArch64::NoRegister;
842 IndexRegNumOut = AArch64::NoRegister;
843 DispValueOut = 0;
844 DispExprOut = nullptr;
845
846 // An instruction referencing memory used by jump instruction (directly or
847 // via register). This location could be an array of function pointers
848 // in case of indirect tail call, or a jump table.
849 MCInst *MemLocInstr = nullptr;
850
851 // Analyze the memory location.
852 int64_t ScaleValue, DispValue;
853 const MCExpr *DispExpr;
854
855 DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
856 computeLocalUDChain(CurInstr: &Instruction, Begin, End);
857 MCInst *PCRelBase;
858 if (!analyzeIndirectBranchFragment(Inst: Instruction, UDChain, JumpTable&: DispExpr,
859 Offset&: DispValue, ScaleValue, PCRelBase))
860 return IndirectBranchType::UNKNOWN;
861
862 MemLocInstrOut = MemLocInstr;
863 DispValueOut = DispValue;
864 DispExprOut = DispExpr;
865 PCRelBaseOut = PCRelBase;
866 return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE;
867 }
868
869 /// Matches PLT entry pattern and returns the associated GOT entry address.
870 /// Typical PLT entry looks like the following:
871 ///
872 /// adrp x16, 230000
873 /// ldr x17, [x16, #3040]
874 /// add x16, x16, #0xbe0
875 /// br x17
876 ///
877 /// The other type of trampolines are located in .plt.got, that are used for
878 /// non-lazy bindings so doesn't use x16 arg to transfer .got entry address:
879 ///
880 /// adrp x16, 230000
881 /// ldr x17, [x16, #3040]
882 /// br x17
883 /// nop
884 ///
885 uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin,
886 InstructionIterator End,
887 uint64_t BeginPC) const override {
888 // Check branch instruction
889 MCInst *Branch = &Instruction;
890 assert(Branch->getOpcode() == AArch64::BR && "Unexpected opcode");
891
892 DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
893 computeLocalUDChain(CurInstr: Branch, Begin, End);
894
895 // Match ldr instruction
896 SmallVector<MCInst *, 4> &BranchUses = UDChain[Branch];
897 if (BranchUses.size() < 1 || BranchUses[0] == nullptr)
898 return 0;
899
900 // Check ldr instruction
901 const MCInst *Ldr = BranchUses[0];
902 if (Ldr->getOpcode() != AArch64::LDRXui)
903 return 0;
904
905 // Get ldr value
906 const unsigned ScaleLdr = 8; // LDRX operates on 8 bytes segments
907 assert(Ldr->getOperand(2).isImm() && "Unexpected ldr operand");
908 const uint64_t Offset = Ldr->getOperand(i: 2).getImm() * ScaleLdr;
909
910 // Match adrp instruction
911 SmallVector<MCInst *, 4> &LdrUses = UDChain[Ldr];
912 if (LdrUses.size() < 2 || LdrUses[1] == nullptr)
913 return 0;
914
915 // Check adrp instruction
916 MCInst *Adrp = LdrUses[1];
917 if (Adrp->getOpcode() != AArch64::ADRP)
918 return 0;
919
920 // Get adrp instruction PC
921 const unsigned InstSize = 4;
922 uint64_t AdrpPC = BeginPC;
923 for (InstructionIterator It = Begin; It != End; ++It) {
924 if (&(*It) == Adrp)
925 break;
926 AdrpPC += InstSize;
927 }
928
929 // Get adrp value
930 uint64_t Base;
931 assert(Adrp->getOperand(1).isImm() && "Unexpected adrp operand");
932 bool Ret = evaluateMemOperandTarget(Inst: *Adrp, Target&: Base, Address: AdrpPC, Size: InstSize);
933 assert(Ret && "Failed to evaluate adrp");
934 (void)Ret;
935
936 return Base + Offset;
937 }
938
939 unsigned getInvertedBranchOpcode(unsigned Opcode) const {
940 switch (Opcode) {
941 default:
942 llvm_unreachable("Failed to invert branch opcode");
943 return Opcode;
944 case AArch64::TBZW: return AArch64::TBNZW;
945 case AArch64::TBZX: return AArch64::TBNZX;
946 case AArch64::TBNZW: return AArch64::TBZW;
947 case AArch64::TBNZX: return AArch64::TBZX;
948 case AArch64::CBZW: return AArch64::CBNZW;
949 case AArch64::CBZX: return AArch64::CBNZX;
950 case AArch64::CBNZW: return AArch64::CBZW;
951 case AArch64::CBNZX: return AArch64::CBZX;
952 }
953 }
954
955 unsigned getCondCode(const MCInst &Inst) const override {
956 // AArch64 does not use conditional codes, so we just return the opcode
957 // of the conditional branch here.
958 return Inst.getOpcode();
959 }
960
961 unsigned getCanonicalBranchCondCode(unsigned Opcode) const override {
962 switch (Opcode) {
963 default:
964 return Opcode;
965 case AArch64::TBNZW: return AArch64::TBZW;
966 case AArch64::TBNZX: return AArch64::TBZX;
967 case AArch64::CBNZW: return AArch64::CBZW;
968 case AArch64::CBNZX: return AArch64::CBZX;
969 }
970 }
971
972 bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
973 MCContext *Ctx) const override {
974 if (isTB(Inst) || isCB(Inst)) {
975 Inst.setOpcode(getInvertedBranchOpcode(Opcode: Inst.getOpcode()));
976 assert(Inst.getOpcode() != 0 && "Invalid branch instruction");
977 } else if (Inst.getOpcode() == AArch64::Bcc) {
978 Inst.getOperand(i: 0).setImm(AArch64CC::getInvertedCondCode(
979 Code: static_cast<AArch64CC::CondCode>(Inst.getOperand(i: 0).getImm())));
980 assert(Inst.getOperand(0).getImm() != AArch64CC::AL &&
981 Inst.getOperand(0).getImm() != AArch64CC::NV &&
982 "Can't reverse ALWAYS cond code");
983 } else {
984 LLVM_DEBUG(Inst.dump());
985 llvm_unreachable("Unrecognized branch instruction");
986 }
987 return replaceBranchTarget(Inst, TBB, Ctx);
988 }
989
990 int getPCRelEncodingSize(const MCInst &Inst) const override {
991 switch (Inst.getOpcode()) {
992 default:
993 llvm_unreachable("Failed to get pcrel encoding size");
994 return 0;
995 case AArch64::TBZW: return 16;
996 case AArch64::TBZX: return 16;
997 case AArch64::TBNZW: return 16;
998 case AArch64::TBNZX: return 16;
999 case AArch64::CBZW: return 21;
1000 case AArch64::CBZX: return 21;
1001 case AArch64::CBNZW: return 21;
1002 case AArch64::CBNZX: return 21;
1003 case AArch64::B: return 28;
1004 case AArch64::BL: return 28;
1005 case AArch64::Bcc: return 21;
1006 }
1007 }
1008
1009 int getShortJmpEncodingSize() const override { return 33; }
1010
1011 int getUncondBranchEncodingSize() const override { return 28; }
1012
1013 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
1014 const MCSymbol *Target,
1015 MCContext *Ctx) const override {
1016 InstructionListType Code;
1017 Code.emplace_back(MCInstBuilder(AArch64::SUBSXri)
1018 .addReg(RegNo)
1019 .addReg(RegNo)
1020 .addImm(Imm)
1021 .addImm(0));
1022 Code.emplace_back(MCInstBuilder(AArch64::Bcc)
1023 .addImm(Imm)
1024 .addExpr(MCSymbolRefExpr::create(
1025 Target, MCSymbolRefExpr::VK_None, *Ctx)));
1026 return Code;
1027 }
1028
1029 void createTailCall(MCInst &Inst, const MCSymbol *Target,
1030 MCContext *Ctx) override {
1031 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
1032 }
1033
1034 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
1035 MCContext *Ctx) override {
1036 createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true);
1037 }
1038
1039 void createTrap(MCInst &Inst) const override {
1040 Inst.clear();
1041 Inst.setOpcode(AArch64::BRK);
1042 Inst.addOperand(Op: MCOperand::createImm(Val: 1));
1043 }
1044
1045 bool convertJmpToTailCall(MCInst &Inst) override {
1046 setTailCall(Inst);
1047 return true;
1048 }
1049
1050 bool convertTailCallToJmp(MCInst &Inst) override {
1051 removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1052 clearOffset(Inst);
1053 if (getConditionalTailCall(Inst))
1054 unsetConditionalTailCall(Inst);
1055 return true;
1056 }
1057
1058 bool lowerTailCall(MCInst &Inst) override {
1059 removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1060 if (getConditionalTailCall(Inst))
1061 unsetConditionalTailCall(Inst);
1062 return true;
1063 }
1064
1065 bool isNoop(const MCInst &Inst) const override {
1066 return Inst.getOpcode() == AArch64::HINT &&
1067 Inst.getOperand(0).getImm() == 0;
1068 }
1069
1070 void createNoop(MCInst &Inst) const override {
1071 Inst.setOpcode(AArch64::HINT);
1072 Inst.clear();
1073 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
1074 }
1075
1076 bool mayStore(const MCInst &Inst) const override { return false; }
1077
1078 void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
1079 bool IsTailCall) override {
1080 Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL);
1081 Inst.clear();
1082 Inst.addOperand(Op: MCOperand::createExpr(Val: getTargetExprFor(
1083 Inst, Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1084 Ctx&: *Ctx, RelType: 0)));
1085 if (IsTailCall)
1086 convertJmpToTailCall(Inst);
1087 }
1088
1089 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1090 const MCSymbol *&TBB, const MCSymbol *&FBB,
1091 MCInst *&CondBranch,
1092 MCInst *&UncondBranch) const override {
1093 auto I = End;
1094
1095 while (I != Begin) {
1096 --I;
1097
1098 // Ignore nops and CFIs
1099 if (isPseudo(Inst: *I) || isNoop(Inst: *I))
1100 continue;
1101
1102 // Stop when we find the first non-terminator
1103 if (!isTerminator(Inst: *I) || isTailCall(Inst: *I) || !isBranch(Inst: *I))
1104 break;
1105
1106 // Handle unconditional branches.
1107 if (isUnconditionalBranch(Inst: *I)) {
1108 // If any code was seen after this unconditional branch, we've seen
1109 // unreachable code. Ignore them.
1110 CondBranch = nullptr;
1111 UncondBranch = &*I;
1112 const MCSymbol *Sym = getTargetSymbol(Inst: *I);
1113 assert(Sym != nullptr &&
1114 "Couldn't extract BB symbol from jump operand");
1115 TBB = Sym;
1116 continue;
1117 }
1118
1119 // Handle conditional branches and ignore indirect branches
1120 if (isIndirectBranch(Inst: *I))
1121 return false;
1122
1123 if (CondBranch == nullptr) {
1124 const MCSymbol *TargetBB = getTargetSymbol(Inst: *I);
1125 if (TargetBB == nullptr) {
1126 // Unrecognized branch target
1127 return false;
1128 }
1129 FBB = TBB;
1130 TBB = TargetBB;
1131 CondBranch = &*I;
1132 continue;
1133 }
1134
1135 llvm_unreachable("multiple conditional branches in one BB");
1136 }
1137 return true;
1138 }
1139
1140 void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,
1141 MCContext *Ctx, bool IsTailCall) override {
1142 // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
1143 // Standard for the ARM 64-bit Architecture (AArch64)".
1144 // The sequence of instructions we create here is the following:
1145 // movz ip0, #:abs_g3:<addr>
1146 // movk ip0, #:abs_g2_nc:<addr>
1147 // movk ip0, #:abs_g1_nc:<addr>
1148 // movk ip0, #:abs_g0_nc:<addr>
1149 // br ip0
1150 MCInst Inst;
1151 Inst.setOpcode(AArch64::MOVZXi);
1152 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1153 Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
1154 Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1155 Kind: AArch64MCExpr::VK_ABS_G3, Ctx&: *Ctx)));
1156 Inst.addOperand(Op: MCOperand::createImm(Val: 0x30));
1157 Seq.emplace_back(args&: Inst);
1158
1159 Inst.clear();
1160 Inst.setOpcode(AArch64::MOVKXi);
1161 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1162 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1163 Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
1164 Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1165 Kind: AArch64MCExpr::VK_ABS_G2_NC, Ctx&: *Ctx)));
1166 Inst.addOperand(Op: MCOperand::createImm(Val: 0x20));
1167 Seq.emplace_back(args&: Inst);
1168
1169 Inst.clear();
1170 Inst.setOpcode(AArch64::MOVKXi);
1171 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1172 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1173 Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
1174 Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1175 Kind: AArch64MCExpr::VK_ABS_G1_NC, Ctx&: *Ctx)));
1176 Inst.addOperand(Op: MCOperand::createImm(Val: 0x10));
1177 Seq.emplace_back(args&: Inst);
1178
1179 Inst.clear();
1180 Inst.setOpcode(AArch64::MOVKXi);
1181 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1182 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1183 Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
1184 Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1185 Kind: AArch64MCExpr::VK_ABS_G0_NC, Ctx&: *Ctx)));
1186 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
1187 Seq.emplace_back(args&: Inst);
1188
1189 Inst.clear();
1190 Inst.setOpcode(AArch64::BR);
1191 Inst.addOperand(MCOperand::createReg(AArch64::X16));
1192 if (IsTailCall)
1193 setTailCall(Inst);
1194 Seq.emplace_back(args&: Inst);
1195 }
1196
1197 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
1198 MCContext *Ctx, bool IsTailCall) override {
1199 // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
1200 // Standard for the ARM 64-bit Architecture (AArch64)".
1201 // The sequence of instructions we create here is the following:
1202 // adrp ip0, imm
1203 // add ip0, ip0, imm
1204 // br ip0
1205 MCPhysReg Reg = AArch64::X16;
1206 InstructionListType Insts = materializeAddress(Target, Ctx, RegName: Reg);
1207 Insts.emplace_back();
1208 MCInst &Inst = Insts.back();
1209 Inst.clear();
1210 Inst.setOpcode(AArch64::BR);
1211 Inst.addOperand(Op: MCOperand::createReg(Reg));
1212 if (IsTailCall)
1213 setTailCall(Inst);
1214 Seq.swap(x&: Insts);
1215 }
1216
1217 /// Matching pattern here is
1218 ///
1219 /// ADRP x16, imm
1220 /// ADD x16, x16, imm
1221 /// BR x16
1222 ///
1223 uint64_t matchLinkerVeneer(InstructionIterator Begin, InstructionIterator End,
1224 uint64_t Address, const MCInst &CurInst,
1225 MCInst *&TargetHiBits, MCInst *&TargetLowBits,
1226 uint64_t &Target) const override {
1227 if (CurInst.getOpcode() != AArch64::BR || !CurInst.getOperand(0).isReg() ||
1228 CurInst.getOperand(0).getReg() != AArch64::X16)
1229 return 0;
1230
1231 auto I = End;
1232 if (I == Begin)
1233 return 0;
1234
1235 --I;
1236 Address -= 4;
1237 if (I == Begin || I->getOpcode() != AArch64::ADDXri ||
1238 MCPlus::getNumPrimeOperands(*I) < 3 || !I->getOperand(0).isReg() ||
1239 !I->getOperand(1).isReg() ||
1240 I->getOperand(0).getReg() != AArch64::X16 ||
1241 I->getOperand(1).getReg() != AArch64::X16 || !I->getOperand(2).isImm())
1242 return 0;
1243 TargetLowBits = &*I;
1244 uint64_t Addr = I->getOperand(i: 2).getImm() & 0xFFF;
1245
1246 --I;
1247 Address -= 4;
1248 if (I->getOpcode() != AArch64::ADRP ||
1249 MCPlus::getNumPrimeOperands(*I) < 2 || !I->getOperand(0).isReg() ||
1250 !I->getOperand(1).isImm() || I->getOperand(0).getReg() != AArch64::X16)
1251 return 0;
1252 TargetHiBits = &*I;
1253 Addr |= (Address + ((int64_t)I->getOperand(i: 1).getImm() << 12)) &
1254 0xFFFFFFFFFFFFF000ULL;
1255 Target = Addr;
1256 return 3;
1257 }
1258
1259 bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const override {
1260 if (!isADRP(Inst: Adrp) || !isAddXri(Inst: Add))
1261 return false;
1262
1263 assert(Adrp.getOperand(0).isReg() &&
1264 "Unexpected operand in ADRP instruction");
1265 MCPhysReg AdrpReg = Adrp.getOperand(i: 0).getReg();
1266 assert(Add.getOperand(1).isReg() &&
1267 "Unexpected operand in ADDXri instruction");
1268 MCPhysReg AddReg = Add.getOperand(i: 1).getReg();
1269 return AdrpReg == AddReg;
1270 }
1271
1272 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
1273 int64_t Addend, MCContext *Ctx, int64_t &Value,
1274 uint64_t RelType) const override {
1275 unsigned ImmOpNo = -1U;
1276 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1277 ++Index) {
1278 if (Inst.getOperand(i: Index).isImm()) {
1279 ImmOpNo = Index;
1280 break;
1281 }
1282 }
1283 if (ImmOpNo == -1U)
1284 return false;
1285
1286 Value = Inst.getOperand(i: ImmOpNo).getImm();
1287
1288 setOperandToSymbolRef(Inst, OpNum: ImmOpNo, Symbol, Addend, Ctx, RelType);
1289
1290 return true;
1291 }
1292
1293 void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
1294 MCContext *Ctx) const override {
1295 Inst.setOpcode(AArch64::B);
1296 Inst.clear();
1297 Inst.addOperand(Op: MCOperand::createExpr(Val: getTargetExprFor(
1298 Inst, Expr: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1299 Ctx&: *Ctx, RelType: 0)));
1300 }
1301
1302 bool shouldRecordCodeRelocation(uint64_t RelType) const override {
1303 switch (RelType) {
1304 case ELF::R_AARCH64_ABS64:
1305 case ELF::R_AARCH64_ABS32:
1306 case ELF::R_AARCH64_ABS16:
1307 case ELF::R_AARCH64_ADD_ABS_LO12_NC:
1308 case ELF::R_AARCH64_ADR_GOT_PAGE:
1309 case ELF::R_AARCH64_ADR_PREL_LO21:
1310 case ELF::R_AARCH64_ADR_PREL_PG_HI21:
1311 case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
1312 case ELF::R_AARCH64_LD64_GOT_LO12_NC:
1313 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
1314 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
1315 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
1316 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
1317 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
1318 case ELF::R_AARCH64_TLSDESC_ADD_LO12:
1319 case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
1320 case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
1321 case ELF::R_AARCH64_TLSDESC_LD64_LO12:
1322 case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
1323 case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
1324 case ELF::R_AARCH64_MOVW_UABS_G0:
1325 case ELF::R_AARCH64_MOVW_UABS_G0_NC:
1326 case ELF::R_AARCH64_MOVW_UABS_G1:
1327 case ELF::R_AARCH64_MOVW_UABS_G1_NC:
1328 case ELF::R_AARCH64_MOVW_UABS_G2:
1329 case ELF::R_AARCH64_MOVW_UABS_G2_NC:
1330 case ELF::R_AARCH64_MOVW_UABS_G3:
1331 case ELF::R_AARCH64_PREL16:
1332 case ELF::R_AARCH64_PREL32:
1333 case ELF::R_AARCH64_PREL64:
1334 return true;
1335 case ELF::R_AARCH64_CALL26:
1336 case ELF::R_AARCH64_JUMP26:
1337 case ELF::R_AARCH64_TSTBR14:
1338 case ELF::R_AARCH64_CONDBR19:
1339 case ELF::R_AARCH64_TLSDESC_CALL:
1340 case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
1341 case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
1342 return false;
1343 default:
1344 llvm_unreachable("Unexpected AArch64 relocation type in code");
1345 }
1346 }
1347
1348 StringRef getTrapFillValue() const override {
1349 return StringRef("\0\0\0\0", 4);
1350 }
1351
1352 void createReturn(MCInst &Inst) const override {
1353 Inst.setOpcode(AArch64::RET);
1354 Inst.clear();
1355 Inst.addOperand(MCOperand::createReg(AArch64::LR));
1356 }
1357
1358 void createStackPointerIncrement(
1359 MCInst &Inst, int Size,
1360 bool NoFlagsClobber = false /*unused for AArch64*/) const override {
1361 Inst.setOpcode(AArch64::SUBXri);
1362 Inst.clear();
1363 Inst.addOperand(MCOperand::createReg(AArch64::SP));
1364 Inst.addOperand(MCOperand::createReg(AArch64::SP));
1365 Inst.addOperand(Op: MCOperand::createImm(Val: Size));
1366 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
1367 }
1368
1369 void createStackPointerDecrement(
1370 MCInst &Inst, int Size,
1371 bool NoFlagsClobber = false /*unused for AArch64*/) const override {
1372 Inst.setOpcode(AArch64::ADDXri);
1373 Inst.clear();
1374 Inst.addOperand(MCOperand::createReg(AArch64::SP));
1375 Inst.addOperand(MCOperand::createReg(AArch64::SP));
1376 Inst.addOperand(Op: MCOperand::createImm(Val: Size));
1377 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
1378 }
1379
1380 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
1381 int64_t Disp) const {
1382 Inst.setOpcode(AArch64::BR);
1383 Inst.clear();
1384 Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg));
1385 }
1386
1387 InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
1388 InstructionListType Insts(5);
1389 // Code sequence for instrumented indirect call handler:
1390 // msr nzcv, x1
1391 // ldp x0, x1, [sp], #16
1392 // ldr x16, [sp], #16
1393 // ldp x0, x1, [sp], #16
1394 // br x16
1395 setSystemFlag(Insts[0], AArch64::X1);
1396 createPopRegisters(Insts[1], AArch64::X0, AArch64::X1);
1397 // Here we load address of the next function which should be called in the
1398 // original binary to X16 register. Writing to X16 is permitted without
1399 // needing to restore.
1400 loadReg(Insts[2], AArch64::X16, AArch64::SP);
1401 createPopRegisters(Insts[3], AArch64::X0, AArch64::X1);
1402 createIndirectBranch(Insts[4], AArch64::X16, 0);
1403 return Insts;
1404 }
1405
1406 InstructionListType
1407 createInstrumentedIndTailCallHandlerExitBB() const override {
1408 return createInstrumentedIndCallHandlerExitBB();
1409 }
1410
1411 InstructionListType createGetter(MCContext *Ctx, const char *name) const {
1412 InstructionListType Insts(4);
1413 MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: name);
1414 InstructionListType Addr = materializeAddress(Locs, Ctx, AArch64::X0);
1415 std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.begin());
1416 assert(Addr.size() == 2 && "Invalid Addr size");
1417 loadReg(Insts[2], AArch64::X0, AArch64::X0);
1418 createReturn(Inst&: Insts[3]);
1419 return Insts;
1420 }
1421
1422 InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
1423 return createGetter(Ctx, name: "__bolt_num_counters");
1424 }
1425
1426 InstructionListType
1427 createInstrLocationsGetter(MCContext *Ctx) const override {
1428 return createGetter(Ctx, name: "__bolt_instr_locations");
1429 }
1430
1431 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
1432 return createGetter(Ctx, name: "__bolt_instr_tables");
1433 }
1434
1435 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
1436 return createGetter(Ctx, name: "__bolt_instr_num_funcs");
1437 }
1438
1439 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1440 bool IsTailCall = isTailCall(Inst);
1441 if (IsTailCall)
1442 removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1443 if (Inst.getOpcode() == AArch64::BR || Inst.getOpcode() == AArch64::BLR) {
1444 Inst.setOpcode(AArch64::ORRXrs);
1445 Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1446 Inst.insert(Inst.begin() + 1, MCOperand::createReg(AArch64::XZR));
1447 Inst.insert(I: Inst.begin() + 3, Op: MCOperand::createImm(Val: 0));
1448 return;
1449 }
1450 llvm_unreachable("not implemented");
1451 }
1452
1453 InstructionListType createLoadImmediate(const MCPhysReg Dest,
1454 uint64_t Imm) const override {
1455 InstructionListType Insts(4);
1456 int Shift = 48;
1457 for (int I = 0; I < 4; I++, Shift -= 16) {
1458 Insts[I].setOpcode(AArch64::MOVKXi);
1459 Insts[I].addOperand(Op: MCOperand::createReg(Reg: Dest));
1460 Insts[I].addOperand(Op: MCOperand::createReg(Reg: Dest));
1461 Insts[I].addOperand(Op: MCOperand::createImm(Val: (Imm >> Shift) & 0xFFFF));
1462 Insts[I].addOperand(Op: MCOperand::createImm(Val: Shift));
1463 }
1464 return Insts;
1465 }
1466
1467 void createIndirectCallInst(MCInst &Inst, bool IsTailCall,
1468 MCPhysReg Reg) const {
1469 Inst.clear();
1470 Inst.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);
1471 Inst.addOperand(Op: MCOperand::createReg(Reg));
1472 }
1473
1474 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
1475 MCSymbol *HandlerFuncAddr,
1476 int CallSiteID,
1477 MCContext *Ctx) override {
1478 InstructionListType Insts;
1479 // Code sequence used to enter indirect call instrumentation helper:
1480 // stp x0, x1, [sp, #-16]! createPushRegisters
1481 // mov target x0 convertIndirectCallToLoad -> orr x0 target xzr
1482 // mov x1 CallSiteID createLoadImmediate ->
1483 // movk x1, #0x0, lsl #48
1484 // movk x1, #0x0, lsl #32
1485 // movk x1, #0x0, lsl #16
1486 // movk x1, #0x0
1487 // stp x0, x1, [sp, #-16]!
1488 // bl *HandlerFuncAddr createIndirectCall ->
1489 // adr x0 *HandlerFuncAddr -> adrp + add
1490 // blr x0
1491 Insts.emplace_back();
1492 createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1);
1493 Insts.emplace_back(args&: CallInst);
1494 convertIndirectCallToLoad(Insts.back(), AArch64::X0);
1495 InstructionListType LoadImm =
1496 createLoadImmediate(Dest: getIntArgRegister(ArgNo: 1), Imm: CallSiteID);
1497 Insts.insert(position: Insts.end(), first: LoadImm.begin(), last: LoadImm.end());
1498 Insts.emplace_back();
1499 createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1);
1500 Insts.resize(new_size: Insts.size() + 2);
1501 InstructionListType Addr =
1502 materializeAddress(HandlerFuncAddr, Ctx, AArch64::X0);
1503 assert(Addr.size() == 2 && "Invalid Addr size");
1504 std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.end() - Addr.size());
1505 Insts.emplace_back();
1506 createIndirectCallInst(Insts.back(), isTailCall(CallInst), AArch64::X0);
1507
1508 // Carry over metadata including tail call marker if present.
1509 stripAnnotations(Inst&: Insts.back());
1510 moveAnnotations(SrcInst: std::move(CallInst), DstInst&: Insts.back());
1511
1512 return Insts;
1513 }
1514
1515 InstructionListType
1516 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
1517 const MCSymbol *IndCallHandler,
1518 MCContext *Ctx) override {
1519 // Code sequence used to check whether InstrTampoline was initialized
1520 // and call it if so, returns via IndCallHandler
1521 // stp x0, x1, [sp, #-16]!
1522 // mrs x1, nzcv
1523 // adr x0, InstrTrampoline -> adrp + add
1524 // ldr x0, [x0]
1525 // subs x0, x0, #0x0
1526 // b.eq IndCallHandler
1527 // str x30, [sp, #-16]!
1528 // blr x0
1529 // ldr x30, [sp], #16
1530 // b IndCallHandler
1531 InstructionListType Insts;
1532 Insts.emplace_back();
1533 createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1);
1534 Insts.emplace_back();
1535 getSystemFlag(Inst&: Insts.back(), RegName: getIntArgRegister(ArgNo: 1));
1536 Insts.emplace_back();
1537 Insts.emplace_back();
1538 InstructionListType Addr =
1539 materializeAddress(InstrTrampoline, Ctx, AArch64::X0);
1540 std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.end() - Addr.size());
1541 assert(Addr.size() == 2 && "Invalid Addr size");
1542 Insts.emplace_back();
1543 loadReg(Insts.back(), AArch64::X0, AArch64::X0);
1544 InstructionListType cmpJmp =
1545 createCmpJE(AArch64::X0, 0, IndCallHandler, Ctx);
1546 Insts.insert(position: Insts.end(), first: cmpJmp.begin(), last: cmpJmp.end());
1547 Insts.emplace_back();
1548 storeReg(Insts.back(), AArch64::LR, AArch64::SP);
1549 Insts.emplace_back();
1550 Insts.back().setOpcode(AArch64::BLR);
1551 Insts.back().addOperand(MCOperand::createReg(AArch64::X0));
1552 Insts.emplace_back();
1553 loadReg(Insts.back(), AArch64::LR, AArch64::SP);
1554 Insts.emplace_back();
1555 createDirectCall(Inst&: Insts.back(), Target: IndCallHandler, Ctx, /*IsTailCall*/ true);
1556 return Insts;
1557 }
1558
1559 InstructionListType
1560 createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
1561 unsigned CodePointerSize) const override {
1562 unsigned int I = 0;
1563 InstructionListType Instrs(IsLeaf ? 12 : 10);
1564
1565 if (IsLeaf)
1566 createStackPointerIncrement(Inst&: Instrs[I++], Size: 128);
1567 createPushRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
1568 getSystemFlag(Instrs[I++], AArch64::X1);
1569 InstructionListType Addr = materializeAddress(Target, Ctx, AArch64::X0);
1570 assert(Addr.size() == 2 && "Invalid Addr size");
1571 std::copy(first: Addr.begin(), last: Addr.end(), result: Instrs.begin() + I);
1572 I += Addr.size();
1573 storeReg(Instrs[I++], AArch64::X2, AArch64::SP);
1574 InstructionListType Insts = createIncMemory(AArch64::X0, AArch64::X2);
1575 assert(Insts.size() == 2 && "Invalid Insts size");
1576 std::copy(first: Insts.begin(), last: Insts.end(), result: Instrs.begin() + I);
1577 I += Insts.size();
1578 loadReg(Instrs[I++], AArch64::X2, AArch64::SP);
1579 setSystemFlag(Instrs[I++], AArch64::X1);
1580 createPopRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
1581 if (IsLeaf)
1582 createStackPointerDecrement(Inst&: Instrs[I++], Size: 128);
1583 return Instrs;
1584 }
1585
1586 std::vector<MCInst> createSymbolTrampoline(const MCSymbol *TgtSym,
1587 MCContext *Ctx) override {
1588 std::vector<MCInst> Insts;
1589 createShortJmp(Seq&: Insts, Target: TgtSym, Ctx, /*IsTailCall*/ true);
1590 return Insts;
1591 }
1592
1593 InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx,
1594 MCPhysReg RegName,
1595 int64_t Addend = 0) const override {
1596 // Get page-aligned address and add page offset
1597 InstructionListType Insts(2);
1598 Insts[0].setOpcode(AArch64::ADRP);
1599 Insts[0].clear();
1600 Insts[0].addOperand(Op: MCOperand::createReg(Reg: RegName));
1601 Insts[0].addOperand(Op: MCOperand::createImm(Val: 0));
1602 setOperandToSymbolRef(Inst&: Insts[0], /* OpNum */ 1, Symbol: Target, Addend, Ctx,
1603 RelType: ELF::R_AARCH64_NONE);
1604 Insts[1].setOpcode(AArch64::ADDXri);
1605 Insts[1].clear();
1606 Insts[1].addOperand(Op: MCOperand::createReg(Reg: RegName));
1607 Insts[1].addOperand(Op: MCOperand::createReg(Reg: RegName));
1608 Insts[1].addOperand(Op: MCOperand::createImm(Val: 0));
1609 Insts[1].addOperand(Op: MCOperand::createImm(Val: 0));
1610 setOperandToSymbolRef(Inst&: Insts[1], /* OpNum */ 2, Symbol: Target, Addend, Ctx,
1611 RelType: ELF::R_AARCH64_ADD_ABS_LO12_NC);
1612 return Insts;
1613 }
1614
1615 std::optional<Relocation>
1616 createRelocation(const MCFixup &Fixup,
1617 const MCAsmBackend &MAB) const override {
1618 const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Kind: Fixup.getKind());
1619
1620 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
1621 const uint64_t RelOffset = Fixup.getOffset();
1622
1623 uint64_t RelType;
1624 if (Fixup.getKind() == MCFixupKind(AArch64::fixup_aarch64_pcrel_call26))
1625 RelType = ELF::R_AARCH64_CALL26;
1626 else if (Fixup.getKind() ==
1627 MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26))
1628 RelType = ELF::R_AARCH64_JUMP26;
1629 else if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
1630 switch (FKI.TargetSize) {
1631 default:
1632 return std::nullopt;
1633 case 16:
1634 RelType = ELF::R_AARCH64_PREL16;
1635 break;
1636 case 32:
1637 RelType = ELF::R_AARCH64_PREL32;
1638 break;
1639 case 64:
1640 RelType = ELF::R_AARCH64_PREL64;
1641 break;
1642 }
1643 } else {
1644 switch (FKI.TargetSize) {
1645 default:
1646 return std::nullopt;
1647 case 16:
1648 RelType = ELF::R_AARCH64_ABS16;
1649 break;
1650 case 32:
1651 RelType = ELF::R_AARCH64_ABS32;
1652 break;
1653 case 64:
1654 RelType = ELF::R_AARCH64_ABS64;
1655 break;
1656 }
1657 }
1658
1659 auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
1660
1661 return Relocation({.Offset: RelOffset, .Symbol: RelSymbol, .Type: RelType, .Addend: RelAddend, .Value: 0});
1662 }
1663
1664 uint16_t getMinFunctionAlignment() const override { return 4; }
1665};
1666
1667} // end anonymous namespace
1668
1669namespace llvm {
1670namespace bolt {
1671
1672MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis,
1673 const MCInstrInfo *Info,
1674 const MCRegisterInfo *RegInfo,
1675 const MCSubtargetInfo *STI) {
1676 return new AArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
1677}
1678
1679} // namespace bolt
1680} // namespace llvm
1681

source code of bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp