1	//===- bolt/Target/AArch64/AArch64MCPlusBuilder.cpp -----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file provides AArch64-specific MCPlus builder.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "AArch64InstrInfo.h"
14	#include "AArch64MCSymbolizer.h"
15	#include "MCTargetDesc/AArch64AddressingModes.h"
16	#include "MCTargetDesc/AArch64FixupKinds.h"
17	#include "MCTargetDesc/AArch64MCExpr.h"
18	#include "MCTargetDesc/AArch64MCTargetDesc.h"
19	#include "Utils/AArch64BaseInfo.h"
20	#include "bolt/Core/BinaryBasicBlock.h"
21	#include "bolt/Core/BinaryFunction.h"
22	#include "bolt/Core/MCPlusBuilder.h"
23	#include "llvm/BinaryFormat/ELF.h"
24	#include "llvm/MC/MCContext.h"
25	#include "llvm/MC/MCFixupKindInfo.h"
26	#include "llvm/MC/MCInstBuilder.h"
27	#include "llvm/MC/MCInstrInfo.h"
28	#include "llvm/MC/MCRegister.h"
29	#include "llvm/MC/MCRegisterInfo.h"
30	#include "llvm/Support/DataExtractor.h"
31	#include "llvm/Support/Debug.h"
32	#include "llvm/Support/ErrorHandling.h"
33
34	#define DEBUG_TYPE "mcplus"
35
36	using namespace llvm;
37	using namespace bolt;
38
39	namespace {
40
41	static void getSystemFlag(MCInst &Inst, MCPhysReg RegName) {
42	Inst.setOpcode(AArch64::MRS);
43	Inst.clear();
44	Inst.addOperand(Op: MCOperand::createReg(Reg: RegName));
45	Inst.addOperand(Op: MCOperand::createImm(AArch64SysReg::Val: NZCV));
46	}
47
48	static void setSystemFlag(MCInst &Inst, MCPhysReg RegName) {
49	Inst.setOpcode(AArch64::MSR);
50	Inst.clear();
51	Inst.addOperand(Op: MCOperand::createImm(AArch64SysReg::Val: NZCV));
52	Inst.addOperand(Op: MCOperand::createReg(Reg: RegName));
53	}
54
55	static void createPushRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {
56	Inst.clear();
57	unsigned NewOpcode = AArch64::STPXpre;
58	Inst.setOpcode(NewOpcode);
59	Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
60	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg1));
61	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg2));
62	Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
63	Inst.addOperand(Op: MCOperand::createImm(Val: -2));
64	}
65
66	static void createPopRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {
67	Inst.clear();
68	unsigned NewOpcode = AArch64::LDPXpost;
69	Inst.setOpcode(NewOpcode);
70	Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
71	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg1));
72	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg2));
73	Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: SP));
74	Inst.addOperand(Op: MCOperand::createImm(Val: 2));
75	}
76
77	static void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From) {
78	Inst.setOpcode(AArch64::LDRXui);
79	Inst.clear();
80	if (From == AArch64::SP) {
81	Inst.setOpcode(AArch64::LDRXpost);
82	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
83	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
84	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
85	Inst.addOperand(Op: MCOperand::createImm(Val: 16));
86	} else {
87	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
88	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
89	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
90	}
91	}
92
93	static void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To) {
94	Inst.setOpcode(AArch64::STRXui);
95	Inst.clear();
96	if (To == AArch64::SP) {
97	Inst.setOpcode(AArch64::STRXpre);
98	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
99	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
100	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
101	Inst.addOperand(Op: MCOperand::createImm(Val: -16));
102	} else {
103	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
104	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
105	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
106	}
107	}
108
109	static void atomicAdd(MCInst &Inst, MCPhysReg RegTo, MCPhysReg RegCnt) {
110	// NOTE: Supports only ARM with LSE extension
111	Inst.setOpcode(AArch64::LDADDX);
112	Inst.clear();
113	Inst.addOperand(Op: MCOperand::createReg(AArch64::Reg: XZR));
114	Inst.addOperand(Op: MCOperand::createReg(Reg: RegCnt));
115	Inst.addOperand(Op: MCOperand::createReg(Reg: RegTo));
116	}
117
118	static void createMovz(MCInst &Inst, MCPhysReg Reg, uint64_t Imm) {
119	assert(Imm <= UINT16_MAX && "Invalid Imm size");
120	Inst.clear();
121	Inst.setOpcode(AArch64::MOVZXi);
122	Inst.addOperand(Op: MCOperand::createReg(Reg));
123	Inst.addOperand(Op: MCOperand::createImm(Val: Imm & 0xFFFF));
124	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
125	}
126
127	static InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegTmp) {
128	InstructionListType Insts;
129	Insts.emplace_back();
130	createMovz(Inst&: Insts.back(), Reg: RegTmp, Imm: 1);
131	Insts.emplace_back();
132	atomicAdd(Inst&: Insts.back(), RegTo, RegCnt: RegTmp);
133	return Insts;
134	}
135	class AArch64MCPlusBuilder : public MCPlusBuilder {
136	public:
137	using MCPlusBuilder::MCPlusBuilder;
138
139	std::unique_ptr<MCSymbolizer>
140	createTargetSymbolizer(BinaryFunction &Function,
141	bool CreateNewSymbols) const override {
142	return std::make_unique<AArch64MCSymbolizer>(args&: Function, args&: CreateNewSymbols);
143	}
144
145	MCPhysReg getStackPointer() const override { return AArch64::SP; }
146	MCPhysReg getFramePointer() const override { return AArch64::FP; }
147
148	bool isPush(const MCInst &Inst) const override {
149	return isStoreToStack(Inst);
150	};
151
152	bool isPop(const MCInst &Inst) const override {
153	return isLoadFromStack(Inst);
154	};
155
156	void createCall(MCInst &Inst, const MCSymbol *Target,
157	MCContext *Ctx) override {
158	createDirectCall(Inst, Target, Ctx, IsTailCall: false);
159	}
160
161	bool convertTailCallToCall(MCInst &Inst) override {
162	int NewOpcode;
163	switch (Inst.getOpcode()) {
164	default:
165	return false;
166	case AArch64::B:
167	NewOpcode = AArch64::BL;
168	break;
169	case AArch64::BR:
170	NewOpcode = AArch64::BLR;
171	break;
172	}
173
174	Inst.setOpcode(NewOpcode);
175	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
176	clearOffset(Inst);
177	return true;
178	}
179
180	bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
181	CompFuncTy Comp) const override {
182	const auto &AArch64ExprA = cast<AArch64MCExpr>(Val: A);
183	const auto &AArch64ExprB = cast<AArch64MCExpr>(Val: B);
184	if (AArch64ExprA.getKind() != AArch64ExprB.getKind())
185	return false;
186
187	return MCPlusBuilder::equals(A: *AArch64ExprA.getSubExpr(),
188	B: *AArch64ExprB.getSubExpr(), Comp);
189	}
190
191	bool shortenInstruction(MCInst &, const MCSubtargetInfo &) const override {
192	return false;
193	}
194
195	SmallVector<MCPhysReg> getTrustedLiveInRegs() const override {
196	return {AArch64::LR};
197	}
198
199	std::optional<MCPhysReg>
200	getWrittenAuthenticatedReg(const MCInst &Inst,
201	bool &IsChecked) const override {
202	IsChecked = false;
203	switch (Inst.getOpcode()) {
204	case AArch64::AUTIAZ:
205	case AArch64::AUTIBZ:
206	case AArch64::AUTIASP:
207	case AArch64::AUTIBSP:
208	case AArch64::AUTIASPPCi:
209	case AArch64::AUTIBSPPCi:
210	case AArch64::AUTIASPPCr:
211	case AArch64::AUTIBSPPCr:
212	return AArch64::LR;
213	case AArch64::AUTIA1716:
214	case AArch64::AUTIB1716:
215	case AArch64::AUTIA171615:
216	case AArch64::AUTIB171615:
217	return AArch64::X17;
218	case AArch64::AUTIA:
219	case AArch64::AUTIB:
220	case AArch64::AUTDA:
221	case AArch64::AUTDB:
222	case AArch64::AUTIZA:
223	case AArch64::AUTIZB:
224	case AArch64::AUTDZA:
225	case AArch64::AUTDZB:
226	return Inst.getOperand(i: 0).getReg();
227	case AArch64::LDRAAwriteback:
228	case AArch64::LDRABwriteback:
229	// Note that LDRA(A|B)indexed are not listed here, as they do not write
230	// an authenticated pointer back to the register.
231	IsChecked = true;
232	return Inst.getOperand(i: 2).getReg();
233	default:
234	return std::nullopt;
235	}
236	}
237
238	std::optional<MCPhysReg> getSignedReg(const MCInst &Inst) const override {
239	switch (Inst.getOpcode()) {
240	case AArch64::PACIA:
241	case AArch64::PACIB:
242	case AArch64::PACDA:
243	case AArch64::PACDB:
244	case AArch64::PACIZA:
245	case AArch64::PACIZB:
246	case AArch64::PACDZA:
247	case AArch64::PACDZB:
248	return Inst.getOperand(i: 0).getReg();
249	case AArch64::PACIAZ:
250	case AArch64::PACIBZ:
251	case AArch64::PACIASP:
252	case AArch64::PACIBSP:
253	case AArch64::PACIASPPC:
254	case AArch64::PACIBSPPC:
255	case AArch64::PACNBIASPPC:
256	case AArch64::PACNBIBSPPC:
257	return AArch64::LR;
258	case AArch64::PACIA1716:
259	case AArch64::PACIB1716:
260	case AArch64::PACIA171615:
261	case AArch64::PACIB171615:
262	return AArch64::X17;
263	default:
264	return std::nullopt;
265	}
266	}
267
268	std::optional<MCPhysReg>
269	getRegUsedAsRetDest(const MCInst &Inst,
270	bool &IsAuthenticatedInternally) const override {
271	assert(isReturn(Inst));
272	switch (Inst.getOpcode()) {
273	case AArch64::RET:
274	IsAuthenticatedInternally = false;
275	return Inst.getOperand(i: 0).getReg();
276
277	case AArch64::RETAA:
278	case AArch64::RETAB:
279	case AArch64::RETAASPPCi:
280	case AArch64::RETABSPPCi:
281	case AArch64::RETAASPPCr:
282	case AArch64::RETABSPPCr:
283	IsAuthenticatedInternally = true;
284	return AArch64::LR;
285	case AArch64::ERET:
286	case AArch64::ERETAA:
287	case AArch64::ERETAB:
288	// The ERET* instructions use either register ELR_EL1, ELR_EL2 or
289	// ELR_EL3, depending on the current Exception Level at run-time.
290	//
291	// Furthermore, these registers are not modelled by LLVM as a regular
292	// MCPhysReg, so there is no way to indicate that through the current API.
293	return std::nullopt;
294	default:
295	llvm_unreachable("Unhandled return instruction");
296	}
297	}
298
299	MCPhysReg getRegUsedAsIndirectBranchDest(
300	const MCInst &Inst, bool &IsAuthenticatedInternally) const override {
301	assert(isIndirectCall(Inst) || isIndirectBranch(Inst));
302
303	switch (Inst.getOpcode()) {
304	case AArch64::BR:
305	case AArch64::BLR:
306	IsAuthenticatedInternally = false;
307	return Inst.getOperand(i: 0).getReg();
308	case AArch64::BRAA:
309	case AArch64::BRAB:
310	case AArch64::BRAAZ:
311	case AArch64::BRABZ:
312	case AArch64::BLRAA:
313	case AArch64::BLRAB:
314	case AArch64::BLRAAZ:
315	case AArch64::BLRABZ:
316	IsAuthenticatedInternally = true;
317	return Inst.getOperand(i: 0).getReg();
318	default:
319	llvm_unreachable("Unhandled indirect branch or call");
320	}
321	}
322
323	std::optional<MCPhysReg>
324	getMaterializedAddressRegForPtrAuth(const MCInst &Inst) const override {
325	switch (Inst.getOpcode()) {
326	case AArch64::ADR:
327	case AArch64::ADRP:
328	// These instructions produce an address value based on the information
329	// encoded into the instruction itself (which should reside in a read-only
330	// code memory) and the value of PC register (that is, the location of
331	// this instruction), so the produced value is not attacker-controlled.
332	return Inst.getOperand(i: 0).getReg();
333	default:
334	return std::nullopt;
335	}
336	}
337
338	std::optional<std::pair<MCPhysReg, MCPhysReg>>
339	analyzeAddressArithmeticsForPtrAuth(const MCInst &Inst) const override {
340	switch (Inst.getOpcode()) {
341	default:
342	return std::nullopt;
343	case AArch64::ADDXri:
344	case AArch64::SUBXri:
345	// The immediate addend is encoded into the instruction itself, so it is
346	// not attacker-controlled under Pointer Authentication threat model.
347	return std::make_pair(x: Inst.getOperand(i: 0).getReg(),
348	y: Inst.getOperand(i: 1).getReg());
349	case AArch64::ORRXrs:
350	// "mov Xd, Xm" is equivalent to "orr Xd, XZR, Xm, lsl #0"
351	if (Inst.getOperand(1).getReg() != AArch64::XZR ||
352	Inst.getOperand(3).getImm() != 0)
353	return std::nullopt;
354
355	return std::make_pair(x: Inst.getOperand(i: 0).getReg(),
356	y: Inst.getOperand(i: 2).getReg());
357	}
358	}
359
360	std::optional<std::pair<MCPhysReg, MCInst *>>
361	getAuthCheckedReg(BinaryBasicBlock &BB) const override {
362	// Match several possible hard-coded sequences of instructions which can be
363	// emitted by LLVM backend to check that the authenticated pointer is
364	// correct (see AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue).
365	//
366	// This function only matches sequences involving branch instructions.
367	// All these sequences have the form:
368	//
369	// (0) ... regular code that authenticates a pointer in Xn ...
370	// (1) analyze Xn
371	// (2) branch to .Lon_success if the pointer is correct
372	// (3) BRK #imm (fall-through basic block)
373	//
374	// In the above pseudocode, (1) + (2) is one of the following sequences:
375	//
376	// - eor Xtmp, Xn, Xn, lsl #1
377	// tbz Xtmp, #62, .Lon_success
378	//
379	// - mov Xtmp, Xn
380	// xpac(i|d) Xn (or xpaclri if Xn is LR)
381	// cmp Xtmp, Xn
382	// b.eq .Lon_success
383	//
384	// Note that any branch destination operand is accepted as .Lon_success -
385	// it is the responsibility of the caller of getAuthCheckedReg to inspect
386	// the list of successors of this basic block as appropriate.
387
388	// Any of the above code sequences assume the fall-through basic block
389	// is a dead-end BRK instruction (any immediate operand is accepted).
390	const BinaryBasicBlock *BreakBB = BB.getFallthrough();
391	if (!BreakBB || BreakBB->empty() ||
392	BreakBB->front().getOpcode() != AArch64::BRK)
393	return std::nullopt;
394
395	// Iterate over the instructions of BB in reverse order, matching opcodes
396	// and operands.
397	MCPhysReg TestedReg = 0;
398	MCPhysReg ScratchReg = 0;
399	auto It = BB.end();
400	auto StepAndGetOpcode = [&It, &BB]() -> int {
401	if (It == BB.begin())
402	return -1;
403	--It;
404	return It->getOpcode();
405	};
406
407	switch (StepAndGetOpcode()) {
408	default:
409	// Not matched the branch instruction.
410	return std::nullopt;
411	case AArch64::Bcc:
412	// Bcc EQ, .Lon_success
413	if (It->getOperand(i: 0).getImm() != AArch64CC::EQ)
414	return std::nullopt;
415	// Not checking .Lon_success (see above).
416
417	// SUBSXrs XZR, TestedReg, ScratchReg, 0 (used by "CMP reg, reg" alias)
418	if (StepAndGetOpcode() != AArch64::SUBSXrs ||
419	It->getOperand(0).getReg() != AArch64::XZR ||
420	It->getOperand(3).getImm() != 0)
421	return std::nullopt;
422	TestedReg = It->getOperand(i: 1).getReg();
423	ScratchReg = It->getOperand(i: 2).getReg();
424
425	// Either XPAC(I|D) ScratchReg, ScratchReg
426	// or XPACLRI
427	switch (StepAndGetOpcode()) {
428	default:
429	return std::nullopt;
430	case AArch64::XPACLRI:
431	// No operands to check, but using XPACLRI forces TestedReg to be X30.
432	if (TestedReg != AArch64::LR)
433	return std::nullopt;
434	break;
435	case AArch64::XPACI:
436	case AArch64::XPACD:
437	if (It->getOperand(i: 0).getReg() != ScratchReg ||
438	It->getOperand(i: 1).getReg() != ScratchReg)
439	return std::nullopt;
440	break;
441	}
442
443	// ORRXrs ScratchReg, XZR, TestedReg, 0 (used by "MOV reg, reg" alias)
444	if (StepAndGetOpcode() != AArch64::ORRXrs)
445	return std::nullopt;
446	if (It->getOperand(0).getReg() != ScratchReg ||
447	It->getOperand(1).getReg() != AArch64::XZR ||
448	It->getOperand(2).getReg() != TestedReg ||
449	It->getOperand(3).getImm() != 0)
450	return std::nullopt;
451
452	return std::make_pair(x&: TestedReg, y: &*It);
453
454	case AArch64::TBZX:
455	// TBZX ScratchReg, 62, .Lon_success
456	ScratchReg = It->getOperand(i: 0).getReg();
457	if (It->getOperand(i: 1).getImm() != 62)
458	return std::nullopt;
459	// Not checking .Lon_success (see above).
460
461	// EORXrs ScratchReg, TestedReg, TestedReg, 1
462	if (StepAndGetOpcode() != AArch64::EORXrs)
463	return std::nullopt;
464	TestedReg = It->getOperand(i: 1).getReg();
465	if (It->getOperand(i: 0).getReg() != ScratchReg ||
466	It->getOperand(i: 2).getReg() != TestedReg ||
467	It->getOperand(i: 3).getImm() != 1)
468	return std::nullopt;
469
470	return std::make_pair(x&: TestedReg, y: &*It);
471	}
472	}
473
474	std::optional<MCPhysReg> getAuthCheckedReg(const MCInst &Inst,
475	bool MayOverwrite) const override {
476	// Cannot trivially reuse AArch64InstrInfo::getMemOperandWithOffsetWidth()
477	// method as it accepts an instance of MachineInstr, not MCInst.
478	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
479
480	// If signing oracles are considered, the particular value left in the base
481	// register after this instruction is important. This function checks that
482	// if the base register was overwritten, it is due to address write-back:
483	//
484	// ; good:
485	// autdza x1 ; x1 is authenticated (may fail)
486	// ldr x0, [x1, #8] ; x1 is checked and not changed
487	// pacdzb x1
488	//
489	// ; also good:
490	// autdza x1
491	// ldr x0, [x1, #8]! ; x1 is checked and incremented by 8
492	// pacdzb x1
493	//
494	// ; bad (the value being signed is not the authenticated one):
495	// autdza x1
496	// ldr x1, [x1, #8] ; x1 is overwritten with an unrelated value
497	// pacdzb x1
498	//
499	// ; also bad:
500	// autdza x1
501	// pacdzb x1 ; possibly signing the result of failed authentication
502	//
503	// Note that this function is not needed for authentication oracles, as the
504	// particular value left in the register after a successful memory access
505	// is not important.
506	auto ClobbersBaseRegExceptWriteback = [&](unsigned BaseRegUseIndex) {
507	// FIXME: Compute the indices of address operands (base reg and written-
508	// back result) in AArch64InstrInfo instead of this ad-hoc code.
509	MCPhysReg BaseReg = Inst.getOperand(i: BaseRegUseIndex).getReg();
510	unsigned WrittenBackDefIndex = Desc.getOperandConstraint(
511	OpNum: BaseRegUseIndex, Constraint: MCOI::OperandConstraint::TIED_TO);
512
513	for (unsigned DefIndex = 0; DefIndex < Desc.getNumDefs(); ++DefIndex) {
514	// Address write-back is permitted:
515	//
516	// autda x0, x2
517	// ; x0 is authenticated
518	// ldr x1, [x0, #8]!
519	// ; x0 is trusted (as authenticated and checked)
520	if (DefIndex == WrittenBackDefIndex)
521	continue;
522
523	// Any other overwriting is not permitted:
524	//
525	// autda x0, x2
526	// ; x0 is authenticated
527	// ldr w0, [x0]
528	// ; x0 is not authenticated anymore
529	if (RegInfo->regsOverlap(RegA: Inst.getOperand(i: DefIndex).getReg(), RegB: BaseReg))
530	return true;
531	}
532
533	return false;
534	};
535
536	// FIXME: Not all load instructions are handled by this->mayLoad(Inst).
537	// On the other hand, MCInstrDesc::mayLoad() is permitted to return
538	// true for non-load instructions (such as AArch64::HINT) which
539	// would result in false negatives.
540	if (mayLoad(Inst)) {
541	// The first Use operand is the base address register.
542	unsigned BaseRegIndex = Desc.getNumDefs();
543
544	// Reject non-immediate offsets, as adding a 64-bit register can change
545	// the resulting address arbitrarily.
546	for (unsigned I = BaseRegIndex + 1, E = Desc.getNumOperands(); I < E; ++I)
547	if (Inst.getOperand(i: I).isReg())
548	return std::nullopt;
549
550	if (!MayOverwrite && ClobbersBaseRegExceptWriteback(BaseRegIndex))
551	return std::nullopt;
552
553	return Inst.getOperand(i: BaseRegIndex).getReg();
554	}
555
556	// Store instructions are not handled yet, as they are not important for
557	// pauthtest ABI. Though, they could be handled similar to loads, if needed.
558
559	return std::nullopt;
560	}
561
562	bool isADRP(const MCInst &Inst) const override {
563	return Inst.getOpcode() == AArch64::ADRP;
564	}
565
566	bool isADR(const MCInst &Inst) const override {
567	return Inst.getOpcode() == AArch64::ADR;
568	}
569
570	bool isAddXri(const MCInst &Inst) const override {
571	return Inst.getOpcode() == AArch64::ADDXri;
572	}
573
574	MCPhysReg getADRReg(const MCInst &Inst) const {
575	assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
576	assert(MCPlus::getNumPrimeOperands(Inst) != 0 &&
577	"No operands for ADR instruction");
578	assert(Inst.getOperand(0).isReg() &&
579	"Unexpected operand in ADR instruction");
580	return Inst.getOperand(i: 0).getReg();
581	}
582
583	InstructionListType undoAdrpAddRelaxation(const MCInst &ADRInst,
584	MCContext *Ctx) const override {
585	assert(isADR(ADRInst) && "ADR instruction expected");
586
587	const MCPhysReg Reg = getADRReg(Inst: ADRInst);
588	const MCSymbol *Target = getTargetSymbol(Inst: ADRInst);
589	const uint64_t Addend = getTargetAddend(Inst: ADRInst);
590	return materializeAddress(Target, Ctx, RegName: Reg, Addend);
591	}
592
593	bool isTB(const MCInst &Inst) const {
594	return (Inst.getOpcode() == AArch64::TBNZW ||
595	Inst.getOpcode() == AArch64::TBNZX ||
596	Inst.getOpcode() == AArch64::TBZW ||
597	Inst.getOpcode() == AArch64::TBZX);
598	}
599
600	bool isCB(const MCInst &Inst) const {
601	return (Inst.getOpcode() == AArch64::CBNZW ||
602	Inst.getOpcode() == AArch64::CBNZX ||
603	Inst.getOpcode() == AArch64::CBZW ||
604	Inst.getOpcode() == AArch64::CBZX);
605	}
606
607	bool isMOVW(const MCInst &Inst) const override {
608	return (Inst.getOpcode() == AArch64::MOVKWi ||
609	Inst.getOpcode() == AArch64::MOVKXi ||
610	Inst.getOpcode() == AArch64::MOVNWi ||
611	Inst.getOpcode() == AArch64::MOVNXi ||
612	Inst.getOpcode() == AArch64::MOVZXi ||
613	Inst.getOpcode() == AArch64::MOVZWi);
614	}
615
616	bool isADD(const MCInst &Inst) const {
617	return (Inst.getOpcode() == AArch64::ADDSWri ||
618	Inst.getOpcode() == AArch64::ADDSWrr ||
619	Inst.getOpcode() == AArch64::ADDSWrs ||
620	Inst.getOpcode() == AArch64::ADDSWrx ||
621	Inst.getOpcode() == AArch64::ADDSXri ||
622	Inst.getOpcode() == AArch64::ADDSXrr ||
623	Inst.getOpcode() == AArch64::ADDSXrs ||
624	Inst.getOpcode() == AArch64::ADDSXrx ||
625	Inst.getOpcode() == AArch64::ADDSXrx64 ||
626	Inst.getOpcode() == AArch64::ADDWri ||
627	Inst.getOpcode() == AArch64::ADDWrr ||
628	Inst.getOpcode() == AArch64::ADDWrs ||
629	Inst.getOpcode() == AArch64::ADDWrx ||
630	Inst.getOpcode() == AArch64::ADDXri ||
631	Inst.getOpcode() == AArch64::ADDXrr ||
632	Inst.getOpcode() == AArch64::ADDXrs ||
633	Inst.getOpcode() == AArch64::ADDXrx ||
634	Inst.getOpcode() == AArch64::ADDXrx64);
635	}
636
637	bool isLDRB(const MCInst &Inst) const {
638	const unsigned opcode = Inst.getOpcode();
639	switch (opcode) {
640	case AArch64::LDRBpost:
641	case AArch64::LDRBBpost:
642	case AArch64::LDRBBpre:
643	case AArch64::LDRBBroW:
644	case AArch64::LDRBroW:
645	case AArch64::LDRBroX:
646	case AArch64::LDRBBroX:
647	case AArch64::LDRBBui:
648	case AArch64::LDRBui:
649	case AArch64::LDRBpre:
650	case AArch64::LDRSBWpost:
651	case AArch64::LDRSBWpre:
652	case AArch64::LDRSBWroW:
653	case AArch64::LDRSBWroX:
654	case AArch64::LDRSBWui:
655	case AArch64::LDRSBXpost:
656	case AArch64::LDRSBXpre:
657	case AArch64::LDRSBXroW:
658	case AArch64::LDRSBXroX:
659	case AArch64::LDRSBXui:
660	case AArch64::LDURBi:
661	case AArch64::LDURBBi:
662	case AArch64::LDURSBWi:
663	case AArch64::LDURSBXi:
664	case AArch64::LDTRBi:
665	case AArch64::LDTRSBWi:
666	case AArch64::LDTRSBXi:
667	return true;
668	default:
669	break;
670	}
671
672	return false;
673	}
674
675	bool isLDRH(const MCInst &Inst) const {
676	const unsigned opcode = Inst.getOpcode();
677	switch (opcode) {
678	case AArch64::LDRHpost:
679	case AArch64::LDRHHpost:
680	case AArch64::LDRHHpre:
681	case AArch64::LDRHroW:
682	case AArch64::LDRHHroW:
683	case AArch64::LDRHroX:
684	case AArch64::LDRHHroX:
685	case AArch64::LDRHHui:
686	case AArch64::LDRHui:
687	case AArch64::LDRHpre:
688	case AArch64::LDRSHWpost:
689	case AArch64::LDRSHWpre:
690	case AArch64::LDRSHWroW:
691	case AArch64::LDRSHWroX:
692	case AArch64::LDRSHWui:
693	case AArch64::LDRSHXpost:
694	case AArch64::LDRSHXpre:
695	case AArch64::LDRSHXroW:
696	case AArch64::LDRSHXroX:
697	case AArch64::LDRSHXui:
698	case AArch64::LDURHi:
699	case AArch64::LDURHHi:
700	case AArch64::LDURSHWi:
701	case AArch64::LDURSHXi:
702	case AArch64::LDTRHi:
703	case AArch64::LDTRSHWi:
704	case AArch64::LDTRSHXi:
705	return true;
706	default:
707	break;
708	}
709
710	return false;
711	}
712
713	bool isLDRW(const MCInst &Inst) const {
714	const unsigned opcode = Inst.getOpcode();
715	switch (opcode) {
716	case AArch64::LDRWpost:
717	case AArch64::LDRWpre:
718	case AArch64::LDRWroW:
719	case AArch64::LDRWroX:
720	case AArch64::LDRWui:
721	case AArch64::LDRWl:
722	case AArch64::LDRSWl:
723	case AArch64::LDURWi:
724	case AArch64::LDRSWpost:
725	case AArch64::LDRSWpre:
726	case AArch64::LDRSWroW:
727	case AArch64::LDRSWroX:
728	case AArch64::LDRSWui:
729	case AArch64::LDURSWi:
730	case AArch64::LDTRWi:
731	case AArch64::LDTRSWi:
732	case AArch64::LDPWi:
733	case AArch64::LDPWpost:
734	case AArch64::LDPWpre:
735	case AArch64::LDPSWi:
736	case AArch64::LDPSWpost:
737	case AArch64::LDPSWpre:
738	case AArch64::LDNPWi:
739	return true;
740	default:
741	break;
742	}
743
744	return false;
745	}
746
747	bool isLDRX(const MCInst &Inst) const {
748	const unsigned opcode = Inst.getOpcode();
749	switch (opcode) {
750	case AArch64::LDRXpost:
751	case AArch64::LDRXpre:
752	case AArch64::LDRXroW:
753	case AArch64::LDRXroX:
754	case AArch64::LDRXui:
755	case AArch64::LDRXl:
756	case AArch64::LDURXi:
757	case AArch64::LDTRXi:
758	case AArch64::LDNPXi:
759	case AArch64::LDPXi:
760	case AArch64::LDPXpost:
761	case AArch64::LDPXpre:
762	return true;
763	default:
764	break;
765	}
766
767	return false;
768	}
769
770	bool isLDRS(const MCInst &Inst) const {
771	const unsigned opcode = Inst.getOpcode();
772	switch (opcode) {
773	case AArch64::LDRSl:
774	case AArch64::LDRSui:
775	case AArch64::LDRSroW:
776	case AArch64::LDRSroX:
777	case AArch64::LDURSi:
778	case AArch64::LDPSi:
779	case AArch64::LDNPSi:
780	case AArch64::LDRSpre:
781	case AArch64::LDRSpost:
782	case AArch64::LDPSpost:
783	case AArch64::LDPSpre:
784	return true;
785	default:
786	break;
787	}
788
789	return false;
790	}
791
792	bool isLDRD(const MCInst &Inst) const {
793	const unsigned opcode = Inst.getOpcode();
794	switch (opcode) {
795	case AArch64::LDRDl:
796	case AArch64::LDRDui:
797	case AArch64::LDRDpre:
798	case AArch64::LDRDpost:
799	case AArch64::LDRDroW:
800	case AArch64::LDRDroX:
801	case AArch64::LDURDi:
802	case AArch64::LDPDi:
803	case AArch64::LDNPDi:
804	case AArch64::LDPDpost:
805	case AArch64::LDPDpre:
806	return true;
807	default:
808	break;
809	}
810
811	return false;
812	}
813
814	bool isLDRQ(const MCInst &Inst) const {
815	const unsigned opcode = Inst.getOpcode();
816	switch (opcode) {
817	case AArch64::LDRQui:
818	case AArch64::LDRQl:
819	case AArch64::LDRQpre:
820	case AArch64::LDRQpost:
821	case AArch64::LDRQroW:
822	case AArch64::LDRQroX:
823	case AArch64::LDURQi:
824	case AArch64::LDPQi:
825	case AArch64::LDNPQi:
826	case AArch64::LDPQpost:
827	case AArch64::LDPQpre:
828	return true;
829	default:
830	break;
831	}
832
833	return false;
834	}
835
836	bool isBRA(const MCInst &Inst) const {
837	switch (Inst.getOpcode()) {
838	case AArch64::BRAA:
839	case AArch64::BRAB:
840	case AArch64::BRAAZ:
841	case AArch64::BRABZ:
842	return true;
843	default:
844	return false;
845	}
846	}
847
848	bool mayLoad(const MCInst &Inst) const override {
849	// FIXME: Probably this could be tablegen-erated not to miss any existing
850	// or future opcodes.
851	return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst) ||
852	isLDRQ(Inst) || isLDRD(Inst) || isLDRS(Inst);
853	}
854
855	bool isAArch64ExclusiveLoad(const MCInst &Inst) const override {
856	return (Inst.getOpcode() == AArch64::LDXPX ||
857	Inst.getOpcode() == AArch64::LDXPW ||
858	Inst.getOpcode() == AArch64::LDXRX ||
859	Inst.getOpcode() == AArch64::LDXRW ||
860	Inst.getOpcode() == AArch64::LDXRH ||
861	Inst.getOpcode() == AArch64::LDXRB ||
862	Inst.getOpcode() == AArch64::LDAXPX ||
863	Inst.getOpcode() == AArch64::LDAXPW ||
864	Inst.getOpcode() == AArch64::LDAXRX ||
865	Inst.getOpcode() == AArch64::LDAXRW ||
866	Inst.getOpcode() == AArch64::LDAXRH ||
867	Inst.getOpcode() == AArch64::LDAXRB);
868	}
869
870	bool isAArch64ExclusiveStore(const MCInst &Inst) const override {
871	return (Inst.getOpcode() == AArch64::STXPX ||
872	Inst.getOpcode() == AArch64::STXPW ||
873	Inst.getOpcode() == AArch64::STXRX ||
874	Inst.getOpcode() == AArch64::STXRW ||
875	Inst.getOpcode() == AArch64::STXRH ||
876	Inst.getOpcode() == AArch64::STXRB ||
877	Inst.getOpcode() == AArch64::STLXPX ||
878	Inst.getOpcode() == AArch64::STLXPW ||
879	Inst.getOpcode() == AArch64::STLXRX ||
880	Inst.getOpcode() == AArch64::STLXRW ||
881	Inst.getOpcode() == AArch64::STLXRH ||
882	Inst.getOpcode() == AArch64::STLXRB);
883	}
884
885	bool isAArch64ExclusiveClear(const MCInst &Inst) const override {
886	return (Inst.getOpcode() == AArch64::CLREX);
887	}
888
889	bool isLoadFromStack(const MCInst &Inst) const {
890	if (!mayLoad(Inst))
891	return false;
892	for (const MCOperand &Operand : useOperands(Inst)) {
893	if (!Operand.isReg())
894	continue;
895	unsigned Reg = Operand.getReg();
896	if (Reg == AArch64::SP || Reg == AArch64::WSP)
897	return true;
898	}
899	return false;
900	}
901
902	bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
903	MCPhysReg &To) const override {
904	if (Inst.getOpcode() == AArch64::FMOVDXr) {
905	From = Inst.getOperand(i: 1).getReg();
906	To = Inst.getOperand(i: 0).getReg();
907	return true;
908	}
909
910	if (Inst.getOpcode() != AArch64::ORRXrs)
911	return false;
912	if (Inst.getOperand(1).getReg() != AArch64::XZR)
913	return false;
914	if (Inst.getOperand(i: 3).getImm() != 0)
915	return false;
916	From = Inst.getOperand(i: 2).getReg();
917	To = Inst.getOperand(i: 0).getReg();
918	return true;
919	}
920
921	bool isIndirectCall(const MCInst &Inst) const override {
922	return isIndirectCallOpcode(Opc: Inst.getOpcode());
923	}
924
925	MCPhysReg getSpRegister(int Size) const {
926	switch (Size) {
927	case 4:
928	return AArch64::WSP;
929	case 8:
930	return AArch64::SP;
931	default:
932	llvm_unreachable("Unexpected size");
933	}
934	}
935
936	MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
937	switch (ArgNo) {
938	case 0:
939	return AArch64::X0;
940	case 1:
941	return AArch64::X1;
942	case 2:
943	return AArch64::X2;
944	case 3:
945	return AArch64::X3;
946	case 4:
947	return AArch64::X4;
948	case 5:
949	return AArch64::X5;
950	case 6:
951	return AArch64::X6;
952	case 7:
953	return AArch64::X7;
954	default:
955	return getNoRegister();
956	}
957	}
958
959	bool hasPCRelOperand(const MCInst &Inst) const override {
960	// ADRP is blacklisted and is an exception. Even though it has a
961	// PC-relative operand, this operand is not a complete symbol reference
962	// and BOLT shouldn't try to process it in isolation.
963	if (isADRP(Inst))
964	return false;
965
966	if (isADR(Inst))
967	return true;
968
969	// Look for literal addressing mode (see C1-143 ARM DDI 0487B.a)
970	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
971	for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I)
972	if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)
973	return true;
974
975	return false;
976	}
977
978	bool evaluateADR(const MCInst &Inst, int64_t &Imm,
979	const MCExpr **DispExpr) const {
980	assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
981
982	const MCOperand &Label = Inst.getOperand(i: 1);
983	if (!Label.isImm()) {
984	assert(Label.isExpr() && "Unexpected ADR operand");
985	assert(DispExpr && "DispExpr must be set");
986	*DispExpr = Label.getExpr();
987	return false;
988	}
989
990	if (Inst.getOpcode() == AArch64::ADR) {
991	Imm = Label.getImm();
992	return true;
993	}
994	Imm = Label.getImm() << 12;
995	return true;
996	}
997
998	bool evaluateAArch64MemoryOperand(const MCInst &Inst, int64_t &DispImm,
999	const MCExpr **DispExpr = nullptr) const {
1000	if (isADR(Inst) || isADRP(Inst))
1001	return evaluateADR(Inst, Imm&: DispImm, DispExpr);
1002
1003	// Literal addressing mode
1004	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1005	for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1006	if (MCII.operands()[I].OperandType != MCOI::OPERAND_PCREL)
1007	continue;
1008
1009	if (!Inst.getOperand(i: I).isImm()) {
1010	assert(Inst.getOperand(I).isExpr() && "Unexpected PCREL operand");
1011	assert(DispExpr && "DispExpr must be set");
1012	*DispExpr = Inst.getOperand(i: I).getExpr();
1013	return true;
1014	}
1015
1016	DispImm = Inst.getOperand(i: I).getImm() * 4;
1017	return true;
1018	}
1019	return false;
1020	}
1021
1022	bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
1023	uint64_t Address,
1024	uint64_t Size) const override {
1025	int64_t DispValue;
1026	const MCExpr *DispExpr = nullptr;
1027	if (!evaluateAArch64MemoryOperand(Inst, DispImm&: DispValue, DispExpr: &DispExpr))
1028	return false;
1029
1030	// Make sure it's a well-formed addressing we can statically evaluate.
1031	if (DispExpr)
1032	return false;
1033
1034	Target = DispValue;
1035	if (Inst.getOpcode() == AArch64::ADRP)
1036	Target += Address & ~0xFFFULL;
1037	else
1038	Target += Address;
1039	return true;
1040	}
1041
1042	MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
1043	MCInst::iterator OI = Inst.begin();
1044	if (isADR(Inst) || isADRP(Inst)) {
1045	assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&
1046	"Unexpected number of operands");
1047	return ++OI;
1048	}
1049	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1050	for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1051	if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)
1052	break;
1053	++OI;
1054	}
1055	assert(OI != Inst.end() && "Literal operand not found");
1056	return OI;
1057	}
1058
1059	bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
1060	MCInst::iterator OI = getMemOperandDisp(Inst);
1061	*OI = Operand;
1062	return true;
1063	}
1064
1065	void getCalleeSavedRegs(BitVector &Regs) const override {
1066	Regs |= getAliases(AArch64::X18);
1067	Regs |= getAliases(AArch64::X19);
1068	Regs |= getAliases(AArch64::X20);
1069	Regs |= getAliases(AArch64::X21);
1070	Regs |= getAliases(AArch64::X22);
1071	Regs |= getAliases(AArch64::X23);
1072	Regs |= getAliases(AArch64::X24);
1073	Regs |= getAliases(AArch64::X25);
1074	Regs |= getAliases(AArch64::X26);
1075	Regs |= getAliases(AArch64::X27);
1076	Regs |= getAliases(AArch64::X28);
1077	Regs |= getAliases(AArch64::LR);
1078	Regs |= getAliases(AArch64::FP);
1079	}
1080
1081	const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr,
1082	MCContext &Ctx,
1083	uint32_t RelType) const override {
1084
1085	if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||
1086	RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {
1087	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx);
1088	} else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||
1089	RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||
1090	RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||
1091	RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 ||
1092	RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
1093	// Never emit a GOT reloc, we handled this in
1094	// RewriteInstance::readRelocations().
1095	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx);
1096	} else {
1097	switch (RelType) {
1098	case ELF::R_AARCH64_ADD_ABS_LO12_NC:
1099	case ELF::R_AARCH64_LD64_GOT_LO12_NC:
1100	case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
1101	case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
1102	case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
1103	case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
1104	case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
1105	case ELF::R_AARCH64_TLSDESC_ADD_LO12:
1106	case ELF::R_AARCH64_TLSDESC_LD64_LO12:
1107	case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
1108	case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
1109	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_LO12, Ctx);
1110	case ELF::R_AARCH64_MOVW_UABS_G3:
1111	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G3, Ctx);
1112	case ELF::R_AARCH64_MOVW_UABS_G2:
1113	case ELF::R_AARCH64_MOVW_UABS_G2_NC:
1114	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G2_NC, Ctx);
1115	case ELF::R_AARCH64_MOVW_UABS_G1:
1116	case ELF::R_AARCH64_MOVW_UABS_G1_NC:
1117	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G1_NC, Ctx);
1118	case ELF::R_AARCH64_MOVW_UABS_G0:
1119	case ELF::R_AARCH64_MOVW_UABS_G0_NC:
1120	return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G0_NC, Ctx);
1121	default:
1122	break;
1123	}
1124	}
1125	return Expr;
1126	}
1127
1128	bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const {
1129	if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1130	return false;
1131
1132	// Auto-select correct operand number
1133	if (OpNum == 0) {
1134	if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst) ||
1135	isMOVW(Inst))
1136	OpNum = 1;
1137	if (isTB(Inst) || isAddXri(Inst))
1138	OpNum = 2;
1139	}
1140
1141	return true;
1142	}
1143
1144	const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1145	auto *AArchExpr = dyn_cast<AArch64MCExpr>(Val: Expr);
1146	if (AArchExpr && AArchExpr->getSubExpr())
1147	return getTargetSymbol(Expr: AArchExpr->getSubExpr());
1148
1149	return MCPlusBuilder::getTargetSymbol(Expr);
1150	}
1151
1152	const MCSymbol *getTargetSymbol(const MCInst &Inst,
1153	unsigned OpNum = 0) const override {
1154	if (!OpNum && !getSymbolRefOperandNum(Inst, OpNum))
1155	return nullptr;
1156
1157	const MCOperand &Op = Inst.getOperand(i: OpNum);
1158	if (!Op.isExpr())
1159	return nullptr;
1160
1161	return getTargetSymbol(Expr: Op.getExpr());
1162	}
1163
1164	int64_t getTargetAddend(const MCExpr *Expr) const override {
1165	auto *AArchExpr = dyn_cast<AArch64MCExpr>(Val: Expr);
1166	if (AArchExpr && AArchExpr->getSubExpr())
1167	return getTargetAddend(Expr: AArchExpr->getSubExpr());
1168
1169	auto *BinExpr = dyn_cast<MCBinaryExpr>(Val: Expr);
1170	if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add)
1171	return getTargetAddend(Expr: BinExpr->getRHS());
1172
1173	auto *ConstExpr = dyn_cast<MCConstantExpr>(Val: Expr);
1174	if (ConstExpr)
1175	return ConstExpr->getValue();
1176
1177	return 0;
1178	}
1179
1180	int64_t getTargetAddend(const MCInst &Inst,
1181	unsigned OpNum = 0) const override {
1182	if (!getSymbolRefOperandNum(Inst, OpNum))
1183	return 0;
1184
1185	const MCOperand &Op = Inst.getOperand(i: OpNum);
1186	if (!Op.isExpr())
1187	return 0;
1188
1189	return getTargetAddend(Expr: Op.getExpr());
1190	}
1191
1192	void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
1193	MCContext *Ctx) const override {
1194	assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
1195	"Invalid instruction");
1196	assert(MCPlus::getNumPrimeOperands(Inst) >= 1 &&
1197	"Invalid number of operands");
1198	MCInst::iterator OI = Inst.begin();
1199
1200	if (isConditionalBranch(Inst)) {
1201	assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&
1202	"Invalid number of operands");
1203	++OI;
1204	}
1205
1206	if (isTB(Inst)) {
1207	assert(MCPlus::getNumPrimeOperands(Inst) >= 3 &&
1208	"Invalid number of operands");
1209	OI = Inst.begin() + 2;
1210	}
1211
1212	*OI = MCOperand::createExpr(
1213	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
1214	}
1215
1216	/// Matches indirect branch patterns in AArch64 related to a jump table (JT),
1217	/// helping us to build the complete CFG. A typical indirect branch to
1218	/// a jump table entry in AArch64 looks like the following:
1219	///
1220	/// adrp x1, #-7585792 # Get JT Page location
1221	/// add x1, x1, #692 # Complement with JT Page offset
1222	/// ldrh w0, [x1, w0, uxtw #1] # Loads JT entry
1223	/// adr x1, #12 # Get PC + 12 (end of this BB) used next
1224	/// add x0, x1, w0, sxth #2 # Finish building branch target
1225	/// # (entries in JT are relative to the end
1226	/// # of this BB)
1227	/// br x0 # Indirect jump instruction
1228	///
1229	/// Return true on successful jump table instruction sequence match, false
1230	/// otherwise.
1231	bool analyzeIndirectBranchFragment(
1232	const MCInst &Inst,
1233	DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain,
1234	const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue,
1235	MCInst *&PCRelBase) const {
1236	// The only kind of indirect branches we match is jump table, thus ignore
1237	// authenticating branch instructions early.
1238	if (isBRA(Inst))
1239	return false;
1240
1241	// Expect AArch64 BR
1242	assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");
1243
1244	JumpTable = nullptr;
1245
1246	// Match the indirect branch pattern for aarch64
1247	SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];
1248	if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr)
1249	return false;
1250
1251	const MCInst *DefAdd = UsesRoot[0];
1252
1253	// Now we match an ADD
1254	if (!isADD(Inst: *DefAdd)) {
1255	// If the address is not broken up in two parts, this is not branching
1256	// according to a jump table entry. Fail.
1257	return false;
1258	}
1259	if (DefAdd->getOpcode() == AArch64::ADDXri) {
1260	// This can happen when there is no offset, but a direct jump that was
1261	// transformed into an indirect one (indirect tail call) :
1262	// ADRP x2, Perl_re_compiler
1263	// ADD x2, x2, :lo12:Perl_re_compiler
1264	// BR x2
1265	return false;
1266	}
1267	if (DefAdd->getOpcode() == AArch64::ADDXrs) {
1268	// Covers the less common pattern where JT entries are relative to
1269	// the JT itself (like x86). Seems less efficient since we can't
1270	// assume the JT is aligned at 4B boundary and thus drop 2 bits from
1271	// JT values.
1272	// cde264:
1273	// adrp x12, #21544960 ; 216a000
1274	// add x12, x12, #1696 ; 216a6a0 (JT object in .rodata)
1275	// ldrsw x8, [x12, x8, lsl #2] --> loads e.g. 0xfeb73bd8
1276	// * add x8, x8, x12 --> = cde278, next block
1277	// br x8
1278	// cde278:
1279	//
1280	// Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:0
1281	return false;
1282	}
1283	if (DefAdd->getOpcode() != AArch64::ADDXrx)
1284	return false;
1285
1286	// Validate ADD operands
1287	int64_t OperandExtension = DefAdd->getOperand(i: 3).getImm();
1288	unsigned ShiftVal = AArch64_AM::getArithShiftValue(Imm: OperandExtension);
1289	AArch64_AM::ShiftExtendType ExtendType =
1290	AArch64_AM::getArithExtendType(Imm: OperandExtension);
1291	if (ShiftVal != 2) {
1292	// TODO: Handle the patten where ShiftVal != 2.
1293	// The following code sequence below has no shift amount,
1294	// the range could be 0 to 4.
1295	// The pattern comes from libc, it occurs when the binary is static.
1296	// adr x6, 0x219fb0 <sigall_set+0x88>
1297	// add x6, x6, x14, lsl #2
1298	// ldr w7, [x6]
1299	// add x6, x6, w7, sxtw => no shift amount
1300	// br x6
1301	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: "
1302	"failed to match indirect branch: ShiftVAL != 2\n");
1303	return false;
1304	}
1305
1306	if (ExtendType == AArch64_AM::SXTB)
1307	ScaleValue = 1LL;
1308	else if (ExtendType == AArch64_AM::SXTH)
1309	ScaleValue = 2LL;
1310	else if (ExtendType == AArch64_AM::SXTW)
1311	ScaleValue = 4LL;
1312	else
1313	return false;
1314
1315	// Match an ADR to load base address to be used when addressing JT targets
1316	SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd];
1317	if (UsesAdd.size() <= 1 || UsesAdd[1] == nullptr || UsesAdd[2] == nullptr) {
1318	// This happens when we don't have enough context about this jump table
1319	// because the jumping code sequence was split in multiple basic blocks.
1320	// This was observed in the wild in HHVM code (dispatchImpl).
1321	return false;
1322	}
1323	MCInst *DefBaseAddr = UsesAdd[1];
1324	if (DefBaseAddr->getOpcode() != AArch64::ADR)
1325	return false;
1326
1327	PCRelBase = DefBaseAddr;
1328	// Match LOAD to load the jump table (relative) target
1329	const MCInst *DefLoad = UsesAdd[2];
1330	if (!mayLoad(Inst: *DefLoad) || (ScaleValue == 1LL && !isLDRB(Inst: *DefLoad)) ||
1331	(ScaleValue == 2LL && !isLDRH(Inst: *DefLoad)))
1332	return false;
1333
1334	// Match ADD that calculates the JumpTable Base Address (not the offset)
1335	SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];
1336	const MCInst *DefJTBaseAdd = UsesLoad[1];
1337	MCPhysReg From, To;
1338	if (DefJTBaseAdd == nullptr || isLoadFromStack(Inst: *DefJTBaseAdd) ||
1339	isRegToRegMove(Inst: *DefJTBaseAdd, From, To)) {
1340	// Sometimes base address may have been defined in another basic block
1341	// (hoisted). Return with no jump table info.
1342	return true;
1343	}
1344
1345	if (DefJTBaseAdd->getOpcode() == AArch64::ADR) {
1346	// TODO: Handle the pattern where there is no adrp/add pair.
1347	// It also occurs when the binary is static.
1348	// adr x13, 0x215a18 <_nl_value_type_LC_COLLATE+0x50>
1349	// ldrh w13, [x13, w12, uxtw #1]
1350	// adr x12, 0x247b30 <__gettextparse+0x5b0>
1351	// add x13, x12, w13, sxth #2
1352	// br x13
1353	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match indirect branch: "
1354	"nop/adr instead of adrp/add\n");
1355	return false;
1356	}
1357
1358	if (DefJTBaseAdd->getOpcode() != AArch64::ADDXri) {
1359	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match jump table base "
1360	"address pattern! (1)\n");
1361	return false;
1362	}
1363
1364	if (DefJTBaseAdd->getOperand(i: 2).isImm())
1365	Offset = DefJTBaseAdd->getOperand(i: 2).getImm();
1366	SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
1367	const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
1368	if (DefJTBasePage == nullptr || isLoadFromStack(Inst: *DefJTBasePage)) {
1369	return true;
1370	}
1371	if (DefJTBasePage->getOpcode() != AArch64::ADRP)
1372	return false;
1373
1374	if (DefJTBasePage->getOperand(i: 1).isExpr())
1375	JumpTable = DefJTBasePage->getOperand(i: 1).getExpr();
1376	return true;
1377	}
1378
1379	DenseMap<const MCInst *, SmallVector<MCInst *, 4>>
1380	computeLocalUDChain(const MCInst *CurInstr, InstructionIterator Begin,
1381	InstructionIterator End) const {
1382	DenseMap<int, MCInst *> RegAliasTable;
1383	DenseMap<const MCInst *, SmallVector<MCInst *, 4>> Uses;
1384
1385	auto addInstrOperands = [&](const MCInst &Instr) {
1386	// Update Uses table
1387	for (const MCOperand &Operand : MCPlus::primeOperands(Inst: Instr)) {
1388	if (!Operand.isReg())
1389	continue;
1390	unsigned Reg = Operand.getReg();
1391	MCInst *AliasInst = RegAliasTable[Reg];
1392	Uses[&Instr].push_back(Elt: AliasInst);
1393	LLVM_DEBUG({
1394	dbgs() << "Adding reg operand " << Reg << " refs ";
1395	if (AliasInst != nullptr)
1396	AliasInst->dump();
1397	else
1398	dbgs() << "\n";
1399	});
1400	}
1401	};
1402
1403	LLVM_DEBUG(dbgs() << "computeLocalUDChain\n");
1404	bool TerminatorSeen = false;
1405	for (auto II = Begin; II != End; ++II) {
1406	MCInst &Instr = *II;
1407	// Ignore nops and CFIs
1408	if (isPseudo(Inst: Instr) || isNoop(Inst: Instr))
1409	continue;
1410	if (TerminatorSeen) {
1411	RegAliasTable.clear();
1412	Uses.clear();
1413	}
1414
1415	LLVM_DEBUG(dbgs() << "Now updating for:\n ");
1416	LLVM_DEBUG(Instr.dump());
1417	addInstrOperands(Instr);
1418
1419	BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
1420	getWrittenRegs(Inst: Instr, Regs);
1421
1422	// Update register definitions after this point
1423	for (int Idx : Regs.set_bits()) {
1424	RegAliasTable[Idx] = &Instr;
1425	LLVM_DEBUG(dbgs() << "Setting reg " << Idx
1426	<< " def to current instr.\n");
1427	}
1428
1429	TerminatorSeen = isTerminator(Inst: Instr);
1430	}
1431
1432	// Process the last instruction, which is not currently added into the
1433	// instruction stream
1434	if (CurInstr)
1435	addInstrOperands(*CurInstr);
1436
1437	return Uses;
1438	}
1439
1440	IndirectBranchType
1441	analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
1442	InstructionIterator End, const unsigned PtrSize,
1443	MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
1444	unsigned &IndexRegNumOut, int64_t &DispValueOut,
1445	const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
1446	MCInst *&FixedEntryLoadInstr) const override {
1447	MemLocInstrOut = nullptr;
1448	BaseRegNumOut = AArch64::NoRegister;
1449	IndexRegNumOut = AArch64::NoRegister;
1450	DispValueOut = 0;
1451	DispExprOut = nullptr;
1452	FixedEntryLoadInstr = nullptr;
1453
1454	// An instruction referencing memory used by jump instruction (directly or
1455	// via register). This location could be an array of function pointers
1456	// in case of indirect tail call, or a jump table.
1457	MCInst *MemLocInstr = nullptr;
1458
1459	// Analyze the memory location.
1460	int64_t ScaleValue, DispValue;
1461	const MCExpr *DispExpr;
1462
1463	DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
1464	computeLocalUDChain(CurInstr: &Instruction, Begin, End);
1465	MCInst *PCRelBase;
1466	if (!analyzeIndirectBranchFragment(Inst: Instruction, UDChain, JumpTable&: DispExpr,
1467	Offset&: DispValue, ScaleValue, PCRelBase))
1468	return IndirectBranchType::UNKNOWN;
1469
1470	MemLocInstrOut = MemLocInstr;
1471	DispValueOut = DispValue;
1472	DispExprOut = DispExpr;
1473	PCRelBaseOut = PCRelBase;
1474	return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE;
1475	}
1476
1477	/// Matches PLT entry pattern and returns the associated GOT entry address.
1478	/// Typical PLT entry looks like the following:
1479	///
1480	/// adrp x16, 230000
1481	/// ldr x17, [x16, #3040]
1482	/// add x16, x16, #0xbe0
1483	/// br x17
1484	///
1485	/// The other type of trampolines are located in .plt.got, that are used for
1486	/// non-lazy bindings so doesn't use x16 arg to transfer .got entry address:
1487	///
1488	/// adrp x16, 230000
1489	/// ldr x17, [x16, #3040]
1490	/// br x17
1491	/// nop
1492	///
1493	uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin,
1494	InstructionIterator End,
1495	uint64_t BeginPC) const override {
1496	// Check branch instruction
1497	MCInst *Branch = &Instruction;
1498	assert(Branch->getOpcode() == AArch64::BR && "Unexpected opcode");
1499
1500	DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
1501	computeLocalUDChain(CurInstr: Branch, Begin, End);
1502
1503	// Match ldr instruction
1504	SmallVector<MCInst *, 4> &BranchUses = UDChain[Branch];
1505	if (BranchUses.size() < 1 || BranchUses[0] == nullptr)
1506	return 0;
1507
1508	// Check ldr instruction
1509	const MCInst *Ldr = BranchUses[0];
1510	if (Ldr->getOpcode() != AArch64::LDRXui)
1511	return 0;
1512
1513	// Get ldr value
1514	const unsigned ScaleLdr = 8; // LDRX operates on 8 bytes segments
1515	assert(Ldr->getOperand(2).isImm() && "Unexpected ldr operand");
1516	const uint64_t Offset = Ldr->getOperand(i: 2).getImm() * ScaleLdr;
1517
1518	// Match adrp instruction
1519	SmallVector<MCInst *, 4> &LdrUses = UDChain[Ldr];
1520	if (LdrUses.size() < 2 || LdrUses[1] == nullptr)
1521	return 0;
1522
1523	// Check adrp instruction
1524	MCInst *Adrp = LdrUses[1];
1525	if (Adrp->getOpcode() != AArch64::ADRP)
1526	return 0;
1527
1528	// Get adrp instruction PC
1529	const unsigned InstSize = 4;
1530	uint64_t AdrpPC = BeginPC;
1531	for (InstructionIterator It = Begin; It != End; ++It) {
1532	if (&(*It) == Adrp)
1533	break;
1534	AdrpPC += InstSize;
1535	}
1536
1537	// Get adrp value
1538	uint64_t Base;
1539	assert(Adrp->getOperand(1).isImm() && "Unexpected adrp operand");
1540	bool Ret = evaluateMemOperandTarget(Inst: *Adrp, Target&: Base, Address: AdrpPC, Size: InstSize);
1541	assert(Ret && "Failed to evaluate adrp");
1542	(void)Ret;
1543
1544	return Base + Offset;
1545	}
1546
1547	unsigned getInvertedBranchOpcode(unsigned Opcode) const {
1548	switch (Opcode) {
1549	default:
1550	llvm_unreachable("Failed to invert branch opcode");
1551	return Opcode;
1552	case AArch64::TBZW: return AArch64::TBNZW;
1553	case AArch64::TBZX: return AArch64::TBNZX;
1554	case AArch64::TBNZW: return AArch64::TBZW;
1555	case AArch64::TBNZX: return AArch64::TBZX;
1556	case AArch64::CBZW: return AArch64::CBNZW;
1557	case AArch64::CBZX: return AArch64::CBNZX;
1558	case AArch64::CBNZW: return AArch64::CBZW;
1559	case AArch64::CBNZX: return AArch64::CBZX;
1560	}
1561	}
1562
1563	unsigned getCondCode(const MCInst &Inst) const override {
1564	// AArch64 does not use conditional codes, so we just return the opcode
1565	// of the conditional branch here.
1566	return Inst.getOpcode();
1567	}
1568
1569	unsigned getCanonicalBranchCondCode(unsigned Opcode) const override {
1570	switch (Opcode) {
1571	default:
1572	return Opcode;
1573	case AArch64::TBNZW: return AArch64::TBZW;
1574	case AArch64::TBNZX: return AArch64::TBZX;
1575	case AArch64::CBNZW: return AArch64::CBZW;
1576	case AArch64::CBNZX: return AArch64::CBZX;
1577	}
1578	}
1579
1580	void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
1581	MCContext *Ctx) const override {
1582	if (isTB(Inst) || isCB(Inst)) {
1583	Inst.setOpcode(getInvertedBranchOpcode(Opcode: Inst.getOpcode()));
1584	assert(Inst.getOpcode() != 0 && "Invalid branch instruction");
1585	} else if (Inst.getOpcode() == AArch64::Bcc) {
1586	Inst.getOperand(i: 0).setImm(AArch64CC::getInvertedCondCode(
1587	Code: static_cast<AArch64CC::CondCode>(Inst.getOperand(i: 0).getImm())));
1588	assert(Inst.getOperand(0).getImm() != AArch64CC::AL &&
1589	Inst.getOperand(0).getImm() != AArch64CC::NV &&
1590	"Can't reverse ALWAYS cond code");
1591	} else {
1592	LLVM_DEBUG(Inst.dump());
1593	llvm_unreachable("Unrecognized branch instruction");
1594	}
1595	replaceBranchTarget(Inst, TBB, Ctx);
1596	}
1597
1598	int getPCRelEncodingSize(const MCInst &Inst) const override {
1599	switch (Inst.getOpcode()) {
1600	default:
1601	llvm_unreachable("Failed to get pcrel encoding size");
1602	return 0;
1603	case AArch64::TBZW: return 16;
1604	case AArch64::TBZX: return 16;
1605	case AArch64::TBNZW: return 16;
1606	case AArch64::TBNZX: return 16;
1607	case AArch64::CBZW: return 21;
1608	case AArch64::CBZX: return 21;
1609	case AArch64::CBNZW: return 21;
1610	case AArch64::CBNZX: return 21;
1611	case AArch64::B: return 28;
1612	case AArch64::BL: return 28;
1613	case AArch64::Bcc: return 21;
1614	}
1615	}
1616
1617	int getShortJmpEncodingSize() const override { return 33; }
1618
1619	int getUncondBranchEncodingSize() const override { return 28; }
1620
1621	// This helper function creates the snippet of code that compares a register
1622	// RegNo with an immedaite Imm, and jumps to Target if they are equal.
1623	// cmp RegNo, #Imm
1624	// b.eq Target
1625	// where cmp is an alias for subs, which results in the code below:
1626	// subs xzr, RegNo, #Imm
1627	// b.eq Target.
1628	InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
1629	const MCSymbol *Target,
1630	MCContext *Ctx) const override {
1631	InstructionListType Code;
1632	Code.emplace_back(MCInstBuilder(AArch64::SUBSXri)
1633	.addReg(AArch64::XZR)
1634	.addReg(RegNo)
1635	.addImm(Imm)
1636	.addImm(0));
1637	Code.emplace_back(MCInstBuilder(AArch64::Bcc)
1638	.addImm(AArch64CC::EQ)
1639	.addExpr(MCSymbolRefExpr::create(
1640	Target, MCSymbolRefExpr::VK_None, *Ctx)));
1641	return Code;
1642	}
1643
1644	// This helper function creates the snippet of code that compares a register
1645	// RegNo with an immedaite Imm, and jumps to Target if they are not equal.
1646	// cmp RegNo, #Imm
1647	// b.ne Target
1648	// where cmp is an alias for subs, which results in the code below:
1649	// subs xzr, RegNo, #Imm
1650	// b.ne Target.
1651	InstructionListType createCmpJNE(MCPhysReg RegNo, int64_t Imm,
1652	const MCSymbol *Target,
1653	MCContext *Ctx) const override {
1654	InstructionListType Code;
1655	Code.emplace_back(MCInstBuilder(AArch64::SUBSXri)
1656	.addReg(AArch64::XZR)
1657	.addReg(RegNo)
1658	.addImm(Imm)
1659	.addImm(0));
1660	Code.emplace_back(MCInstBuilder(AArch64::Bcc)
1661	.addImm(AArch64CC::NE)
1662	.addExpr(MCSymbolRefExpr::create(
1663	Target, MCSymbolRefExpr::VK_None, *Ctx)));
1664	return Code;
1665	}
1666
1667	void createTailCall(MCInst &Inst, const MCSymbol *Target,
1668	MCContext *Ctx) override {
1669	return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
1670	}
1671
1672	void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
1673	MCContext *Ctx) override {
1674	createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true);
1675	}
1676
1677	void createTrap(MCInst &Inst) const override {
1678	Inst.clear();
1679	Inst.setOpcode(AArch64::BRK);
1680	Inst.addOperand(Op: MCOperand::createImm(Val: 1));
1681	}
1682
1683	bool convertJmpToTailCall(MCInst &Inst) override {
1684	setTailCall(Inst);
1685	return true;
1686	}
1687
1688	bool convertTailCallToJmp(MCInst &Inst) override {
1689	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1690	clearOffset(Inst);
1691	if (getConditionalTailCall(Inst))
1692	unsetConditionalTailCall(Inst);
1693	return true;
1694	}
1695
1696	InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
1697	const MCSymbol *TargetLocation,
1698	MCContext *Ctx) override {
1699	const bool IsTailCall = isTailCall(Inst: DirectCall);
1700	assert((DirectCall.getOpcode() == AArch64::BL ||
1701	(DirectCall.getOpcode() == AArch64::B && IsTailCall)) &&
1702	"64-bit direct (tail) call instruction expected");
1703
1704	InstructionListType Code;
1705	// Code sequence for indirect plt call:
1706	// adrp x16 <symbol>
1707	// ldr x17, [x16, #<offset>]
1708	// blr x17 ; or 'br' for tail calls
1709
1710	MCInst InstAdrp;
1711	InstAdrp.setOpcode(AArch64::ADRP);
1712	InstAdrp.addOperand(MCOperand::createReg(AArch64::X16));
1713	InstAdrp.addOperand(Op: MCOperand::createImm(Val: 0));
1714	setOperandToSymbolRef(Inst&: InstAdrp, /* OpNum */ 1, Symbol: TargetLocation,
1715	/* Addend */ 0, Ctx, RelType: ELF::R_AARCH64_ADR_GOT_PAGE);
1716	Code.emplace_back(args&: InstAdrp);
1717
1718	MCInst InstLoad;
1719	InstLoad.setOpcode(AArch64::LDRXui);
1720	InstLoad.addOperand(MCOperand::createReg(AArch64::X17));
1721	InstLoad.addOperand(MCOperand::createReg(AArch64::X16));
1722	InstLoad.addOperand(Op: MCOperand::createImm(Val: 0));
1723	setOperandToSymbolRef(Inst&: InstLoad, /* OpNum */ 2, Symbol: TargetLocation,
1724	/* Addend */ 0, Ctx, RelType: ELF::R_AARCH64_LD64_GOT_LO12_NC);
1725	Code.emplace_back(args&: InstLoad);
1726
1727	MCInst InstCall;
1728	InstCall.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);
1729	InstCall.addOperand(MCOperand::createReg(AArch64::X17));
1730	moveAnnotations(SrcInst: std::move(DirectCall), DstInst&: InstCall);
1731	Code.emplace_back(args&: InstCall);
1732
1733	return Code;
1734	}
1735
1736	bool lowerTailCall(MCInst &Inst) override {
1737	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1738	if (getConditionalTailCall(Inst))
1739	unsetConditionalTailCall(Inst);
1740	return true;
1741	}
1742
1743	bool isNoop(const MCInst &Inst) const override {
1744	return Inst.getOpcode() == AArch64::HINT &&
1745	Inst.getOperand(0).getImm() == 0;
1746	}
1747
1748	void createNoop(MCInst &Inst) const override {
1749	Inst.setOpcode(AArch64::HINT);
1750	Inst.clear();
1751	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
1752	}
1753
1754	bool isStorePair(const MCInst &Inst) const {
1755	const unsigned opcode = Inst.getOpcode();
1756
1757	auto isStorePairImmOffset = [&]() {
1758	switch (opcode) {
1759	case AArch64::STPWi:
1760	case AArch64::STPXi:
1761	case AArch64::STPSi:
1762	case AArch64::STPDi:
1763	case AArch64::STPQi:
1764	case AArch64::STNPWi:
1765	case AArch64::STNPXi:
1766	case AArch64::STNPSi:
1767	case AArch64::STNPDi:
1768	case AArch64::STNPQi:
1769	return true;
1770	default:
1771	break;
1772	}
1773
1774	return false;
1775	};
1776
1777	auto isStorePairPostIndex = [&]() {
1778	switch (opcode) {
1779	case AArch64::STPWpost:
1780	case AArch64::STPXpost:
1781	case AArch64::STPSpost:
1782	case AArch64::STPDpost:
1783	case AArch64::STPQpost:
1784	return true;
1785	default:
1786	break;
1787	}
1788
1789	return false;
1790	};
1791
1792	auto isStorePairPreIndex = [&]() {
1793	switch (opcode) {
1794	case AArch64::STPWpre:
1795	case AArch64::STPXpre:
1796	case AArch64::STPSpre:
1797	case AArch64::STPDpre:
1798	case AArch64::STPQpre:
1799	return true;
1800	default:
1801	break;
1802	}
1803
1804	return false;
1805	};
1806
1807	return isStorePairImmOffset() || isStorePairPostIndex() ||
1808	isStorePairPreIndex();
1809	}
1810
1811	bool isStoreReg(const MCInst &Inst) const {
1812	const unsigned opcode = Inst.getOpcode();
1813
1814	auto isStoreRegUnscaleImm = [&]() {
1815	switch (opcode) {
1816	case AArch64::STURBi:
1817	case AArch64::STURBBi:
1818	case AArch64::STURHi:
1819	case AArch64::STURHHi:
1820	case AArch64::STURWi:
1821	case AArch64::STURXi:
1822	case AArch64::STURSi:
1823	case AArch64::STURDi:
1824	case AArch64::STURQi:
1825	return true;
1826	default:
1827	break;
1828	}
1829
1830	return false;
1831	};
1832
1833	auto isStoreRegScaledImm = [&]() {
1834	switch (opcode) {
1835	case AArch64::STRBui:
1836	case AArch64::STRBBui:
1837	case AArch64::STRHui:
1838	case AArch64::STRHHui:
1839	case AArch64::STRWui:
1840	case AArch64::STRXui:
1841	case AArch64::STRSui:
1842	case AArch64::STRDui:
1843	case AArch64::STRQui:
1844	return true;
1845	default:
1846	break;
1847	}
1848
1849	return false;
1850	};
1851
1852	auto isStoreRegImmPostIndexed = [&]() {
1853	switch (opcode) {
1854	case AArch64::STRBpost:
1855	case AArch64::STRBBpost:
1856	case AArch64::STRHpost:
1857	case AArch64::STRHHpost:
1858	case AArch64::STRWpost:
1859	case AArch64::STRXpost:
1860	case AArch64::STRSpost:
1861	case AArch64::STRDpost:
1862	case AArch64::STRQpost:
1863	return true;
1864	default:
1865	break;
1866	}
1867
1868	return false;
1869	};
1870
1871	auto isStoreRegImmPreIndexed = [&]() {
1872	switch (opcode) {
1873	case AArch64::STRBpre:
1874	case AArch64::STRBBpre:
1875	case AArch64::STRHpre:
1876	case AArch64::STRHHpre:
1877	case AArch64::STRWpre:
1878	case AArch64::STRXpre:
1879	case AArch64::STRSpre:
1880	case AArch64::STRDpre:
1881	case AArch64::STRQpre:
1882	return true;
1883	default:
1884	break;
1885	}
1886
1887	return false;
1888	};
1889
1890	auto isStoreRegUnscaleUnpriv = [&]() {
1891	switch (opcode) {
1892	case AArch64::STTRBi:
1893	case AArch64::STTRHi:
1894	case AArch64::STTRWi:
1895	case AArch64::STTRXi:
1896	return true;
1897	default:
1898	break;
1899	}
1900
1901	return false;
1902	};
1903
1904	auto isStoreRegTrunc = [&]() {
1905	switch (opcode) {
1906	case AArch64::STRBBroW:
1907	case AArch64::STRBBroX:
1908	case AArch64::STRBroW:
1909	case AArch64::STRBroX:
1910	case AArch64::STRDroW:
1911	case AArch64::STRDroX:
1912	case AArch64::STRHHroW:
1913	case AArch64::STRHHroX:
1914	case AArch64::STRHroW:
1915	case AArch64::STRHroX:
1916	case AArch64::STRQroW:
1917	case AArch64::STRQroX:
1918	case AArch64::STRSroW:
1919	case AArch64::STRSroX:
1920	case AArch64::STRWroW:
1921	case AArch64::STRWroX:
1922	case AArch64::STRXroW:
1923	case AArch64::STRXroX:
1924	return true;
1925	default:
1926	break;
1927	}
1928
1929	return false;
1930	};
1931
1932	return isStoreRegUnscaleImm() || isStoreRegScaledImm() ||
1933	isStoreRegImmPreIndexed() || isStoreRegImmPostIndexed() ||
1934	isStoreRegUnscaleUnpriv() || isStoreRegTrunc();
1935	}
1936
1937	bool mayStore(const MCInst &Inst) const override {
1938	return isStorePair(Inst) || isStoreReg(Inst) ||
1939	isAArch64ExclusiveStore(Inst);
1940	}
1941
1942	bool isStoreToStack(const MCInst &Inst) const {
1943	if (!mayStore(Inst))
1944	return false;
1945
1946	for (const MCOperand &Operand : useOperands(Inst)) {
1947	if (!Operand.isReg())
1948	continue;
1949
1950	unsigned Reg = Operand.getReg();
1951	if (Reg == AArch64::SP || Reg == AArch64::WSP)
1952	return true;
1953	}
1954
1955	return false;
1956	}
1957
1958	void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
1959	bool IsTailCall) override {
1960	Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL);
1961	Inst.clear();
1962	Inst.addOperand(Op: MCOperand::createExpr(Val: getTargetExprFor(
1963	Inst, Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
1964	Ctx&: *Ctx, RelType: 0)));
1965	if (IsTailCall)
1966	convertJmpToTailCall(Inst);
1967	}
1968
1969	bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1970	const MCSymbol *&TBB, const MCSymbol *&FBB,
1971	MCInst *&CondBranch,
1972	MCInst *&UncondBranch) const override {
1973	auto I = End;
1974
1975	while (I != Begin) {
1976	--I;
1977
1978	// Ignore nops and CFIs
1979	if (isPseudo(Inst: *I) || isNoop(Inst: *I))
1980	continue;
1981
1982	// Stop when we find the first non-terminator
1983	if (!isTerminator(Inst: *I) || isTailCall(Inst: *I) || !isBranch(Inst: *I))
1984	break;
1985
1986	// Handle unconditional branches.
1987	if (isUnconditionalBranch(Inst: *I)) {
1988	// If any code was seen after this unconditional branch, we've seen
1989	// unreachable code. Ignore them.
1990	CondBranch = nullptr;
1991	UncondBranch = &*I;
1992	const MCSymbol *Sym = getTargetSymbol(Inst: *I);
1993	assert(Sym != nullptr &&
1994	"Couldn't extract BB symbol from jump operand");
1995	TBB = Sym;
1996	continue;
1997	}
1998
1999	// Handle conditional branches and ignore indirect branches
2000	if (isIndirectBranch(Inst: *I))
2001	return false;
2002
2003	if (CondBranch == nullptr) {
2004	const MCSymbol *TargetBB = getTargetSymbol(Inst: *I);
2005	if (TargetBB == nullptr) {
2006	// Unrecognized branch target
2007	return false;
2008	}
2009	FBB = TBB;
2010	TBB = TargetBB;
2011	CondBranch = &*I;
2012	continue;
2013	}
2014
2015	llvm_unreachable("multiple conditional branches in one BB");
2016	}
2017	return true;
2018	}
2019
2020	void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,
2021	MCContext *Ctx, bool IsTailCall) override {
2022	// ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
2023	// Standard for the ARM 64-bit Architecture (AArch64)".
2024	// The sequence of instructions we create here is the following:
2025	// movz ip0, #:abs_g3:<addr>
2026	// movk ip0, #:abs_g2_nc:<addr>
2027	// movk ip0, #:abs_g1_nc:<addr>
2028	// movk ip0, #:abs_g0_nc:<addr>
2029	// br ip0
2030	MCInst Inst;
2031	Inst.setOpcode(AArch64::MOVZXi);
2032	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2033	Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
2034	Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
2035	AArch64MCExpr::VK_ABS_G3, Ctx&: *Ctx)));
2036	Inst.addOperand(Op: MCOperand::createImm(Val: 0x30));
2037	Seq.emplace_back(args&: Inst);
2038
2039	Inst.clear();
2040	Inst.setOpcode(AArch64::MOVKXi);
2041	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2042	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2043	Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
2044	Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
2045	AArch64MCExpr::VK_ABS_G2_NC, Ctx&: *Ctx)));
2046	Inst.addOperand(Op: MCOperand::createImm(Val: 0x20));
2047	Seq.emplace_back(args&: Inst);
2048
2049	Inst.clear();
2050	Inst.setOpcode(AArch64::MOVKXi);
2051	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2052	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2053	Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
2054	Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
2055	AArch64MCExpr::VK_ABS_G1_NC, Ctx&: *Ctx)));
2056	Inst.addOperand(Op: MCOperand::createImm(Val: 0x10));
2057	Seq.emplace_back(args&: Inst);
2058
2059	Inst.clear();
2060	Inst.setOpcode(AArch64::MOVKXi);
2061	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2062	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2063	Inst.addOperand(Op: MCOperand::createExpr(Val: AArch64MCExpr::create(
2064	Expr: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
2065	AArch64MCExpr::VK_ABS_G0_NC, Ctx&: *Ctx)));
2066	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
2067	Seq.emplace_back(args&: Inst);
2068
2069	Inst.clear();
2070	Inst.setOpcode(AArch64::BR);
2071	Inst.addOperand(MCOperand::createReg(AArch64::X16));
2072	if (IsTailCall)
2073	setTailCall(Inst);
2074	Seq.emplace_back(args&: Inst);
2075	}
2076
2077	void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
2078	MCContext *Ctx, bool IsTailCall) override {
2079	// ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
2080	// Standard for the ARM 64-bit Architecture (AArch64)".
2081	// The sequence of instructions we create here is the following:
2082	// adrp ip0, imm
2083	// add ip0, ip0, imm
2084	// br ip0
2085	MCPhysReg Reg = AArch64::X16;
2086	InstructionListType Insts = materializeAddress(Target, Ctx, RegName: Reg);
2087	Insts.emplace_back();
2088	MCInst &Inst = Insts.back();
2089	Inst.clear();
2090	Inst.setOpcode(AArch64::BR);
2091	Inst.addOperand(Op: MCOperand::createReg(Reg));
2092	if (IsTailCall)
2093	setTailCall(Inst);
2094	Seq.swap(x&: Insts);
2095	}
2096
2097	/// Matching pattern here is
2098	///
2099	/// ADRP x16, imm
2100	/// ADD x16, x16, imm
2101	/// BR x16
2102	///
2103	uint64_t matchLinkerVeneer(InstructionIterator Begin, InstructionIterator End,
2104	uint64_t Address, const MCInst &CurInst,
2105	MCInst *&TargetHiBits, MCInst *&TargetLowBits,
2106	uint64_t &Target) const override {
2107	if (CurInst.getOpcode() != AArch64::BR || !CurInst.getOperand(0).isReg() ||
2108	CurInst.getOperand(0).getReg() != AArch64::X16)
2109	return 0;
2110
2111	auto I = End;
2112	if (I == Begin)
2113	return 0;
2114
2115	--I;
2116	Address -= 4;
2117	if (I == Begin || I->getOpcode() != AArch64::ADDXri ||
2118	MCPlus::getNumPrimeOperands(*I) < 3 || !I->getOperand(0).isReg() ||
2119	!I->getOperand(1).isReg() ||
2120	I->getOperand(0).getReg() != AArch64::X16 ||
2121	I->getOperand(1).getReg() != AArch64::X16 || !I->getOperand(2).isImm())
2122	return 0;
2123	TargetLowBits = &*I;
2124	uint64_t Addr = I->getOperand(i: 2).getImm() & 0xFFF;
2125
2126	--I;
2127	Address -= 4;
2128	if (I->getOpcode() != AArch64::ADRP ||
2129	MCPlus::getNumPrimeOperands(*I) < 2 || !I->getOperand(0).isReg() ||
2130	!I->getOperand(1).isImm() || I->getOperand(0).getReg() != AArch64::X16)
2131	return 0;
2132	TargetHiBits = &*I;
2133	Addr |= (Address + ((int64_t)I->getOperand(i: 1).getImm() << 12)) &
2134	0xFFFFFFFFFFFFF000ULL;
2135	Target = Addr;
2136	return 3;
2137	}
2138
2139	/// Match the following pattern:
2140	///
2141	/// LDR x16, .L1
2142	/// BR x16
2143	/// L1:
2144	/// .quad Target
2145	///
2146	/// Populate \p TargetAddress with the Target value on successful match.
2147	bool matchAbsLongVeneer(const BinaryFunction &BF,
2148	uint64_t &TargetAddress) const override {
2149	if (BF.size() != 1 || BF.getMaxSize() < 16)
2150	return false;
2151
2152	if (!BF.hasConstantIsland())
2153	return false;
2154
2155	const BinaryBasicBlock &BB = BF.front();
2156	if (BB.size() != 2)
2157	return false;
2158
2159	const MCInst &LDRInst = BB.getInstructionAtIndex(Index: 0);
2160	if (LDRInst.getOpcode() != AArch64::LDRXl)
2161	return false;
2162
2163	if (!LDRInst.getOperand(0).isReg() ||
2164	LDRInst.getOperand(0).getReg() != AArch64::X16)
2165	return false;
2166
2167	const MCSymbol *TargetSym = getTargetSymbol(Inst: LDRInst, OpNum: 1);
2168	if (!TargetSym)
2169	return false;
2170
2171	const MCInst &BRInst = BB.getInstructionAtIndex(Index: 1);
2172	if (BRInst.getOpcode() != AArch64::BR)
2173	return false;
2174	if (!BRInst.getOperand(0).isReg() ||
2175	BRInst.getOperand(0).getReg() != AArch64::X16)
2176	return false;
2177
2178	const BinaryFunction::IslandInfo &IInfo = BF.getIslandInfo();
2179	if (IInfo.HasDynamicRelocations)
2180	return false;
2181
2182	auto Iter = IInfo.Offsets.find(x: 8);
2183	if (Iter == IInfo.Offsets.end() || Iter->second != TargetSym)
2184	return false;
2185
2186	// Extract the absolute value stored inside the island.
2187	StringRef SectionContents = BF.getOriginSection()->getContents();
2188	StringRef FunctionContents = SectionContents.substr(
2189	Start: BF.getAddress() - BF.getOriginSection()->getAddress(), N: BF.getMaxSize());
2190
2191	const BinaryContext &BC = BF.getBinaryContext();
2192	DataExtractor DE(FunctionContents, BC.AsmInfo->isLittleEndian(),
2193	BC.AsmInfo->getCodePointerSize());
2194	uint64_t Offset = 8;
2195	TargetAddress = DE.getAddress(offset_ptr: &Offset);
2196
2197	return true;
2198	}
2199
2200	bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const override {
2201	if (!isADRP(Inst: Adrp) || !isAddXri(Inst: Add))
2202	return false;
2203
2204	assert(Adrp.getOperand(0).isReg() &&
2205	"Unexpected operand in ADRP instruction");
2206	MCPhysReg AdrpReg = Adrp.getOperand(i: 0).getReg();
2207	assert(Add.getOperand(1).isReg() &&
2208	"Unexpected operand in ADDXri instruction");
2209	MCPhysReg AddReg = Add.getOperand(i: 1).getReg();
2210	return AdrpReg == AddReg;
2211	}
2212
2213	bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2214	int64_t Addend, MCContext *Ctx, int64_t &Value,
2215	uint32_t RelType) const override {
2216	unsigned ImmOpNo = -1U;
2217	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2218	++Index) {
2219	if (Inst.getOperand(i: Index).isImm()) {
2220	ImmOpNo = Index;
2221	break;
2222	}
2223	}
2224	if (ImmOpNo == -1U)
2225	return false;
2226
2227	Value = Inst.getOperand(i: ImmOpNo).getImm();
2228
2229	setOperandToSymbolRef(Inst, OpNum: ImmOpNo, Symbol, Addend, Ctx, RelType);
2230
2231	return true;
2232	}
2233
2234	void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2235	MCContext *Ctx) const override {
2236	Inst.setOpcode(AArch64::B);
2237	Inst.clear();
2238	Inst.addOperand(Op: MCOperand::createExpr(Val: getTargetExprFor(
2239	Inst, Expr: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx),
2240	Ctx&: *Ctx, RelType: 0)));
2241	}
2242
2243	bool shouldRecordCodeRelocation(uint32_t RelType) const override {
2244	switch (RelType) {
2245	case ELF::R_AARCH64_ABS64:
2246	case ELF::R_AARCH64_ABS32:
2247	case ELF::R_AARCH64_ABS16:
2248	case ELF::R_AARCH64_ADD_ABS_LO12_NC:
2249	case ELF::R_AARCH64_ADR_GOT_PAGE:
2250	case ELF::R_AARCH64_ADR_PREL_LO21:
2251	case ELF::R_AARCH64_ADR_PREL_PG_HI21:
2252	case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
2253	case ELF::R_AARCH64_LD64_GOT_LO12_NC:
2254	case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
2255	case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
2256	case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
2257	case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
2258	case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
2259	case ELF::R_AARCH64_TLSDESC_ADD_LO12:
2260	case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
2261	case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
2262	case ELF::R_AARCH64_TLSDESC_LD64_LO12:
2263	case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
2264	case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
2265	case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0:
2266	case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
2267	case ELF::R_AARCH64_MOVW_UABS_G0:
2268	case ELF::R_AARCH64_MOVW_UABS_G0_NC:
2269	case ELF::R_AARCH64_MOVW_UABS_G1:
2270	case ELF::R_AARCH64_MOVW_UABS_G1_NC:
2271	case ELF::R_AARCH64_MOVW_UABS_G2:
2272	case ELF::R_AARCH64_MOVW_UABS_G2_NC:
2273	case ELF::R_AARCH64_MOVW_UABS_G3:
2274	case ELF::R_AARCH64_PREL16:
2275	case ELF::R_AARCH64_PREL32:
2276	case ELF::R_AARCH64_PREL64:
2277	return true;
2278	case ELF::R_AARCH64_CALL26:
2279	case ELF::R_AARCH64_JUMP26:
2280	case ELF::R_AARCH64_TSTBR14:
2281	case ELF::R_AARCH64_CONDBR19:
2282	case ELF::R_AARCH64_TLSDESC_CALL:
2283	case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
2284	case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
2285	return false;
2286	default:
2287	llvm_unreachable("Unexpected AArch64 relocation type in code");
2288	}
2289	}
2290
2291	StringRef getTrapFillValue() const override {
2292	return StringRef("\0\0\0\0", 4);
2293	}
2294
2295	void createReturn(MCInst &Inst) const override {
2296	Inst.setOpcode(AArch64::RET);
2297	Inst.clear();
2298	Inst.addOperand(MCOperand::createReg(AArch64::LR));
2299	}
2300
2301	void createStackPointerIncrement(
2302	MCInst &Inst, int Size,
2303	bool NoFlagsClobber = false /*unused for AArch64*/) const override {
2304	Inst.setOpcode(AArch64::SUBXri);
2305	Inst.clear();
2306	Inst.addOperand(MCOperand::createReg(AArch64::SP));
2307	Inst.addOperand(MCOperand::createReg(AArch64::SP));
2308	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2309	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
2310	}
2311
2312	void createStackPointerDecrement(
2313	MCInst &Inst, int Size,
2314	bool NoFlagsClobber = false /*unused for AArch64*/) const override {
2315	Inst.setOpcode(AArch64::ADDXri);
2316	Inst.clear();
2317	Inst.addOperand(MCOperand::createReg(AArch64::SP));
2318	Inst.addOperand(MCOperand::createReg(AArch64::SP));
2319	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2320	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
2321	}
2322
2323	void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
2324	int64_t Disp) const {
2325	Inst.setOpcode(AArch64::BR);
2326	Inst.clear();
2327	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg));
2328	}
2329
2330	InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
2331	InstructionListType Insts(5);
2332	// Code sequence for instrumented indirect call handler:
2333	// msr nzcv, x1
2334	// ldp x0, x1, [sp], #16
2335	// ldr x16, [sp], #16
2336	// ldp x0, x1, [sp], #16
2337	// br x16
2338	setSystemFlag(Insts[0], AArch64::X1);
2339	createPopRegisters(Insts[1], AArch64::X0, AArch64::X1);
2340	// Here we load address of the next function which should be called in the
2341	// original binary to X16 register. Writing to X16 is permitted without
2342	// needing to restore.
2343	loadReg(Insts[2], AArch64::X16, AArch64::SP);
2344	createPopRegisters(Insts[3], AArch64::X0, AArch64::X1);
2345	createIndirectBranch(Insts[4], AArch64::X16, 0);
2346	return Insts;
2347	}
2348
2349	InstructionListType
2350	createInstrumentedIndTailCallHandlerExitBB() const override {
2351	return createInstrumentedIndCallHandlerExitBB();
2352	}
2353
2354	InstructionListType createGetter(MCContext *Ctx, const char *name) const {
2355	InstructionListType Insts(4);
2356	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: name);
2357	InstructionListType Addr = materializeAddress(Locs, Ctx, AArch64::X0);
2358	std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.begin());
2359	assert(Addr.size() == 2 && "Invalid Addr size");
2360	loadReg(Insts[2], AArch64::X0, AArch64::X0);
2361	createReturn(Inst&: Insts[3]);
2362	return Insts;
2363	}
2364
2365	InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
2366	return createGetter(Ctx, name: "__bolt_num_counters");
2367	}
2368
2369	InstructionListType
2370	createInstrLocationsGetter(MCContext *Ctx) const override {
2371	return createGetter(Ctx, name: "__bolt_instr_locations");
2372	}
2373
2374	InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
2375	return createGetter(Ctx, name: "__bolt_instr_tables");
2376	}
2377
2378	InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
2379	return createGetter(Ctx, name: "__bolt_instr_num_funcs");
2380	}
2381
2382	void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
2383	bool IsTailCall = isTailCall(Inst);
2384	if (IsTailCall)
2385	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
2386	if (Inst.getOpcode() == AArch64::BR || Inst.getOpcode() == AArch64::BLR) {
2387	Inst.setOpcode(AArch64::ORRXrs);
2388	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
2389	Inst.insert(Inst.begin() + 1, MCOperand::createReg(AArch64::XZR));
2390	Inst.insert(I: Inst.begin() + 3, Op: MCOperand::createImm(Val: 0));
2391	return;
2392	}
2393	llvm_unreachable("not implemented");
2394	}
2395
2396	InstructionListType createLoadImmediate(const MCPhysReg Dest,
2397	uint64_t Imm) const override {
2398	InstructionListType Insts(4);
2399	int Shift = 48;
2400	for (int I = 0; I < 4; I++, Shift -= 16) {
2401	Insts[I].setOpcode(AArch64::MOVKXi);
2402	Insts[I].addOperand(Op: MCOperand::createReg(Reg: Dest));
2403	Insts[I].addOperand(Op: MCOperand::createReg(Reg: Dest));
2404	Insts[I].addOperand(Op: MCOperand::createImm(Val: (Imm >> Shift) & 0xFFFF));
2405	Insts[I].addOperand(Op: MCOperand::createImm(Val: Shift));
2406	}
2407	return Insts;
2408	}
2409
2410	void createIndirectCallInst(MCInst &Inst, bool IsTailCall,
2411	MCPhysReg Reg) const {
2412	Inst.clear();
2413	Inst.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);
2414	Inst.addOperand(Op: MCOperand::createReg(Reg));
2415	}
2416
2417	InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
2418	MCSymbol *HandlerFuncAddr,
2419	int CallSiteID,
2420	MCContext *Ctx) override {
2421	InstructionListType Insts;
2422	// Code sequence used to enter indirect call instrumentation helper:
2423	// stp x0, x1, [sp, #-16]! createPushRegisters
2424	// mov target x0 convertIndirectCallToLoad -> orr x0 target xzr
2425	// mov x1 CallSiteID createLoadImmediate ->
2426	// movk x1, #0x0, lsl #48
2427	// movk x1, #0x0, lsl #32
2428	// movk x1, #0x0, lsl #16
2429	// movk x1, #0x0
2430	// stp x0, x1, [sp, #-16]!
2431	// bl *HandlerFuncAddr createIndirectCall ->
2432	// adr x0 *HandlerFuncAddr -> adrp + add
2433	// blr x0
2434	Insts.emplace_back();
2435	createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1);
2436	Insts.emplace_back(args&: CallInst);
2437	convertIndirectCallToLoad(Insts.back(), AArch64::X0);
2438	InstructionListType LoadImm =
2439	createLoadImmediate(Dest: getIntArgRegister(ArgNo: 1), Imm: CallSiteID);
2440	Insts.insert(position: Insts.end(), first: LoadImm.begin(), last: LoadImm.end());
2441	Insts.emplace_back();
2442	createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1);
2443	Insts.resize(new_size: Insts.size() + 2);
2444	InstructionListType Addr =
2445	materializeAddress(HandlerFuncAddr, Ctx, AArch64::X0);
2446	assert(Addr.size() == 2 && "Invalid Addr size");
2447	std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.end() - Addr.size());
2448	Insts.emplace_back();
2449	createIndirectCallInst(Insts.back(), isTailCall(CallInst), AArch64::X0);
2450
2451	// Carry over metadata including tail call marker if present.
2452	stripAnnotations(Inst&: Insts.back());
2453	moveAnnotations(SrcInst: std::move(CallInst), DstInst&: Insts.back());
2454
2455	return Insts;
2456	}
2457
2458	InstructionListType
2459	createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
2460	const MCSymbol *IndCallHandler,
2461	MCContext *Ctx) override {
2462	// Code sequence used to check whether InstrTampoline was initialized
2463	// and call it if so, returns via IndCallHandler
2464	// stp x0, x1, [sp, #-16]!
2465	// mrs x1, nzcv
2466	// adr x0, InstrTrampoline -> adrp + add
2467	// ldr x0, [x0]
2468	// subs x0, x0, #0x0
2469	// b.eq IndCallHandler
2470	// str x30, [sp, #-16]!
2471	// blr x0
2472	// ldr x30, [sp], #16
2473	// b IndCallHandler
2474	InstructionListType Insts;
2475	Insts.emplace_back();
2476	createPushRegisters(Insts.back(), AArch64::X0, AArch64::X1);
2477	Insts.emplace_back();
2478	getSystemFlag(Inst&: Insts.back(), RegName: getIntArgRegister(ArgNo: 1));
2479	Insts.emplace_back();
2480	Insts.emplace_back();
2481	InstructionListType Addr =
2482	materializeAddress(InstrTrampoline, Ctx, AArch64::X0);
2483	std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.end() - Addr.size());
2484	assert(Addr.size() == 2 && "Invalid Addr size");
2485	Insts.emplace_back();
2486	loadReg(Insts.back(), AArch64::X0, AArch64::X0);
2487	InstructionListType cmpJmp =
2488	createCmpJE(AArch64::X0, 0, IndCallHandler, Ctx);
2489	Insts.insert(position: Insts.end(), first: cmpJmp.begin(), last: cmpJmp.end());
2490	Insts.emplace_back();
2491	storeReg(Insts.back(), AArch64::LR, AArch64::SP);
2492	Insts.emplace_back();
2493	Insts.back().setOpcode(AArch64::BLR);
2494	Insts.back().addOperand(MCOperand::createReg(AArch64::X0));
2495	Insts.emplace_back();
2496	loadReg(Insts.back(), AArch64::LR, AArch64::SP);
2497	Insts.emplace_back();
2498	createDirectCall(Inst&: Insts.back(), Target: IndCallHandler, Ctx, /*IsTailCall*/ true);
2499	return Insts;
2500	}
2501
2502	InstructionListType
2503	createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
2504	unsigned CodePointerSize) const override {
2505	unsigned int I = 0;
2506	InstructionListType Instrs(IsLeaf ? 12 : 10);
2507
2508	if (IsLeaf)
2509	createStackPointerIncrement(Inst&: Instrs[I++], Size: 128);
2510	createPushRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
2511	getSystemFlag(Instrs[I++], AArch64::X1);
2512	InstructionListType Addr = materializeAddress(Target, Ctx, AArch64::X0);
2513	assert(Addr.size() == 2 && "Invalid Addr size");
2514	std::copy(first: Addr.begin(), last: Addr.end(), result: Instrs.begin() + I);
2515	I += Addr.size();
2516	storeReg(Instrs[I++], AArch64::X2, AArch64::SP);
2517	InstructionListType Insts = createIncMemory(AArch64::X0, AArch64::X2);
2518	assert(Insts.size() == 2 && "Invalid Insts size");
2519	std::copy(first: Insts.begin(), last: Insts.end(), result: Instrs.begin() + I);
2520	I += Insts.size();
2521	loadReg(Instrs[I++], AArch64::X2, AArch64::SP);
2522	setSystemFlag(Instrs[I++], AArch64::X1);
2523	createPopRegisters(Instrs[I++], AArch64::X0, AArch64::X1);
2524	if (IsLeaf)
2525	createStackPointerDecrement(Inst&: Instrs[I++], Size: 128);
2526	return Instrs;
2527	}
2528
2529	std::vector<MCInst> createSymbolTrampoline(const MCSymbol *TgtSym,
2530	MCContext *Ctx) override {
2531	std::vector<MCInst> Insts;
2532	createShortJmp(Seq&: Insts, Target: TgtSym, Ctx, /*IsTailCall*/ true);
2533	return Insts;
2534	}
2535
2536	InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx,
2537	MCPhysReg RegName,
2538	int64_t Addend = 0) const override {
2539	// Get page-aligned address and add page offset
2540	InstructionListType Insts(2);
2541	Insts[0].setOpcode(AArch64::ADRP);
2542	Insts[0].clear();
2543	Insts[0].addOperand(Op: MCOperand::createReg(Reg: RegName));
2544	Insts[0].addOperand(Op: MCOperand::createImm(Val: 0));
2545	setOperandToSymbolRef(Inst&: Insts[0], /* OpNum */ 1, Symbol: Target, Addend, Ctx,
2546	RelType: ELF::R_AARCH64_NONE);
2547	Insts[1].setOpcode(AArch64::ADDXri);
2548	Insts[1].clear();
2549	Insts[1].addOperand(Op: MCOperand::createReg(Reg: RegName));
2550	Insts[1].addOperand(Op: MCOperand::createReg(Reg: RegName));
2551	Insts[1].addOperand(Op: MCOperand::createImm(Val: 0));
2552	Insts[1].addOperand(Op: MCOperand::createImm(Val: 0));
2553	setOperandToSymbolRef(Inst&: Insts[1], /* OpNum */ 2, Symbol: Target, Addend, Ctx,
2554	RelType: ELF::R_AARCH64_ADD_ABS_LO12_NC);
2555	return Insts;
2556	}
2557
2558	std::optional<Relocation>
2559	createRelocation(const MCFixup &Fixup,
2560	const MCAsmBackend &MAB) const override {
2561	MCFixupKindInfo FKI = MAB.getFixupKindInfo(Kind: Fixup.getKind());
2562
2563	assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2564	const uint64_t RelOffset = Fixup.getOffset();
2565
2566	uint32_t RelType;
2567	if (Fixup.getKind() == MCFixupKind(AArch64::fixup_aarch64_pcrel_call26))
2568	RelType = ELF::R_AARCH64_CALL26;
2569	else if (Fixup.getKind() ==
2570	MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26))
2571	RelType = ELF::R_AARCH64_JUMP26;
2572	else if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2573	switch (FKI.TargetSize) {
2574	default:
2575	return std::nullopt;
2576	case 16:
2577	RelType = ELF::R_AARCH64_PREL16;
2578	break;
2579	case 32:
2580	RelType = ELF::R_AARCH64_PREL32;
2581	break;
2582	case 64:
2583	RelType = ELF::R_AARCH64_PREL64;
2584	break;
2585	}
2586	} else {
2587	switch (FKI.TargetSize) {
2588	default:
2589	return std::nullopt;
2590	case 16:
2591	RelType = ELF::R_AARCH64_ABS16;
2592	break;
2593	case 32:
2594	RelType = ELF::R_AARCH64_ABS32;
2595	break;
2596	case 64:
2597	RelType = ELF::R_AARCH64_ABS64;
2598	break;
2599	}
2600	}
2601
2602	auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
2603
2604	return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
2605	}
2606
2607	uint16_t getMinFunctionAlignment() const override { return 4; }
2608
2609	std::optional<uint32_t>
2610	getInstructionSize(const MCInst &Inst) const override {
2611	return 4;
2612	}
2613	};
2614
2615	} // end anonymous namespace
2616
2617	namespace llvm {
2618	namespace bolt {
2619
2620	MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis,
2621	const MCInstrInfo *Info,
2622	const MCRegisterInfo *RegInfo,
2623	const MCSubtargetInfo *STI) {
2624	return new AArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
2625	}
2626
2627	} // namespace bolt
2628	} // namespace llvm
2629