1	//===- bolt/Target/AArch64/AArch64MCPlusBuilder.cpp -----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file provides AArch64-specific MCPlus builder.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "AArch64InstrInfo.h"
14	#include "AArch64MCSymbolizer.h"
15	#include "MCTargetDesc/AArch64AddressingModes.h"
16	#include "MCTargetDesc/AArch64FixupKinds.h"
17	#include "MCTargetDesc/AArch64MCAsmInfo.h"
18	#include "MCTargetDesc/AArch64MCTargetDesc.h"
19	#include "Utils/AArch64BaseInfo.h"
20	#include "bolt/Core/BinaryBasicBlock.h"
21	#include "bolt/Core/BinaryFunction.h"
22	#include "bolt/Core/MCPlusBuilder.h"
23	#include "llvm/BinaryFormat/ELF.h"
24	#include "llvm/MC/MCContext.h"
25	#include "llvm/MC/MCInstBuilder.h"
26	#include "llvm/MC/MCInstrInfo.h"
27	#include "llvm/MC/MCRegister.h"
28	#include "llvm/MC/MCRegisterInfo.h"
29	#include "llvm/Support/DataExtractor.h"
30	#include "llvm/Support/Debug.h"
31	#include "llvm/Support/ErrorHandling.h"
32
33	#define DEBUG_TYPE "mcplus"
34
35	using namespace llvm;
36	using namespace bolt;
37
38	namespace {
39
40	static void getSystemFlag(MCInst &Inst, MCPhysReg RegName) {
41	Inst.setOpcode(AArch64::MRS);
42	Inst.clear();
43	Inst.addOperand(Op: MCOperand::createReg(Reg: RegName));
44	Inst.addOperand(Op: MCOperand::createImm(Val: AArch64SysReg::NZCV));
45	}
46
47	static void setSystemFlag(MCInst &Inst, MCPhysReg RegName) {
48	Inst.setOpcode(AArch64::MSR);
49	Inst.clear();
50	Inst.addOperand(Op: MCOperand::createImm(Val: AArch64SysReg::NZCV));
51	Inst.addOperand(Op: MCOperand::createReg(Reg: RegName));
52	}
53
54	static void createPushRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {
55	Inst.clear();
56	unsigned NewOpcode = AArch64::STPXpre;
57	Inst.setOpcode(NewOpcode);
58	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
59	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg1));
60	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg2));
61	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
62	Inst.addOperand(Op: MCOperand::createImm(Val: -2));
63	}
64
65	static void createPopRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {
66	Inst.clear();
67	unsigned NewOpcode = AArch64::LDPXpost;
68	Inst.setOpcode(NewOpcode);
69	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
70	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg1));
71	Inst.addOperand(Op: MCOperand::createReg(Reg: Reg2));
72	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
73	Inst.addOperand(Op: MCOperand::createImm(Val: 2));
74	}
75
76	static void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From) {
77	Inst.setOpcode(AArch64::LDRXui);
78	Inst.clear();
79	if (From == AArch64::SP) {
80	Inst.setOpcode(AArch64::LDRXpost);
81	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
82	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
83	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
84	Inst.addOperand(Op: MCOperand::createImm(Val: 16));
85	} else {
86	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
87	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
88	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
89	}
90	}
91
92	static void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To) {
93	Inst.setOpcode(AArch64::STRXui);
94	Inst.clear();
95	if (To == AArch64::SP) {
96	Inst.setOpcode(AArch64::STRXpre);
97	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
98	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
99	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
100	Inst.addOperand(Op: MCOperand::createImm(Val: -16));
101	} else {
102	Inst.addOperand(Op: MCOperand::createReg(Reg: From));
103	Inst.addOperand(Op: MCOperand::createReg(Reg: To));
104	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
105	}
106	}
107
108	static void atomicAdd(MCInst &Inst, MCPhysReg RegTo, MCPhysReg RegCnt) {
109	// NOTE: Supports only ARM with LSE extension
110	Inst.setOpcode(AArch64::LDADDX);
111	Inst.clear();
112	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::XZR));
113	Inst.addOperand(Op: MCOperand::createReg(Reg: RegCnt));
114	Inst.addOperand(Op: MCOperand::createReg(Reg: RegTo));
115	}
116
117	static void createMovz(MCInst &Inst, MCPhysReg Reg, uint64_t Imm) {
118	assert(Imm <= UINT16_MAX && "Invalid Imm size");
119	Inst.clear();
120	Inst.setOpcode(AArch64::MOVZXi);
121	Inst.addOperand(Op: MCOperand::createReg(Reg));
122	Inst.addOperand(Op: MCOperand::createImm(Val: Imm & 0xFFFF));
123	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
124	}
125
126	static InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegTmp) {
127	InstructionListType Insts;
128	Insts.emplace_back();
129	createMovz(Inst&: Insts.back(), Reg: RegTmp, Imm: 1);
130	Insts.emplace_back();
131	atomicAdd(Inst&: Insts.back(), RegTo, RegCnt: RegTmp);
132	return Insts;
133	}
134	class AArch64MCPlusBuilder : public MCPlusBuilder {
135	public:
136	using MCPlusBuilder::MCPlusBuilder;
137
138	std::unique_ptr<MCSymbolizer>
139	createTargetSymbolizer(BinaryFunction &Function,
140	bool CreateNewSymbols) const override {
141	return std::make_unique<AArch64MCSymbolizer>(args&: Function, args&: CreateNewSymbols);
142	}
143
144	MCPhysReg getStackPointer() const override { return AArch64::SP; }
145	MCPhysReg getFramePointer() const override { return AArch64::FP; }
146
147	bool isPush(const MCInst &Inst) const override {
148	return isStoreToStack(Inst);
149	};
150
151	bool isPop(const MCInst &Inst) const override {
152	return isLoadFromStack(Inst);
153	};
154
155	void createCall(MCInst &Inst, const MCSymbol *Target,
156	MCContext *Ctx) override {
157	createDirectCall(Inst, Target, Ctx, IsTailCall: false);
158	}
159
160	bool convertTailCallToCall(MCInst &Inst) override {
161	int NewOpcode;
162	switch (Inst.getOpcode()) {
163	default:
164	return false;
165	case AArch64::B:
166	NewOpcode = AArch64::BL;
167	break;
168	case AArch64::BR:
169	NewOpcode = AArch64::BLR;
170	break;
171	}
172
173	Inst.setOpcode(NewOpcode);
174	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
175	clearOffset(Inst);
176	return true;
177	}
178
179	bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
180	CompFuncTy Comp) const override {
181	if (A.getSpecifier() != B.getSpecifier())
182	return false;
183
184	return MCPlusBuilder::equals(A: *A.getSubExpr(), B: *B.getSubExpr(), Comp);
185	}
186
187	bool shortenInstruction(MCInst &, const MCSubtargetInfo &) const override {
188	return false;
189	}
190
191	SmallVector<MCPhysReg> getTrustedLiveInRegs() const override {
192	return {AArch64::LR};
193	}
194
195	std::optional<MCPhysReg>
196	getWrittenAuthenticatedReg(const MCInst &Inst,
197	bool &IsChecked) const override {
198	IsChecked = false;
199	switch (Inst.getOpcode()) {
200	case AArch64::AUTIAZ:
201	case AArch64::AUTIBZ:
202	case AArch64::AUTIASP:
203	case AArch64::AUTIBSP:
204	case AArch64::AUTIASPPCi:
205	case AArch64::AUTIBSPPCi:
206	case AArch64::AUTIASPPCr:
207	case AArch64::AUTIBSPPCr:
208	return AArch64::LR;
209	case AArch64::AUTIA1716:
210	case AArch64::AUTIB1716:
211	case AArch64::AUTIA171615:
212	case AArch64::AUTIB171615:
213	return AArch64::X17;
214	case AArch64::AUTIA:
215	case AArch64::AUTIB:
216	case AArch64::AUTDA:
217	case AArch64::AUTDB:
218	case AArch64::AUTIZA:
219	case AArch64::AUTIZB:
220	case AArch64::AUTDZA:
221	case AArch64::AUTDZB:
222	return Inst.getOperand(i: 0).getReg();
223	case AArch64::LDRAAwriteback:
224	case AArch64::LDRABwriteback:
225	// Note that LDRA(A|B)indexed are not listed here, as they do not write
226	// an authenticated pointer back to the register.
227	IsChecked = true;
228	return Inst.getOperand(i: 2).getReg();
229	default:
230	return std::nullopt;
231	}
232	}
233
234	std::optional<MCPhysReg> getSignedReg(const MCInst &Inst) const override {
235	switch (Inst.getOpcode()) {
236	case AArch64::PACIA:
237	case AArch64::PACIB:
238	case AArch64::PACDA:
239	case AArch64::PACDB:
240	case AArch64::PACIZA:
241	case AArch64::PACIZB:
242	case AArch64::PACDZA:
243	case AArch64::PACDZB:
244	return Inst.getOperand(i: 0).getReg();
245	case AArch64::PACIAZ:
246	case AArch64::PACIBZ:
247	case AArch64::PACIASP:
248	case AArch64::PACIBSP:
249	case AArch64::PACIASPPC:
250	case AArch64::PACIBSPPC:
251	case AArch64::PACNBIASPPC:
252	case AArch64::PACNBIBSPPC:
253	return AArch64::LR;
254	case AArch64::PACIA1716:
255	case AArch64::PACIB1716:
256	case AArch64::PACIA171615:
257	case AArch64::PACIB171615:
258	return AArch64::X17;
259	default:
260	return std::nullopt;
261	}
262	}
263
264	std::optional<MCPhysReg>
265	getRegUsedAsRetDest(const MCInst &Inst,
266	bool &IsAuthenticatedInternally) const override {
267	assert(isReturn(Inst));
268	switch (Inst.getOpcode()) {
269	case AArch64::RET:
270	IsAuthenticatedInternally = false;
271	return Inst.getOperand(i: 0).getReg();
272
273	case AArch64::RETAA:
274	case AArch64::RETAB:
275	case AArch64::RETAASPPCi:
276	case AArch64::RETABSPPCi:
277	case AArch64::RETAASPPCr:
278	case AArch64::RETABSPPCr:
279	IsAuthenticatedInternally = true;
280	return AArch64::LR;
281	case AArch64::ERET:
282	case AArch64::ERETAA:
283	case AArch64::ERETAB:
284	// The ERET* instructions use either register ELR_EL1, ELR_EL2 or
285	// ELR_EL3, depending on the current Exception Level at run-time.
286	//
287	// Furthermore, these registers are not modelled by LLVM as a regular
288	// MCPhysReg, so there is no way to indicate that through the current API.
289	return std::nullopt;
290	default:
291	llvm_unreachable("Unhandled return instruction");
292	}
293	}
294
295	MCPhysReg getRegUsedAsIndirectBranchDest(
296	const MCInst &Inst, bool &IsAuthenticatedInternally) const override {
297	assert(isIndirectCall(Inst) || isIndirectBranch(Inst));
298
299	switch (Inst.getOpcode()) {
300	case AArch64::BR:
301	case AArch64::BLR:
302	IsAuthenticatedInternally = false;
303	return Inst.getOperand(i: 0).getReg();
304	case AArch64::BRAA:
305	case AArch64::BRAB:
306	case AArch64::BRAAZ:
307	case AArch64::BRABZ:
308	case AArch64::BLRAA:
309	case AArch64::BLRAB:
310	case AArch64::BLRAAZ:
311	case AArch64::BLRABZ:
312	IsAuthenticatedInternally = true;
313	return Inst.getOperand(i: 0).getReg();
314	default:
315	llvm_unreachable("Unhandled indirect branch or call");
316	}
317	}
318
319	std::optional<MCPhysReg>
320	getMaterializedAddressRegForPtrAuth(const MCInst &Inst) const override {
321	switch (Inst.getOpcode()) {
322	case AArch64::ADR:
323	case AArch64::ADRP:
324	// These instructions produce an address value based on the information
325	// encoded into the instruction itself (which should reside in a read-only
326	// code memory) and the value of PC register (that is, the location of
327	// this instruction), so the produced value is not attacker-controlled.
328	return Inst.getOperand(i: 0).getReg();
329	default:
330	return std::nullopt;
331	}
332	}
333
334	std::optional<std::pair<MCPhysReg, MCPhysReg>>
335	analyzeAddressArithmeticsForPtrAuth(const MCInst &Inst) const override {
336	switch (Inst.getOpcode()) {
337	default:
338	return std::nullopt;
339	case AArch64::ADDXri:
340	case AArch64::SUBXri:
341	// The immediate addend is encoded into the instruction itself, so it is
342	// not attacker-controlled under Pointer Authentication threat model.
343	return std::make_pair(x: Inst.getOperand(i: 0).getReg(),
344	y: Inst.getOperand(i: 1).getReg());
345	case AArch64::ORRXrs:
346	// "mov Xd, Xm" is equivalent to "orr Xd, XZR, Xm, lsl #0"
347	if (Inst.getOperand(i: 1).getReg() != AArch64::XZR ||
348	Inst.getOperand(i: 3).getImm() != 0)
349	return std::nullopt;
350
351	return std::make_pair(x: Inst.getOperand(i: 0).getReg(),
352	y: Inst.getOperand(i: 2).getReg());
353	}
354	}
355
356	std::optional<std::pair<MCPhysReg, MCInst *>>
357	getAuthCheckedReg(BinaryBasicBlock &BB) const override {
358	// Match several possible hard-coded sequences of instructions which can be
359	// emitted by LLVM backend to check that the authenticated pointer is
360	// correct (see AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue).
361	//
362	// This function only matches sequences involving branch instructions.
363	// All these sequences have the form:
364	//
365	// (0) ... regular code that authenticates a pointer in Xn ...
366	// (1) analyze Xn
367	// (2) branch to .Lon_success if the pointer is correct
368	// (3) BRK #imm (fall-through basic block)
369	//
370	// In the above pseudocode, (1) + (2) is one of the following sequences:
371	//
372	// - eor Xtmp, Xn, Xn, lsl #1
373	// tbz Xtmp, #62, .Lon_success
374	//
375	// - mov Xtmp, Xn
376	// xpac(i|d) Xn (or xpaclri if Xn is LR)
377	// cmp Xtmp, Xn
378	// b.eq .Lon_success
379	//
380	// Note that any branch destination operand is accepted as .Lon_success -
381	// it is the responsibility of the caller of getAuthCheckedReg to inspect
382	// the list of successors of this basic block as appropriate.
383
384	// Any of the above code sequences assume the fall-through basic block
385	// is a dead-end BRK instruction (any immediate operand is accepted).
386	const BinaryBasicBlock *BreakBB = BB.getFallthrough();
387	if (!BreakBB || BreakBB->empty() ||
388	BreakBB->front().getOpcode() != AArch64::BRK)
389	return std::nullopt;
390
391	// Iterate over the instructions of BB in reverse order, matching opcodes
392	// and operands.
393	MCPhysReg TestedReg = 0;
394	MCPhysReg ScratchReg = 0;
395	auto It = BB.end();
396	auto StepAndGetOpcode = [&It, &BB]() -> int {
397	if (It == BB.begin())
398	return -1;
399	--It;
400	return It->getOpcode();
401	};
402
403	switch (StepAndGetOpcode()) {
404	default:
405	// Not matched the branch instruction.
406	return std::nullopt;
407	case AArch64::Bcc:
408	// Bcc EQ, .Lon_success
409	if (It->getOperand(i: 0).getImm() != AArch64CC::EQ)
410	return std::nullopt;
411	// Not checking .Lon_success (see above).
412
413	// SUBSXrs XZR, TestedReg, ScratchReg, 0 (used by "CMP reg, reg" alias)
414	if (StepAndGetOpcode() != AArch64::SUBSXrs ||
415	It->getOperand(i: 0).getReg() != AArch64::XZR ||
416	It->getOperand(i: 3).getImm() != 0)
417	return std::nullopt;
418	TestedReg = It->getOperand(i: 1).getReg();
419	ScratchReg = It->getOperand(i: 2).getReg();
420
421	// Either XPAC(I|D) ScratchReg, ScratchReg
422	// or XPACLRI
423	switch (StepAndGetOpcode()) {
424	default:
425	return std::nullopt;
426	case AArch64::XPACLRI:
427	// No operands to check, but using XPACLRI forces TestedReg to be X30.
428	if (TestedReg != AArch64::LR)
429	return std::nullopt;
430	break;
431	case AArch64::XPACI:
432	case AArch64::XPACD:
433	if (It->getOperand(i: 0).getReg() != ScratchReg ||
434	It->getOperand(i: 1).getReg() != ScratchReg)
435	return std::nullopt;
436	break;
437	}
438
439	// ORRXrs ScratchReg, XZR, TestedReg, 0 (used by "MOV reg, reg" alias)
440	if (StepAndGetOpcode() != AArch64::ORRXrs)
441	return std::nullopt;
442	if (It->getOperand(i: 0).getReg() != ScratchReg ||
443	It->getOperand(i: 1).getReg() != AArch64::XZR ||
444	It->getOperand(i: 2).getReg() != TestedReg ||
445	It->getOperand(i: 3).getImm() != 0)
446	return std::nullopt;
447
448	return std::make_pair(x&: TestedReg, y: &*It);
449
450	case AArch64::TBZX:
451	// TBZX ScratchReg, 62, .Lon_success
452	ScratchReg = It->getOperand(i: 0).getReg();
453	if (It->getOperand(i: 1).getImm() != 62)
454	return std::nullopt;
455	// Not checking .Lon_success (see above).
456
457	// EORXrs ScratchReg, TestedReg, TestedReg, 1
458	if (StepAndGetOpcode() != AArch64::EORXrs)
459	return std::nullopt;
460	TestedReg = It->getOperand(i: 1).getReg();
461	if (It->getOperand(i: 0).getReg() != ScratchReg ||
462	It->getOperand(i: 2).getReg() != TestedReg ||
463	It->getOperand(i: 3).getImm() != 1)
464	return std::nullopt;
465
466	return std::make_pair(x&: TestedReg, y: &*It);
467	}
468	}
469
470	std::optional<MCPhysReg> getAuthCheckedReg(const MCInst &Inst,
471	bool MayOverwrite) const override {
472	// Cannot trivially reuse AArch64InstrInfo::getMemOperandWithOffsetWidth()
473	// method as it accepts an instance of MachineInstr, not MCInst.
474	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
475
476	// If signing oracles are considered, the particular value left in the base
477	// register after this instruction is important. This function checks that
478	// if the base register was overwritten, it is due to address write-back:
479	//
480	// ; good:
481	// autdza x1 ; x1 is authenticated (may fail)
482	// ldr x0, [x1, #8] ; x1 is checked and not changed
483	// pacdzb x1
484	//
485	// ; also good:
486	// autdza x1
487	// ldr x0, [x1, #8]! ; x1 is checked and incremented by 8
488	// pacdzb x1
489	//
490	// ; bad (the value being signed is not the authenticated one):
491	// autdza x1
492	// ldr x1, [x1, #8] ; x1 is overwritten with an unrelated value
493	// pacdzb x1
494	//
495	// ; also bad:
496	// autdza x1
497	// pacdzb x1 ; possibly signing the result of failed authentication
498	//
499	// Note that this function is not needed for authentication oracles, as the
500	// particular value left in the register after a successful memory access
501	// is not important.
502	auto ClobbersBaseRegExceptWriteback = [&](unsigned BaseRegUseIndex) {
503	// FIXME: Compute the indices of address operands (base reg and written-
504	// back result) in AArch64InstrInfo instead of this ad-hoc code.
505	MCPhysReg BaseReg = Inst.getOperand(i: BaseRegUseIndex).getReg();
506	unsigned WrittenBackDefIndex = Desc.getOperandConstraint(
507	OpNum: BaseRegUseIndex, Constraint: MCOI::OperandConstraint::TIED_TO);
508
509	for (unsigned DefIndex = 0; DefIndex < Desc.getNumDefs(); ++DefIndex) {
510	// Address write-back is permitted:
511	//
512	// autda x0, x2
513	// ; x0 is authenticated
514	// ldr x1, [x0, #8]!
515	// ; x0 is trusted (as authenticated and checked)
516	if (DefIndex == WrittenBackDefIndex)
517	continue;
518
519	// Any other overwriting is not permitted:
520	//
521	// autda x0, x2
522	// ; x0 is authenticated
523	// ldr w0, [x0]
524	// ; x0 is not authenticated anymore
525	if (RegInfo->regsOverlap(RegA: Inst.getOperand(i: DefIndex).getReg(), RegB: BaseReg))
526	return true;
527	}
528
529	return false;
530	};
531
532	// FIXME: Not all load instructions are handled by this->mayLoad(Inst).
533	// On the other hand, MCInstrDesc::mayLoad() is permitted to return
534	// true for non-load instructions (such as AArch64::HINT) which
535	// would result in false negatives.
536	if (mayLoad(Inst)) {
537	// The first Use operand is the base address register.
538	unsigned BaseRegIndex = Desc.getNumDefs();
539
540	// Reject non-immediate offsets, as adding a 64-bit register can change
541	// the resulting address arbitrarily.
542	for (unsigned I = BaseRegIndex + 1, E = Desc.getNumOperands(); I < E; ++I)
543	if (Inst.getOperand(i: I).isReg())
544	return std::nullopt;
545
546	if (!MayOverwrite && ClobbersBaseRegExceptWriteback(BaseRegIndex))
547	return std::nullopt;
548
549	return Inst.getOperand(i: BaseRegIndex).getReg();
550	}
551
552	// Store instructions are not handled yet, as they are not important for
553	// pauthtest ABI. Though, they could be handled similar to loads, if needed.
554
555	return std::nullopt;
556	}
557
558	bool isADRP(const MCInst &Inst) const override {
559	return Inst.getOpcode() == AArch64::ADRP;
560	}
561
562	bool isADR(const MCInst &Inst) const override {
563	return Inst.getOpcode() == AArch64::ADR;
564	}
565
566	bool isAddXri(const MCInst &Inst) const override {
567	return Inst.getOpcode() == AArch64::ADDXri;
568	}
569
570	MCPhysReg getADRReg(const MCInst &Inst) const {
571	assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
572	assert(MCPlus::getNumPrimeOperands(Inst) != 0 &&
573	"No operands for ADR instruction");
574	assert(Inst.getOperand(0).isReg() &&
575	"Unexpected operand in ADR instruction");
576	return Inst.getOperand(i: 0).getReg();
577	}
578
579	InstructionListType undoAdrpAddRelaxation(const MCInst &ADRInst,
580	MCContext *Ctx) const override {
581	assert(isADR(ADRInst) && "ADR instruction expected");
582
583	const MCPhysReg Reg = getADRReg(Inst: ADRInst);
584	const MCSymbol *Target = getTargetSymbol(Inst: ADRInst);
585	const uint64_t Addend = getTargetAddend(Inst: ADRInst);
586	return materializeAddress(Target, Ctx, RegName: Reg, Addend);
587	}
588
589	bool isTB(const MCInst &Inst) const {
590	return (Inst.getOpcode() == AArch64::TBNZW ||
591	Inst.getOpcode() == AArch64::TBNZX ||
592	Inst.getOpcode() == AArch64::TBZW ||
593	Inst.getOpcode() == AArch64::TBZX);
594	}
595
596	bool isCB(const MCInst &Inst) const {
597	return (Inst.getOpcode() == AArch64::CBNZW ||
598	Inst.getOpcode() == AArch64::CBNZX ||
599	Inst.getOpcode() == AArch64::CBZW ||
600	Inst.getOpcode() == AArch64::CBZX);
601	}
602
603	bool isMOVW(const MCInst &Inst) const override {
604	return (Inst.getOpcode() == AArch64::MOVKWi ||
605	Inst.getOpcode() == AArch64::MOVKXi ||
606	Inst.getOpcode() == AArch64::MOVNWi ||
607	Inst.getOpcode() == AArch64::MOVNXi ||
608	Inst.getOpcode() == AArch64::MOVZXi ||
609	Inst.getOpcode() == AArch64::MOVZWi);
610	}
611
612	bool isADD(const MCInst &Inst) const {
613	return (Inst.getOpcode() == AArch64::ADDSWri ||
614	Inst.getOpcode() == AArch64::ADDSWrr ||
615	Inst.getOpcode() == AArch64::ADDSWrs ||
616	Inst.getOpcode() == AArch64::ADDSWrx ||
617	Inst.getOpcode() == AArch64::ADDSXri ||
618	Inst.getOpcode() == AArch64::ADDSXrr ||
619	Inst.getOpcode() == AArch64::ADDSXrs ||
620	Inst.getOpcode() == AArch64::ADDSXrx ||
621	Inst.getOpcode() == AArch64::ADDSXrx64 ||
622	Inst.getOpcode() == AArch64::ADDWri ||
623	Inst.getOpcode() == AArch64::ADDWrr ||
624	Inst.getOpcode() == AArch64::ADDWrs ||
625	Inst.getOpcode() == AArch64::ADDWrx ||
626	Inst.getOpcode() == AArch64::ADDXri ||
627	Inst.getOpcode() == AArch64::ADDXrr ||
628	Inst.getOpcode() == AArch64::ADDXrs ||
629	Inst.getOpcode() == AArch64::ADDXrx ||
630	Inst.getOpcode() == AArch64::ADDXrx64);
631	}
632
633	bool isLDRB(const MCInst &Inst) const {
634	const unsigned opcode = Inst.getOpcode();
635	switch (opcode) {
636	case AArch64::LDRBpost:
637	case AArch64::LDRBBpost:
638	case AArch64::LDRBBpre:
639	case AArch64::LDRBBroW:
640	case AArch64::LDRBroW:
641	case AArch64::LDRBroX:
642	case AArch64::LDRBBroX:
643	case AArch64::LDRBBui:
644	case AArch64::LDRBui:
645	case AArch64::LDRBpre:
646	case AArch64::LDRSBWpost:
647	case AArch64::LDRSBWpre:
648	case AArch64::LDRSBWroW:
649	case AArch64::LDRSBWroX:
650	case AArch64::LDRSBWui:
651	case AArch64::LDRSBXpost:
652	case AArch64::LDRSBXpre:
653	case AArch64::LDRSBXroW:
654	case AArch64::LDRSBXroX:
655	case AArch64::LDRSBXui:
656	case AArch64::LDURBi:
657	case AArch64::LDURBBi:
658	case AArch64::LDURSBWi:
659	case AArch64::LDURSBXi:
660	case AArch64::LDTRBi:
661	case AArch64::LDTRSBWi:
662	case AArch64::LDTRSBXi:
663	return true;
664	default:
665	break;
666	}
667
668	return false;
669	}
670
671	bool isLDRH(const MCInst &Inst) const {
672	const unsigned opcode = Inst.getOpcode();
673	switch (opcode) {
674	case AArch64::LDRHpost:
675	case AArch64::LDRHHpost:
676	case AArch64::LDRHHpre:
677	case AArch64::LDRHroW:
678	case AArch64::LDRHHroW:
679	case AArch64::LDRHroX:
680	case AArch64::LDRHHroX:
681	case AArch64::LDRHHui:
682	case AArch64::LDRHui:
683	case AArch64::LDRHpre:
684	case AArch64::LDRSHWpost:
685	case AArch64::LDRSHWpre:
686	case AArch64::LDRSHWroW:
687	case AArch64::LDRSHWroX:
688	case AArch64::LDRSHWui:
689	case AArch64::LDRSHXpost:
690	case AArch64::LDRSHXpre:
691	case AArch64::LDRSHXroW:
692	case AArch64::LDRSHXroX:
693	case AArch64::LDRSHXui:
694	case AArch64::LDURHi:
695	case AArch64::LDURHHi:
696	case AArch64::LDURSHWi:
697	case AArch64::LDURSHXi:
698	case AArch64::LDTRHi:
699	case AArch64::LDTRSHWi:
700	case AArch64::LDTRSHXi:
701	return true;
702	default:
703	break;
704	}
705
706	return false;
707	}
708
709	bool isLDRW(const MCInst &Inst) const {
710	const unsigned opcode = Inst.getOpcode();
711	switch (opcode) {
712	case AArch64::LDRWpost:
713	case AArch64::LDRWpre:
714	case AArch64::LDRWroW:
715	case AArch64::LDRWroX:
716	case AArch64::LDRWui:
717	case AArch64::LDRWl:
718	case AArch64::LDRSWl:
719	case AArch64::LDURWi:
720	case AArch64::LDRSWpost:
721	case AArch64::LDRSWpre:
722	case AArch64::LDRSWroW:
723	case AArch64::LDRSWroX:
724	case AArch64::LDRSWui:
725	case AArch64::LDURSWi:
726	case AArch64::LDTRWi:
727	case AArch64::LDTRSWi:
728	case AArch64::LDPWi:
729	case AArch64::LDPWpost:
730	case AArch64::LDPWpre:
731	case AArch64::LDPSWi:
732	case AArch64::LDPSWpost:
733	case AArch64::LDPSWpre:
734	case AArch64::LDNPWi:
735	return true;
736	default:
737	break;
738	}
739
740	return false;
741	}
742
743	bool isLDRX(const MCInst &Inst) const {
744	const unsigned opcode = Inst.getOpcode();
745	switch (opcode) {
746	case AArch64::LDRXpost:
747	case AArch64::LDRXpre:
748	case AArch64::LDRXroW:
749	case AArch64::LDRXroX:
750	case AArch64::LDRXui:
751	case AArch64::LDRXl:
752	case AArch64::LDURXi:
753	case AArch64::LDTRXi:
754	case AArch64::LDNPXi:
755	case AArch64::LDPXi:
756	case AArch64::LDPXpost:
757	case AArch64::LDPXpre:
758	return true;
759	default:
760	break;
761	}
762
763	return false;
764	}
765
766	bool isLDRS(const MCInst &Inst) const {
767	const unsigned opcode = Inst.getOpcode();
768	switch (opcode) {
769	case AArch64::LDRSl:
770	case AArch64::LDRSui:
771	case AArch64::LDRSroW:
772	case AArch64::LDRSroX:
773	case AArch64::LDURSi:
774	case AArch64::LDPSi:
775	case AArch64::LDNPSi:
776	case AArch64::LDRSpre:
777	case AArch64::LDRSpost:
778	case AArch64::LDPSpost:
779	case AArch64::LDPSpre:
780	return true;
781	default:
782	break;
783	}
784
785	return false;
786	}
787
788	bool isLDRD(const MCInst &Inst) const {
789	const unsigned opcode = Inst.getOpcode();
790	switch (opcode) {
791	case AArch64::LDRDl:
792	case AArch64::LDRDui:
793	case AArch64::LDRDpre:
794	case AArch64::LDRDpost:
795	case AArch64::LDRDroW:
796	case AArch64::LDRDroX:
797	case AArch64::LDURDi:
798	case AArch64::LDPDi:
799	case AArch64::LDNPDi:
800	case AArch64::LDPDpost:
801	case AArch64::LDPDpre:
802	return true;
803	default:
804	break;
805	}
806
807	return false;
808	}
809
810	bool isLDRQ(const MCInst &Inst) const {
811	const unsigned opcode = Inst.getOpcode();
812	switch (opcode) {
813	case AArch64::LDRQui:
814	case AArch64::LDRQl:
815	case AArch64::LDRQpre:
816	case AArch64::LDRQpost:
817	case AArch64::LDRQroW:
818	case AArch64::LDRQroX:
819	case AArch64::LDURQi:
820	case AArch64::LDPQi:
821	case AArch64::LDNPQi:
822	case AArch64::LDPQpost:
823	case AArch64::LDPQpre:
824	return true;
825	default:
826	break;
827	}
828
829	return false;
830	}
831
832	bool isBRA(const MCInst &Inst) const {
833	switch (Inst.getOpcode()) {
834	case AArch64::BRAA:
835	case AArch64::BRAB:
836	case AArch64::BRAAZ:
837	case AArch64::BRABZ:
838	return true;
839	default:
840	return false;
841	}
842	}
843
844	bool mayLoad(const MCInst &Inst) const override {
845	// FIXME: Probably this could be tablegen-erated not to miss any existing
846	// or future opcodes.
847	return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst) ||
848	isLDRQ(Inst) || isLDRD(Inst) || isLDRS(Inst);
849	}
850
851	bool isAArch64ExclusiveLoad(const MCInst &Inst) const override {
852	return (Inst.getOpcode() == AArch64::LDXPX ||
853	Inst.getOpcode() == AArch64::LDXPW ||
854	Inst.getOpcode() == AArch64::LDXRX ||
855	Inst.getOpcode() == AArch64::LDXRW ||
856	Inst.getOpcode() == AArch64::LDXRH ||
857	Inst.getOpcode() == AArch64::LDXRB ||
858	Inst.getOpcode() == AArch64::LDAXPX ||
859	Inst.getOpcode() == AArch64::LDAXPW ||
860	Inst.getOpcode() == AArch64::LDAXRX ||
861	Inst.getOpcode() == AArch64::LDAXRW ||
862	Inst.getOpcode() == AArch64::LDAXRH ||
863	Inst.getOpcode() == AArch64::LDAXRB);
864	}
865
866	bool isAArch64ExclusiveStore(const MCInst &Inst) const override {
867	return (Inst.getOpcode() == AArch64::STXPX ||
868	Inst.getOpcode() == AArch64::STXPW ||
869	Inst.getOpcode() == AArch64::STXRX ||
870	Inst.getOpcode() == AArch64::STXRW ||
871	Inst.getOpcode() == AArch64::STXRH ||
872	Inst.getOpcode() == AArch64::STXRB ||
873	Inst.getOpcode() == AArch64::STLXPX ||
874	Inst.getOpcode() == AArch64::STLXPW ||
875	Inst.getOpcode() == AArch64::STLXRX ||
876	Inst.getOpcode() == AArch64::STLXRW ||
877	Inst.getOpcode() == AArch64::STLXRH ||
878	Inst.getOpcode() == AArch64::STLXRB);
879	}
880
881	bool isAArch64ExclusiveClear(const MCInst &Inst) const override {
882	return (Inst.getOpcode() == AArch64::CLREX);
883	}
884
885	bool isLoadFromStack(const MCInst &Inst) const {
886	if (!mayLoad(Inst))
887	return false;
888	for (const MCOperand &Operand : useOperands(Inst)) {
889	if (!Operand.isReg())
890	continue;
891	unsigned Reg = Operand.getReg();
892	if (Reg == AArch64::SP || Reg == AArch64::WSP)
893	return true;
894	}
895	return false;
896	}
897
898	bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
899	MCPhysReg &To) const override {
900	if (Inst.getOpcode() == AArch64::FMOVDXr) {
901	From = Inst.getOperand(i: 1).getReg();
902	To = Inst.getOperand(i: 0).getReg();
903	return true;
904	}
905
906	if (Inst.getOpcode() != AArch64::ORRXrs)
907	return false;
908	if (Inst.getOperand(i: 1).getReg() != AArch64::XZR)
909	return false;
910	if (Inst.getOperand(i: 3).getImm() != 0)
911	return false;
912	From = Inst.getOperand(i: 2).getReg();
913	To = Inst.getOperand(i: 0).getReg();
914	return true;
915	}
916
917	bool isIndirectCall(const MCInst &Inst) const override {
918	return isIndirectCallOpcode(Opc: Inst.getOpcode());
919	}
920
921	MCPhysReg getSpRegister(int Size) const {
922	switch (Size) {
923	case 4:
924	return AArch64::WSP;
925	case 8:
926	return AArch64::SP;
927	default:
928	llvm_unreachable("Unexpected size");
929	}
930	}
931
932	MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
933	switch (ArgNo) {
934	case 0:
935	return AArch64::X0;
936	case 1:
937	return AArch64::X1;
938	case 2:
939	return AArch64::X2;
940	case 3:
941	return AArch64::X3;
942	case 4:
943	return AArch64::X4;
944	case 5:
945	return AArch64::X5;
946	case 6:
947	return AArch64::X6;
948	case 7:
949	return AArch64::X7;
950	default:
951	return getNoRegister();
952	}
953	}
954
955	bool hasPCRelOperand(const MCInst &Inst) const override {
956	// ADRP is blacklisted and is an exception. Even though it has a
957	// PC-relative operand, this operand is not a complete symbol reference
958	// and BOLT shouldn't try to process it in isolation.
959	if (isADRP(Inst))
960	return false;
961
962	if (isADR(Inst))
963	return true;
964
965	// Look for literal addressing mode (see C1-143 ARM DDI 0487B.a)
966	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
967	for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I)
968	if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)
969	return true;
970
971	return false;
972	}
973
974	bool evaluateADR(const MCInst &Inst, int64_t &Imm,
975	const MCExpr **DispExpr) const {
976	assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
977
978	const MCOperand &Label = Inst.getOperand(i: 1);
979	if (!Label.isImm()) {
980	assert(Label.isExpr() && "Unexpected ADR operand");
981	assert(DispExpr && "DispExpr must be set");
982	*DispExpr = Label.getExpr();
983	return false;
984	}
985
986	if (Inst.getOpcode() == AArch64::ADR) {
987	Imm = Label.getImm();
988	return true;
989	}
990	Imm = Label.getImm() << 12;
991	return true;
992	}
993
994	bool evaluateAArch64MemoryOperand(const MCInst &Inst, int64_t &DispImm,
995	const MCExpr **DispExpr = nullptr) const {
996	if (isADR(Inst) || isADRP(Inst))
997	return evaluateADR(Inst, Imm&: DispImm, DispExpr);
998
999	// Literal addressing mode
1000	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1001	for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1002	if (MCII.operands()[I].OperandType != MCOI::OPERAND_PCREL)
1003	continue;
1004
1005	if (!Inst.getOperand(i: I).isImm()) {
1006	assert(Inst.getOperand(I).isExpr() && "Unexpected PCREL operand");
1007	assert(DispExpr && "DispExpr must be set");
1008	*DispExpr = Inst.getOperand(i: I).getExpr();
1009	return true;
1010	}
1011
1012	DispImm = Inst.getOperand(i: I).getImm() * 4;
1013	return true;
1014	}
1015	return false;
1016	}
1017
1018	bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
1019	uint64_t Address,
1020	uint64_t Size) const override {
1021	int64_t DispValue;
1022	const MCExpr *DispExpr = nullptr;
1023	if (!evaluateAArch64MemoryOperand(Inst, DispImm&: DispValue, DispExpr: &DispExpr))
1024	return false;
1025
1026	// Make sure it's a well-formed addressing we can statically evaluate.
1027	if (DispExpr)
1028	return false;
1029
1030	Target = DispValue;
1031	if (Inst.getOpcode() == AArch64::ADRP)
1032	Target += Address & ~0xFFFULL;
1033	else
1034	Target += Address;
1035	return true;
1036	}
1037
1038	MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
1039	MCInst::iterator OI = Inst.begin();
1040	if (isADR(Inst) || isADRP(Inst)) {
1041	assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&
1042	"Unexpected number of operands");
1043	return ++OI;
1044	}
1045	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1046	for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {
1047	if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)
1048	break;
1049	++OI;
1050	}
1051	assert(OI != Inst.end() && "Literal operand not found");
1052	return OI;
1053	}
1054
1055	bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
1056	MCInst::iterator OI = getMemOperandDisp(Inst);
1057	*OI = Operand;
1058	return true;
1059	}
1060
1061	void getCalleeSavedRegs(BitVector &Regs) const override {
1062	Regs |= getAliases(Reg: AArch64::X18);
1063	Regs |= getAliases(Reg: AArch64::X19);
1064	Regs |= getAliases(Reg: AArch64::X20);
1065	Regs |= getAliases(Reg: AArch64::X21);
1066	Regs |= getAliases(Reg: AArch64::X22);
1067	Regs |= getAliases(Reg: AArch64::X23);
1068	Regs |= getAliases(Reg: AArch64::X24);
1069	Regs |= getAliases(Reg: AArch64::X25);
1070	Regs |= getAliases(Reg: AArch64::X26);
1071	Regs |= getAliases(Reg: AArch64::X27);
1072	Regs |= getAliases(Reg: AArch64::X28);
1073	Regs |= getAliases(Reg: AArch64::LR);
1074	Regs |= getAliases(Reg: AArch64::FP);
1075	}
1076
1077	const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr,
1078	MCContext &Ctx,
1079	uint32_t RelType) const override {
1080
1081	if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||
1082	RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {
1083	return MCSpecifierExpr::create(Expr, S: AArch64::S_ABS, Ctx);
1084	} else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||
1085	RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||
1086	RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||
1087	RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 ||
1088	RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
1089	// Never emit a GOT reloc, we handled this in
1090	// RewriteInstance::readRelocations().
1091	return MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_PAGE, Ctx);
1092	} else {
1093	switch (RelType) {
1094	case ELF::R_AARCH64_ADD_ABS_LO12_NC:
1095	case ELF::R_AARCH64_LD64_GOT_LO12_NC:
1096	case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
1097	case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
1098	case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
1099	case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
1100	case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
1101	case ELF::R_AARCH64_TLSDESC_ADD_LO12:
1102	case ELF::R_AARCH64_TLSDESC_LD64_LO12:
1103	case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
1104	case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
1105	return MCSpecifierExpr::create(Expr, S: AArch64::S_LO12, Ctx);
1106	case ELF::R_AARCH64_MOVW_UABS_G3:
1107	return MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_G3, Ctx);
1108	case ELF::R_AARCH64_MOVW_UABS_G2:
1109	case ELF::R_AARCH64_MOVW_UABS_G2_NC:
1110	return MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_G2_NC, Ctx);
1111	case ELF::R_AARCH64_MOVW_UABS_G1:
1112	case ELF::R_AARCH64_MOVW_UABS_G1_NC:
1113	return MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_G1_NC, Ctx);
1114	case ELF::R_AARCH64_MOVW_UABS_G0:
1115	case ELF::R_AARCH64_MOVW_UABS_G0_NC:
1116	return MCSpecifierExpr::create(Expr, S: AArch64::S_ABS_G0_NC, Ctx);
1117	default:
1118	break;
1119	}
1120	}
1121	return Expr;
1122	}
1123
1124	bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const {
1125	if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1126	return false;
1127
1128	// Auto-select correct operand number
1129	if (OpNum == 0) {
1130	if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst) ||
1131	isMOVW(Inst))
1132	OpNum = 1;
1133	if (isTB(Inst) || isAddXri(Inst))
1134	OpNum = 2;
1135	}
1136
1137	return true;
1138	}
1139
1140	const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1141	auto *AArchExpr = dyn_cast<MCSpecifierExpr>(Val: Expr);
1142	if (AArchExpr && AArchExpr->getSubExpr())
1143	return getTargetSymbol(Expr: AArchExpr->getSubExpr());
1144
1145	return MCPlusBuilder::getTargetSymbol(Expr);
1146	}
1147
1148	const MCSymbol *getTargetSymbol(const MCInst &Inst,
1149	unsigned OpNum = 0) const override {
1150	if (!OpNum && !getSymbolRefOperandNum(Inst, OpNum))
1151	return nullptr;
1152
1153	const MCOperand &Op = Inst.getOperand(i: OpNum);
1154	if (!Op.isExpr())
1155	return nullptr;
1156
1157	return getTargetSymbol(Expr: Op.getExpr());
1158	}
1159
1160	int64_t getTargetAddend(const MCExpr *Expr) const override {
1161	auto *AArchExpr = dyn_cast<MCSpecifierExpr>(Val: Expr);
1162	if (AArchExpr && AArchExpr->getSubExpr())
1163	return getTargetAddend(Expr: AArchExpr->getSubExpr());
1164
1165	auto *BinExpr = dyn_cast<MCBinaryExpr>(Val: Expr);
1166	if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add)
1167	return getTargetAddend(Expr: BinExpr->getRHS());
1168
1169	auto *ConstExpr = dyn_cast<MCConstantExpr>(Val: Expr);
1170	if (ConstExpr)
1171	return ConstExpr->getValue();
1172
1173	return 0;
1174	}
1175
1176	int64_t getTargetAddend(const MCInst &Inst,
1177	unsigned OpNum = 0) const override {
1178	if (!getSymbolRefOperandNum(Inst, OpNum))
1179	return 0;
1180
1181	const MCOperand &Op = Inst.getOperand(i: OpNum);
1182	if (!Op.isExpr())
1183	return 0;
1184
1185	return getTargetAddend(Expr: Op.getExpr());
1186	}
1187
1188	void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
1189	MCContext *Ctx) const override {
1190	assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
1191	"Invalid instruction");
1192	assert(MCPlus::getNumPrimeOperands(Inst) >= 1 &&
1193	"Invalid number of operands");
1194	MCInst::iterator OI = Inst.begin();
1195
1196	if (isConditionalBranch(Inst)) {
1197	assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&
1198	"Invalid number of operands");
1199	++OI;
1200	}
1201
1202	if (isTB(Inst)) {
1203	assert(MCPlus::getNumPrimeOperands(Inst) >= 3 &&
1204	"Invalid number of operands");
1205	OI = Inst.begin() + 2;
1206	}
1207
1208	*OI = MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: TBB, Ctx&: *Ctx));
1209	}
1210
1211	/// Matches indirect branch patterns in AArch64 related to a jump table (JT),
1212	/// helping us to build the complete CFG. A typical indirect branch to
1213	/// a jump table entry in AArch64 looks like the following:
1214	///
1215	/// adrp x1, #-7585792 # Get JT Page location
1216	/// add x1, x1, #692 # Complement with JT Page offset
1217	/// ldrh w0, [x1, w0, uxtw #1] # Loads JT entry
1218	/// adr x1, #12 # Get PC + 12 (end of this BB) used next
1219	/// add x0, x1, w0, sxth #2 # Finish building branch target
1220	/// # (entries in JT are relative to the end
1221	/// # of this BB)
1222	/// br x0 # Indirect jump instruction
1223	///
1224	/// Return true on successful jump table instruction sequence match, false
1225	/// otherwise.
1226	bool analyzeIndirectBranchFragment(
1227	const MCInst &Inst,
1228	DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain,
1229	const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue,
1230	MCInst *&PCRelBase) const {
1231	// The only kind of indirect branches we match is jump table, thus ignore
1232	// authenticating branch instructions early.
1233	if (isBRA(Inst))
1234	return false;
1235
1236	// Expect AArch64 BR
1237	assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");
1238
1239	JumpTable = nullptr;
1240
1241	// Match the indirect branch pattern for aarch64
1242	SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];
1243	if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr)
1244	return false;
1245
1246	const MCInst *DefAdd = UsesRoot[0];
1247
1248	// Now we match an ADD
1249	if (!isADD(Inst: *DefAdd)) {
1250	// If the address is not broken up in two parts, this is not branching
1251	// according to a jump table entry. Fail.
1252	return false;
1253	}
1254	if (DefAdd->getOpcode() == AArch64::ADDXri) {
1255	// This can happen when there is no offset, but a direct jump that was
1256	// transformed into an indirect one (indirect tail call) :
1257	// ADRP x2, Perl_re_compiler
1258	// ADD x2, x2, :lo12:Perl_re_compiler
1259	// BR x2
1260	return false;
1261	}
1262	if (DefAdd->getOpcode() == AArch64::ADDXrs) {
1263	// Covers the less common pattern where JT entries are relative to
1264	// the JT itself (like x86). Seems less efficient since we can't
1265	// assume the JT is aligned at 4B boundary and thus drop 2 bits from
1266	// JT values.
1267	// cde264:
1268	// adrp x12, #21544960 ; 216a000
1269	// add x12, x12, #1696 ; 216a6a0 (JT object in .rodata)
1270	// ldrsw x8, [x12, x8, lsl #2] --> loads e.g. 0xfeb73bd8
1271	// * add x8, x8, x12 --> = cde278, next block
1272	// br x8
1273	// cde278:
1274	//
1275	// Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:0
1276	return false;
1277	}
1278	if (DefAdd->getOpcode() != AArch64::ADDXrx)
1279	return false;
1280
1281	// Validate ADD operands
1282	int64_t OperandExtension = DefAdd->getOperand(i: 3).getImm();
1283	unsigned ShiftVal = AArch64_AM::getArithShiftValue(Imm: OperandExtension);
1284	AArch64_AM::ShiftExtendType ExtendType =
1285	AArch64_AM::getArithExtendType(Imm: OperandExtension);
1286	if (ShiftVal != 2) {
1287	// TODO: Handle the patten where ShiftVal != 2.
1288	// The following code sequence below has no shift amount,
1289	// the range could be 0 to 4.
1290	// The pattern comes from libc, it occurs when the binary is static.
1291	// adr x6, 0x219fb0 <sigall_set+0x88>
1292	// add x6, x6, x14, lsl #2
1293	// ldr w7, [x6]
1294	// add x6, x6, w7, sxtw => no shift amount
1295	// br x6
1296	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: "
1297	"failed to match indirect branch: ShiftVAL != 2\n");
1298	return false;
1299	}
1300
1301	if (ExtendType == AArch64_AM::SXTB)
1302	ScaleValue = 1LL;
1303	else if (ExtendType == AArch64_AM::SXTH)
1304	ScaleValue = 2LL;
1305	else if (ExtendType == AArch64_AM::SXTW)
1306	ScaleValue = 4LL;
1307	else
1308	return false;
1309
1310	// Match an ADR to load base address to be used when addressing JT targets
1311	SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd];
1312	if (UsesAdd.size() <= 1 || UsesAdd[1] == nullptr || UsesAdd[2] == nullptr) {
1313	// This happens when we don't have enough context about this jump table
1314	// because the jumping code sequence was split in multiple basic blocks.
1315	// This was observed in the wild in HHVM code (dispatchImpl).
1316	return false;
1317	}
1318	MCInst *DefBaseAddr = UsesAdd[1];
1319	if (DefBaseAddr->getOpcode() != AArch64::ADR)
1320	return false;
1321
1322	PCRelBase = DefBaseAddr;
1323	// Match LOAD to load the jump table (relative) target
1324	const MCInst *DefLoad = UsesAdd[2];
1325	if (!mayLoad(Inst: *DefLoad) || (ScaleValue == 1LL && !isLDRB(Inst: *DefLoad)) ||
1326	(ScaleValue == 2LL && !isLDRH(Inst: *DefLoad)))
1327	return false;
1328
1329	// Match ADD that calculates the JumpTable Base Address (not the offset)
1330	SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];
1331	const MCInst *DefJTBaseAdd = UsesLoad[1];
1332	MCPhysReg From, To;
1333	if (DefJTBaseAdd == nullptr || isLoadFromStack(Inst: *DefJTBaseAdd) ||
1334	isRegToRegMove(Inst: *DefJTBaseAdd, From, To)) {
1335	// Sometimes base address may have been defined in another basic block
1336	// (hoisted). Return with no jump table info.
1337	return true;
1338	}
1339
1340	if (DefJTBaseAdd->getOpcode() == AArch64::ADR) {
1341	// TODO: Handle the pattern where there is no adrp/add pair.
1342	// It also occurs when the binary is static.
1343	// adr x13, 0x215a18 <_nl_value_type_LC_COLLATE+0x50>
1344	// ldrh w13, [x13, w12, uxtw #1]
1345	// adr x12, 0x247b30 <__gettextparse+0x5b0>
1346	// add x13, x12, w13, sxth #2
1347	// br x13
1348	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match indirect branch: "
1349	"nop/adr instead of adrp/add\n");
1350	return false;
1351	}
1352
1353	if (DefJTBaseAdd->getOpcode() != AArch64::ADDXri) {
1354	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match jump table base "
1355	"address pattern! (1)\n");
1356	return false;
1357	}
1358
1359	if (DefJTBaseAdd->getOperand(i: 2).isImm())
1360	Offset = DefJTBaseAdd->getOperand(i: 2).getImm();
1361	SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];
1362	const MCInst *DefJTBasePage = UsesJTBaseAdd[1];
1363	if (DefJTBasePage == nullptr || isLoadFromStack(Inst: *DefJTBasePage)) {
1364	return true;
1365	}
1366	if (DefJTBasePage->getOpcode() != AArch64::ADRP)
1367	return false;
1368
1369	if (DefJTBasePage->getOperand(i: 1).isExpr())
1370	JumpTable = DefJTBasePage->getOperand(i: 1).getExpr();
1371	return true;
1372	}
1373
1374	DenseMap<const MCInst *, SmallVector<MCInst *, 4>>
1375	computeLocalUDChain(const MCInst *CurInstr, InstructionIterator Begin,
1376	InstructionIterator End) const {
1377	DenseMap<int, MCInst *> RegAliasTable;
1378	DenseMap<const MCInst *, SmallVector<MCInst *, 4>> Uses;
1379
1380	auto addInstrOperands = [&](const MCInst &Instr) {
1381	// Update Uses table
1382	for (const MCOperand &Operand : MCPlus::primeOperands(Inst: Instr)) {
1383	if (!Operand.isReg())
1384	continue;
1385	unsigned Reg = Operand.getReg();
1386	MCInst *AliasInst = RegAliasTable[Reg];
1387	Uses[&Instr].push_back(Elt: AliasInst);
1388	LLVM_DEBUG({
1389	dbgs() << "Adding reg operand " << Reg << " refs ";
1390	if (AliasInst != nullptr)
1391	AliasInst->dump();
1392	else
1393	dbgs() << "\n";
1394	});
1395	}
1396	};
1397
1398	LLVM_DEBUG(dbgs() << "computeLocalUDChain\n");
1399	bool TerminatorSeen = false;
1400	for (auto II = Begin; II != End; ++II) {
1401	MCInst &Instr = *II;
1402	// Ignore nops and CFIs
1403	if (isPseudo(Inst: Instr) || isNoop(Inst: Instr))
1404	continue;
1405	if (TerminatorSeen) {
1406	RegAliasTable.clear();
1407	Uses.clear();
1408	}
1409
1410	LLVM_DEBUG(dbgs() << "Now updating for:\n ");
1411	LLVM_DEBUG(Instr.dump());
1412	addInstrOperands(Instr);
1413
1414	BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
1415	getWrittenRegs(Inst: Instr, Regs);
1416
1417	// Update register definitions after this point
1418	for (int Idx : Regs.set_bits()) {
1419	RegAliasTable[Idx] = &Instr;
1420	LLVM_DEBUG(dbgs() << "Setting reg " << Idx
1421	<< " def to current instr.\n");
1422	}
1423
1424	TerminatorSeen = isTerminator(Inst: Instr);
1425	}
1426
1427	// Process the last instruction, which is not currently added into the
1428	// instruction stream
1429	if (CurInstr)
1430	addInstrOperands(*CurInstr);
1431
1432	return Uses;
1433	}
1434
1435	IndirectBranchType
1436	analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
1437	InstructionIterator End, const unsigned PtrSize,
1438	MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
1439	unsigned &IndexRegNumOut, int64_t &DispValueOut,
1440	const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
1441	MCInst *&FixedEntryLoadInstr) const override {
1442	MemLocInstrOut = nullptr;
1443	BaseRegNumOut = AArch64::NoRegister;
1444	IndexRegNumOut = AArch64::NoRegister;
1445	DispValueOut = 0;
1446	DispExprOut = nullptr;
1447	FixedEntryLoadInstr = nullptr;
1448
1449	// An instruction referencing memory used by jump instruction (directly or
1450	// via register). This location could be an array of function pointers
1451	// in case of indirect tail call, or a jump table.
1452	MCInst *MemLocInstr = nullptr;
1453
1454	// Analyze the memory location.
1455	int64_t ScaleValue, DispValue;
1456	const MCExpr *DispExpr;
1457
1458	DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
1459	computeLocalUDChain(CurInstr: &Instruction, Begin, End);
1460	MCInst *PCRelBase;
1461	if (!analyzeIndirectBranchFragment(Inst: Instruction, UDChain, JumpTable&: DispExpr,
1462	Offset&: DispValue, ScaleValue, PCRelBase))
1463	return IndirectBranchType::UNKNOWN;
1464
1465	MemLocInstrOut = MemLocInstr;
1466	DispValueOut = DispValue;
1467	DispExprOut = DispExpr;
1468	PCRelBaseOut = PCRelBase;
1469	return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE;
1470	}
1471
1472	/// Matches PLT entry pattern and returns the associated GOT entry address.
1473	/// Typical PLT entry looks like the following:
1474	///
1475	/// adrp x16, 230000
1476	/// ldr x17, [x16, #3040]
1477	/// add x16, x16, #0xbe0
1478	/// br x17
1479	///
1480	/// The other type of trampolines are located in .plt.got, that are used for
1481	/// non-lazy bindings so doesn't use x16 arg to transfer .got entry address:
1482	///
1483	/// adrp x16, 230000
1484	/// ldr x17, [x16, #3040]
1485	/// br x17
1486	/// nop
1487	///
1488	uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin,
1489	InstructionIterator End,
1490	uint64_t BeginPC) const override {
1491	// Check branch instruction
1492	MCInst *Branch = &Instruction;
1493	assert(Branch->getOpcode() == AArch64::BR && "Unexpected opcode");
1494
1495	DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =
1496	computeLocalUDChain(CurInstr: Branch, Begin, End);
1497
1498	// Match ldr instruction
1499	SmallVector<MCInst *, 4> &BranchUses = UDChain[Branch];
1500	if (BranchUses.size() < 1 || BranchUses[0] == nullptr)
1501	return 0;
1502
1503	// Check ldr instruction
1504	const MCInst *Ldr = BranchUses[0];
1505	if (Ldr->getOpcode() != AArch64::LDRXui)
1506	return 0;
1507
1508	// Get ldr value
1509	const unsigned ScaleLdr = 8; // LDRX operates on 8 bytes segments
1510	assert(Ldr->getOperand(2).isImm() && "Unexpected ldr operand");
1511	const uint64_t Offset = Ldr->getOperand(i: 2).getImm() * ScaleLdr;
1512
1513	// Match adrp instruction
1514	SmallVector<MCInst *, 4> &LdrUses = UDChain[Ldr];
1515	if (LdrUses.size() < 2 || LdrUses[1] == nullptr)
1516	return 0;
1517
1518	// Check adrp instruction
1519	MCInst *Adrp = LdrUses[1];
1520	if (Adrp->getOpcode() != AArch64::ADRP)
1521	return 0;
1522
1523	// Get adrp instruction PC
1524	const unsigned InstSize = 4;
1525	uint64_t AdrpPC = BeginPC;
1526	for (InstructionIterator It = Begin; It != End; ++It) {
1527	if (&(*It) == Adrp)
1528	break;
1529	AdrpPC += InstSize;
1530	}
1531
1532	// Get adrp value
1533	uint64_t Base;
1534	assert(Adrp->getOperand(1).isImm() && "Unexpected adrp operand");
1535	bool Ret = evaluateMemOperandTarget(Inst: *Adrp, Target&: Base, Address: AdrpPC, Size: InstSize);
1536	assert(Ret && "Failed to evaluate adrp");
1537	(void)Ret;
1538
1539	return Base + Offset;
1540	}
1541
1542	unsigned getInvertedBranchOpcode(unsigned Opcode) const {
1543	switch (Opcode) {
1544	default:
1545	llvm_unreachable("Failed to invert branch opcode");
1546	return Opcode;
1547	case AArch64::TBZW: return AArch64::TBNZW;
1548	case AArch64::TBZX: return AArch64::TBNZX;
1549	case AArch64::TBNZW: return AArch64::TBZW;
1550	case AArch64::TBNZX: return AArch64::TBZX;
1551	case AArch64::CBZW: return AArch64::CBNZW;
1552	case AArch64::CBZX: return AArch64::CBNZX;
1553	case AArch64::CBNZW: return AArch64::CBZW;
1554	case AArch64::CBNZX: return AArch64::CBZX;
1555	}
1556	}
1557
1558	unsigned getCondCode(const MCInst &Inst) const override {
1559	// AArch64 does not use conditional codes, so we just return the opcode
1560	// of the conditional branch here.
1561	return Inst.getOpcode();
1562	}
1563
1564	unsigned getCanonicalBranchCondCode(unsigned Opcode) const override {
1565	switch (Opcode) {
1566	default:
1567	return Opcode;
1568	case AArch64::TBNZW: return AArch64::TBZW;
1569	case AArch64::TBNZX: return AArch64::TBZX;
1570	case AArch64::CBNZW: return AArch64::CBZW;
1571	case AArch64::CBNZX: return AArch64::CBZX;
1572	}
1573	}
1574
1575	void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
1576	MCContext *Ctx) const override {
1577	if (isTB(Inst) || isCB(Inst)) {
1578	Inst.setOpcode(getInvertedBranchOpcode(Opcode: Inst.getOpcode()));
1579	assert(Inst.getOpcode() != 0 && "Invalid branch instruction");
1580	} else if (Inst.getOpcode() == AArch64::Bcc) {
1581	Inst.getOperand(i: 0).setImm(AArch64CC::getInvertedCondCode(
1582	Code: static_cast<AArch64CC::CondCode>(Inst.getOperand(i: 0).getImm())));
1583	assert(Inst.getOperand(0).getImm() != AArch64CC::AL &&
1584	Inst.getOperand(0).getImm() != AArch64CC::NV &&
1585	"Can't reverse ALWAYS cond code");
1586	} else {
1587	LLVM_DEBUG(Inst.dump());
1588	llvm_unreachable("Unrecognized branch instruction");
1589	}
1590	replaceBranchTarget(Inst, TBB, Ctx);
1591	}
1592
1593	int getPCRelEncodingSize(const MCInst &Inst) const override {
1594	switch (Inst.getOpcode()) {
1595	default:
1596	llvm_unreachable("Failed to get pcrel encoding size");
1597	return 0;
1598	case AArch64::TBZW: return 16;
1599	case AArch64::TBZX: return 16;
1600	case AArch64::TBNZW: return 16;
1601	case AArch64::TBNZX: return 16;
1602	case AArch64::CBZW: return 21;
1603	case AArch64::CBZX: return 21;
1604	case AArch64::CBNZW: return 21;
1605	case AArch64::CBNZX: return 21;
1606	case AArch64::B: return 28;
1607	case AArch64::BL: return 28;
1608	case AArch64::Bcc: return 21;
1609	}
1610	}
1611
1612	int getShortJmpEncodingSize() const override { return 33; }
1613
1614	int getUncondBranchEncodingSize() const override { return 28; }
1615
1616	// This helper function creates the snippet of code that compares a register
1617	// RegNo with an immedaite Imm, and jumps to Target if they are equal.
1618	// cmp RegNo, #Imm
1619	// b.eq Target
1620	// where cmp is an alias for subs, which results in the code below:
1621	// subs xzr, RegNo, #Imm
1622	// b.eq Target.
1623	InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
1624	const MCSymbol *Target,
1625	MCContext *Ctx) const override {
1626	InstructionListType Code;
1627	Code.emplace_back(args&: MCInstBuilder(AArch64::SUBSXri)
1628	.addReg(Reg: AArch64::XZR)
1629	.addReg(Reg: RegNo)
1630	.addImm(Val: Imm)
1631	.addImm(Val: 0));
1632	Code.emplace_back(args&: MCInstBuilder(AArch64::Bcc)
1633	.addImm(Val: AArch64CC::EQ)
1634	.addExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
1635	return Code;
1636	}
1637
1638	// This helper function creates the snippet of code that compares a register
1639	// RegNo with an immedaite Imm, and jumps to Target if they are not equal.
1640	// cmp RegNo, #Imm
1641	// b.ne Target
1642	// where cmp is an alias for subs, which results in the code below:
1643	// subs xzr, RegNo, #Imm
1644	// b.ne Target.
1645	InstructionListType createCmpJNE(MCPhysReg RegNo, int64_t Imm,
1646	const MCSymbol *Target,
1647	MCContext *Ctx) const override {
1648	InstructionListType Code;
1649	Code.emplace_back(args&: MCInstBuilder(AArch64::SUBSXri)
1650	.addReg(Reg: AArch64::XZR)
1651	.addReg(Reg: RegNo)
1652	.addImm(Val: Imm)
1653	.addImm(Val: 0));
1654	Code.emplace_back(args&: MCInstBuilder(AArch64::Bcc)
1655	.addImm(Val: AArch64CC::NE)
1656	.addExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
1657	return Code;
1658	}
1659
1660	void createTailCall(MCInst &Inst, const MCSymbol *Target,
1661	MCContext *Ctx) override {
1662	return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
1663	}
1664
1665	void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
1666	MCContext *Ctx) override {
1667	createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true);
1668	}
1669
1670	void createTrap(MCInst &Inst) const override {
1671	Inst.clear();
1672	Inst.setOpcode(AArch64::BRK);
1673	Inst.addOperand(Op: MCOperand::createImm(Val: 1));
1674	}
1675
1676	bool convertJmpToTailCall(MCInst &Inst) override {
1677	setTailCall(Inst);
1678	return true;
1679	}
1680
1681	bool convertTailCallToJmp(MCInst &Inst) override {
1682	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1683	clearOffset(Inst);
1684	if (getConditionalTailCall(Inst))
1685	unsetConditionalTailCall(Inst);
1686	return true;
1687	}
1688
1689	InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
1690	const MCSymbol *TargetLocation,
1691	MCContext *Ctx) override {
1692	const bool IsTailCall = isTailCall(Inst: DirectCall);
1693	assert((DirectCall.getOpcode() == AArch64::BL ||
1694	(DirectCall.getOpcode() == AArch64::B && IsTailCall)) &&
1695	"64-bit direct (tail) call instruction expected");
1696
1697	InstructionListType Code;
1698	// Code sequence for indirect plt call:
1699	// adrp x16 <symbol>
1700	// ldr x17, [x16, #<offset>]
1701	// blr x17 ; or 'br' for tail calls
1702
1703	MCInst InstAdrp;
1704	InstAdrp.setOpcode(AArch64::ADRP);
1705	InstAdrp.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
1706	InstAdrp.addOperand(Op: MCOperand::createImm(Val: 0));
1707	setOperandToSymbolRef(Inst&: InstAdrp, /* OpNum */ 1, Symbol: TargetLocation,
1708	/* Addend */ 0, Ctx, RelType: ELF::R_AARCH64_ADR_GOT_PAGE);
1709	Code.emplace_back(args&: InstAdrp);
1710
1711	MCInst InstLoad;
1712	InstLoad.setOpcode(AArch64::LDRXui);
1713	InstLoad.addOperand(Op: MCOperand::createReg(Reg: AArch64::X17));
1714	InstLoad.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
1715	InstLoad.addOperand(Op: MCOperand::createImm(Val: 0));
1716	setOperandToSymbolRef(Inst&: InstLoad, /* OpNum */ 2, Symbol: TargetLocation,
1717	/* Addend */ 0, Ctx, RelType: ELF::R_AARCH64_LD64_GOT_LO12_NC);
1718	Code.emplace_back(args&: InstLoad);
1719
1720	MCInst InstCall;
1721	InstCall.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);
1722	InstCall.addOperand(Op: MCOperand::createReg(Reg: AArch64::X17));
1723	moveAnnotations(SrcInst: std::move(DirectCall), DstInst&: InstCall);
1724	Code.emplace_back(args&: InstCall);
1725
1726	return Code;
1727	}
1728
1729	bool lowerTailCall(MCInst &Inst) override {
1730	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1731	if (getConditionalTailCall(Inst))
1732	unsetConditionalTailCall(Inst);
1733	return true;
1734	}
1735
1736	bool isNoop(const MCInst &Inst) const override {
1737	return Inst.getOpcode() == AArch64::HINT &&
1738	Inst.getOperand(i: 0).getImm() == 0;
1739	}
1740
1741	void createNoop(MCInst &Inst) const override {
1742	Inst.setOpcode(AArch64::HINT);
1743	Inst.clear();
1744	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
1745	}
1746
1747	bool isStorePair(const MCInst &Inst) const {
1748	const unsigned opcode = Inst.getOpcode();
1749
1750	auto isStorePairImmOffset = [&]() {
1751	switch (opcode) {
1752	case AArch64::STPWi:
1753	case AArch64::STPXi:
1754	case AArch64::STPSi:
1755	case AArch64::STPDi:
1756	case AArch64::STPQi:
1757	case AArch64::STNPWi:
1758	case AArch64::STNPXi:
1759	case AArch64::STNPSi:
1760	case AArch64::STNPDi:
1761	case AArch64::STNPQi:
1762	return true;
1763	default:
1764	break;
1765	}
1766
1767	return false;
1768	};
1769
1770	auto isStorePairPostIndex = [&]() {
1771	switch (opcode) {
1772	case AArch64::STPWpost:
1773	case AArch64::STPXpost:
1774	case AArch64::STPSpost:
1775	case AArch64::STPDpost:
1776	case AArch64::STPQpost:
1777	return true;
1778	default:
1779	break;
1780	}
1781
1782	return false;
1783	};
1784
1785	auto isStorePairPreIndex = [&]() {
1786	switch (opcode) {
1787	case AArch64::STPWpre:
1788	case AArch64::STPXpre:
1789	case AArch64::STPSpre:
1790	case AArch64::STPDpre:
1791	case AArch64::STPQpre:
1792	return true;
1793	default:
1794	break;
1795	}
1796
1797	return false;
1798	};
1799
1800	return isStorePairImmOffset() || isStorePairPostIndex() ||
1801	isStorePairPreIndex();
1802	}
1803
1804	bool isStoreReg(const MCInst &Inst) const {
1805	const unsigned opcode = Inst.getOpcode();
1806
1807	auto isStoreRegUnscaleImm = [&]() {
1808	switch (opcode) {
1809	case AArch64::STURBi:
1810	case AArch64::STURBBi:
1811	case AArch64::STURHi:
1812	case AArch64::STURHHi:
1813	case AArch64::STURWi:
1814	case AArch64::STURXi:
1815	case AArch64::STURSi:
1816	case AArch64::STURDi:
1817	case AArch64::STURQi:
1818	return true;
1819	default:
1820	break;
1821	}
1822
1823	return false;
1824	};
1825
1826	auto isStoreRegScaledImm = [&]() {
1827	switch (opcode) {
1828	case AArch64::STRBui:
1829	case AArch64::STRBBui:
1830	case AArch64::STRHui:
1831	case AArch64::STRHHui:
1832	case AArch64::STRWui:
1833	case AArch64::STRXui:
1834	case AArch64::STRSui:
1835	case AArch64::STRDui:
1836	case AArch64::STRQui:
1837	return true;
1838	default:
1839	break;
1840	}
1841
1842	return false;
1843	};
1844
1845	auto isStoreRegImmPostIndexed = [&]() {
1846	switch (opcode) {
1847	case AArch64::STRBpost:
1848	case AArch64::STRBBpost:
1849	case AArch64::STRHpost:
1850	case AArch64::STRHHpost:
1851	case AArch64::STRWpost:
1852	case AArch64::STRXpost:
1853	case AArch64::STRSpost:
1854	case AArch64::STRDpost:
1855	case AArch64::STRQpost:
1856	return true;
1857	default:
1858	break;
1859	}
1860
1861	return false;
1862	};
1863
1864	auto isStoreRegImmPreIndexed = [&]() {
1865	switch (opcode) {
1866	case AArch64::STRBpre:
1867	case AArch64::STRBBpre:
1868	case AArch64::STRHpre:
1869	case AArch64::STRHHpre:
1870	case AArch64::STRWpre:
1871	case AArch64::STRXpre:
1872	case AArch64::STRSpre:
1873	case AArch64::STRDpre:
1874	case AArch64::STRQpre:
1875	return true;
1876	default:
1877	break;
1878	}
1879
1880	return false;
1881	};
1882
1883	auto isStoreRegUnscaleUnpriv = [&]() {
1884	switch (opcode) {
1885	case AArch64::STTRBi:
1886	case AArch64::STTRHi:
1887	case AArch64::STTRWi:
1888	case AArch64::STTRXi:
1889	return true;
1890	default:
1891	break;
1892	}
1893
1894	return false;
1895	};
1896
1897	auto isStoreRegTrunc = [&]() {
1898	switch (opcode) {
1899	case AArch64::STRBBroW:
1900	case AArch64::STRBBroX:
1901	case AArch64::STRBroW:
1902	case AArch64::STRBroX:
1903	case AArch64::STRDroW:
1904	case AArch64::STRDroX:
1905	case AArch64::STRHHroW:
1906	case AArch64::STRHHroX:
1907	case AArch64::STRHroW:
1908	case AArch64::STRHroX:
1909	case AArch64::STRQroW:
1910	case AArch64::STRQroX:
1911	case AArch64::STRSroW:
1912	case AArch64::STRSroX:
1913	case AArch64::STRWroW:
1914	case AArch64::STRWroX:
1915	case AArch64::STRXroW:
1916	case AArch64::STRXroX:
1917	return true;
1918	default:
1919	break;
1920	}
1921
1922	return false;
1923	};
1924
1925	return isStoreRegUnscaleImm() || isStoreRegScaledImm() ||
1926	isStoreRegImmPreIndexed() || isStoreRegImmPostIndexed() ||
1927	isStoreRegUnscaleUnpriv() || isStoreRegTrunc();
1928	}
1929
1930	bool mayStore(const MCInst &Inst) const override {
1931	return isStorePair(Inst) || isStoreReg(Inst) ||
1932	isAArch64ExclusiveStore(Inst);
1933	}
1934
1935	bool isStoreToStack(const MCInst &Inst) const {
1936	if (!mayStore(Inst))
1937	return false;
1938
1939	for (const MCOperand &Operand : useOperands(Inst)) {
1940	if (!Operand.isReg())
1941	continue;
1942
1943	unsigned Reg = Operand.getReg();
1944	if (Reg == AArch64::SP || Reg == AArch64::WSP)
1945	return true;
1946	}
1947
1948	return false;
1949	}
1950
1951	void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
1952	bool IsTailCall) override {
1953	Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL);
1954	Inst.clear();
1955	Inst.addOperand(Op: MCOperand::createExpr(Val: getTargetExprFor(
1956	Inst, Expr: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx), Ctx&: *Ctx, RelType: 0)));
1957	if (IsTailCall)
1958	convertJmpToTailCall(Inst);
1959	}
1960
1961	bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1962	const MCSymbol *&TBB, const MCSymbol *&FBB,
1963	MCInst *&CondBranch,
1964	MCInst *&UncondBranch) const override {
1965	auto I = End;
1966
1967	while (I != Begin) {
1968	--I;
1969
1970	// Ignore nops and CFIs
1971	if (isPseudo(Inst: *I) || isNoop(Inst: *I))
1972	continue;
1973
1974	// Stop when we find the first non-terminator
1975	if (!isTerminator(Inst: *I) || isTailCall(Inst: *I) || !isBranch(Inst: *I))
1976	break;
1977
1978	// Handle unconditional branches.
1979	if (isUnconditionalBranch(Inst: *I)) {
1980	// If any code was seen after this unconditional branch, we've seen
1981	// unreachable code. Ignore them.
1982	CondBranch = nullptr;
1983	UncondBranch = &*I;
1984	const MCSymbol *Sym = getTargetSymbol(Inst: *I);
1985	assert(Sym != nullptr &&
1986	"Couldn't extract BB symbol from jump operand");
1987	TBB = Sym;
1988	continue;
1989	}
1990
1991	// Handle conditional branches and ignore indirect branches
1992	if (isIndirectBranch(Inst: *I))
1993	return false;
1994
1995	if (CondBranch == nullptr) {
1996	const MCSymbol *TargetBB = getTargetSymbol(Inst: *I);
1997	if (TargetBB == nullptr) {
1998	// Unrecognized branch target
1999	return false;
2000	}
2001	FBB = TBB;
2002	TBB = TargetBB;
2003	CondBranch = &*I;
2004	continue;
2005	}
2006
2007	llvm_unreachable("multiple conditional branches in one BB");
2008	}
2009	return true;
2010	}
2011
2012	void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,
2013	MCContext *Ctx, bool IsTailCall) override {
2014	// ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
2015	// Standard for the ARM 64-bit Architecture (AArch64)".
2016	// The sequence of instructions we create here is the following:
2017	// movz ip0, #:abs_g3:<addr>
2018	// movk ip0, #:abs_g2_nc:<addr>
2019	// movk ip0, #:abs_g1_nc:<addr>
2020	// movk ip0, #:abs_g0_nc:<addr>
2021	// br ip0
2022	MCInst Inst;
2023	Inst.setOpcode(AArch64::MOVZXi);
2024	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2025	Inst.addOperand(Op: MCOperand::createExpr(
2026	Val: MCSpecifierExpr::create(Sym: Target, S: AArch64::S_ABS_G3, Ctx&: *Ctx)));
2027	Inst.addOperand(Op: MCOperand::createImm(Val: 0x30));
2028	Seq.emplace_back(args&: Inst);
2029
2030	Inst.clear();
2031	Inst.setOpcode(AArch64::MOVKXi);
2032	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2033	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2034	Inst.addOperand(Op: MCOperand::createExpr(
2035	Val: MCSpecifierExpr::create(Sym: Target, S: AArch64::S_ABS_G2_NC, Ctx&: *Ctx)));
2036	Inst.addOperand(Op: MCOperand::createImm(Val: 0x20));
2037	Seq.emplace_back(args&: Inst);
2038
2039	Inst.clear();
2040	Inst.setOpcode(AArch64::MOVKXi);
2041	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2042	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2043	Inst.addOperand(Op: MCOperand::createExpr(
2044	Val: MCSpecifierExpr::create(Sym: Target, S: AArch64::S_ABS_G1_NC, Ctx&: *Ctx)));
2045	Inst.addOperand(Op: MCOperand::createImm(Val: 0x10));
2046	Seq.emplace_back(args&: Inst);
2047
2048	Inst.clear();
2049	Inst.setOpcode(AArch64::MOVKXi);
2050	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2051	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2052	Inst.addOperand(Op: MCOperand::createExpr(
2053	Val: MCSpecifierExpr::create(Sym: Target, S: AArch64::S_ABS_G0_NC, Ctx&: *Ctx)));
2054	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
2055	Seq.emplace_back(args&: Inst);
2056
2057	Inst.clear();
2058	Inst.setOpcode(AArch64::BR);
2059	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::X16));
2060	if (IsTailCall)
2061	setTailCall(Inst);
2062	Seq.emplace_back(args&: Inst);
2063	}
2064
2065	void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
2066	MCContext *Ctx, bool IsTailCall) override {
2067	// ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call
2068	// Standard for the ARM 64-bit Architecture (AArch64)".
2069	// The sequence of instructions we create here is the following:
2070	// adrp ip0, imm
2071	// add ip0, ip0, imm
2072	// br ip0
2073	MCPhysReg Reg = AArch64::X16;
2074	InstructionListType Insts = materializeAddress(Target, Ctx, RegName: Reg);
2075	Insts.emplace_back();
2076	MCInst &Inst = Insts.back();
2077	Inst.clear();
2078	Inst.setOpcode(AArch64::BR);
2079	Inst.addOperand(Op: MCOperand::createReg(Reg));
2080	if (IsTailCall)
2081	setTailCall(Inst);
2082	Seq.swap(x&: Insts);
2083	}
2084
2085	/// Matching pattern here is
2086	///
2087	/// ADRP x16, imm
2088	/// ADD x16, x16, imm
2089	/// BR x16
2090	///
2091	uint64_t matchLinkerVeneer(InstructionIterator Begin, InstructionIterator End,
2092	uint64_t Address, const MCInst &CurInst,
2093	MCInst *&TargetHiBits, MCInst *&TargetLowBits,
2094	uint64_t &Target) const override {
2095	if (CurInst.getOpcode() != AArch64::BR || !CurInst.getOperand(i: 0).isReg() ||
2096	CurInst.getOperand(i: 0).getReg() != AArch64::X16)
2097	return 0;
2098
2099	auto I = End;
2100	if (I == Begin)
2101	return 0;
2102
2103	--I;
2104	Address -= 4;
2105	if (I == Begin || I->getOpcode() != AArch64::ADDXri ||
2106	MCPlus::getNumPrimeOperands(Inst: *I) < 3 || !I->getOperand(i: 0).isReg() ||
2107	!I->getOperand(i: 1).isReg() ||
2108	I->getOperand(i: 0).getReg() != AArch64::X16 ||
2109	I->getOperand(i: 1).getReg() != AArch64::X16 || !I->getOperand(i: 2).isImm())
2110	return 0;
2111	TargetLowBits = &*I;
2112	uint64_t Addr = I->getOperand(i: 2).getImm() & 0xFFF;
2113
2114	--I;
2115	Address -= 4;
2116	if (I->getOpcode() != AArch64::ADRP ||
2117	MCPlus::getNumPrimeOperands(Inst: *I) < 2 || !I->getOperand(i: 0).isReg() ||
2118	!I->getOperand(i: 1).isImm() || I->getOperand(i: 0).getReg() != AArch64::X16)
2119	return 0;
2120	TargetHiBits = &*I;
2121	Addr |= (Address + ((int64_t)I->getOperand(i: 1).getImm() << 12)) &
2122	0xFFFFFFFFFFFFF000ULL;
2123	Target = Addr;
2124	return 3;
2125	}
2126
2127	/// Match the following pattern:
2128	///
2129	/// LDR x16, .L1
2130	/// BR x16
2131	/// L1:
2132	/// .quad Target
2133	///
2134	/// Populate \p TargetAddress with the Target value on successful match.
2135	bool matchAbsLongVeneer(const BinaryFunction &BF,
2136	uint64_t &TargetAddress) const override {
2137	if (BF.size() != 1 || BF.getMaxSize() < 16)
2138	return false;
2139
2140	if (!BF.hasConstantIsland())
2141	return false;
2142
2143	const BinaryBasicBlock &BB = BF.front();
2144	if (BB.size() != 2)
2145	return false;
2146
2147	const MCInst &LDRInst = BB.getInstructionAtIndex(Index: 0);
2148	if (LDRInst.getOpcode() != AArch64::LDRXl)
2149	return false;
2150
2151	if (!LDRInst.getOperand(i: 0).isReg() ||
2152	LDRInst.getOperand(i: 0).getReg() != AArch64::X16)
2153	return false;
2154
2155	const MCSymbol *TargetSym = getTargetSymbol(Inst: LDRInst, OpNum: 1);
2156	if (!TargetSym)
2157	return false;
2158
2159	const MCInst &BRInst = BB.getInstructionAtIndex(Index: 1);
2160	if (BRInst.getOpcode() != AArch64::BR)
2161	return false;
2162	if (!BRInst.getOperand(i: 0).isReg() ||
2163	BRInst.getOperand(i: 0).getReg() != AArch64::X16)
2164	return false;
2165
2166	const BinaryFunction::IslandInfo &IInfo = BF.getIslandInfo();
2167	if (IInfo.HasDynamicRelocations)
2168	return false;
2169
2170	auto Iter = IInfo.Offsets.find(x: 8);
2171	if (Iter == IInfo.Offsets.end() || Iter->second != TargetSym)
2172	return false;
2173
2174	// Extract the absolute value stored inside the island.
2175	StringRef SectionContents = BF.getOriginSection()->getContents();
2176	StringRef FunctionContents = SectionContents.substr(
2177	Start: BF.getAddress() - BF.getOriginSection()->getAddress(), N: BF.getMaxSize());
2178
2179	const BinaryContext &BC = BF.getBinaryContext();
2180	DataExtractor DE(FunctionContents, BC.AsmInfo->isLittleEndian(),
2181	BC.AsmInfo->getCodePointerSize());
2182	uint64_t Offset = 8;
2183	TargetAddress = DE.getAddress(offset_ptr: &Offset);
2184
2185	return true;
2186	}
2187
2188	bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const override {
2189	if (!isADRP(Inst: Adrp) || !isAddXri(Inst: Add))
2190	return false;
2191
2192	assert(Adrp.getOperand(0).isReg() &&
2193	"Unexpected operand in ADRP instruction");
2194	MCPhysReg AdrpReg = Adrp.getOperand(i: 0).getReg();
2195	assert(Add.getOperand(1).isReg() &&
2196	"Unexpected operand in ADDXri instruction");
2197	MCPhysReg AddReg = Add.getOperand(i: 1).getReg();
2198	return AdrpReg == AddReg;
2199	}
2200
2201	bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2202	int64_t Addend, MCContext *Ctx, int64_t &Value,
2203	uint32_t RelType) const override {
2204	unsigned ImmOpNo = -1U;
2205	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2206	++Index) {
2207	if (Inst.getOperand(i: Index).isImm()) {
2208	ImmOpNo = Index;
2209	break;
2210	}
2211	}
2212	if (ImmOpNo == -1U)
2213	return false;
2214
2215	Value = Inst.getOperand(i: ImmOpNo).getImm();
2216
2217	setOperandToSymbolRef(Inst, OpNum: ImmOpNo, Symbol, Addend, Ctx, RelType);
2218
2219	return true;
2220	}
2221
2222	void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2223	MCContext *Ctx) const override {
2224	Inst.setOpcode(AArch64::B);
2225	Inst.clear();
2226	Inst.addOperand(Op: MCOperand::createExpr(
2227	Val: getTargetExprFor(Inst, Expr: MCSymbolRefExpr::create(Symbol: TBB, Ctx&: *Ctx), Ctx&: *Ctx, RelType: 0)));
2228	}
2229
2230	bool shouldRecordCodeRelocation(uint32_t RelType) const override {
2231	switch (RelType) {
2232	case ELF::R_AARCH64_ABS64:
2233	case ELF::R_AARCH64_ABS32:
2234	case ELF::R_AARCH64_ABS16:
2235	case ELF::R_AARCH64_ADD_ABS_LO12_NC:
2236	case ELF::R_AARCH64_ADR_GOT_PAGE:
2237	case ELF::R_AARCH64_ADR_PREL_LO21:
2238	case ELF::R_AARCH64_ADR_PREL_PG_HI21:
2239	case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
2240	case ELF::R_AARCH64_LD64_GOT_LO12_NC:
2241	case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
2242	case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
2243	case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
2244	case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
2245	case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
2246	case ELF::R_AARCH64_TLSDESC_ADD_LO12:
2247	case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
2248	case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
2249	case ELF::R_AARCH64_TLSDESC_LD64_LO12:
2250	case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
2251	case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
2252	case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0:
2253	case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
2254	case ELF::R_AARCH64_MOVW_UABS_G0:
2255	case ELF::R_AARCH64_MOVW_UABS_G0_NC:
2256	case ELF::R_AARCH64_MOVW_UABS_G1:
2257	case ELF::R_AARCH64_MOVW_UABS_G1_NC:
2258	case ELF::R_AARCH64_MOVW_UABS_G2:
2259	case ELF::R_AARCH64_MOVW_UABS_G2_NC:
2260	case ELF::R_AARCH64_MOVW_UABS_G3:
2261	case ELF::R_AARCH64_PREL16:
2262	case ELF::R_AARCH64_PREL32:
2263	case ELF::R_AARCH64_PREL64:
2264	return true;
2265	case ELF::R_AARCH64_CALL26:
2266	case ELF::R_AARCH64_JUMP26:
2267	case ELF::R_AARCH64_TSTBR14:
2268	case ELF::R_AARCH64_CONDBR19:
2269	case ELF::R_AARCH64_TLSDESC_CALL:
2270	case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
2271	case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
2272	return false;
2273	default:
2274	llvm_unreachable("Unexpected AArch64 relocation type in code");
2275	}
2276	}
2277
2278	StringRef getTrapFillValue() const override {
2279	return StringRef("\0\0\0\0", 4);
2280	}
2281
2282	void createReturn(MCInst &Inst) const override {
2283	Inst.setOpcode(AArch64::RET);
2284	Inst.clear();
2285	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::LR));
2286	}
2287
2288	void createStackPointerIncrement(
2289	MCInst &Inst, int Size,
2290	bool NoFlagsClobber = false /*unused for AArch64*/) const override {
2291	Inst.setOpcode(AArch64::SUBXri);
2292	Inst.clear();
2293	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
2294	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
2295	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2296	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
2297	}
2298
2299	void createStackPointerDecrement(
2300	MCInst &Inst, int Size,
2301	bool NoFlagsClobber = false /*unused for AArch64*/) const override {
2302	Inst.setOpcode(AArch64::ADDXri);
2303	Inst.clear();
2304	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
2305	Inst.addOperand(Op: MCOperand::createReg(Reg: AArch64::SP));
2306	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2307	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
2308	}
2309
2310	void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
2311	int64_t Disp) const {
2312	Inst.setOpcode(AArch64::BR);
2313	Inst.clear();
2314	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg));
2315	}
2316
2317	InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
2318	InstructionListType Insts(5);
2319	// Code sequence for instrumented indirect call handler:
2320	// msr nzcv, x1
2321	// ldp x0, x1, [sp], #16
2322	// ldr x16, [sp], #16
2323	// ldp x0, x1, [sp], #16
2324	// br x16
2325	setSystemFlag(Inst&: Insts[0], RegName: AArch64::X1);
2326	createPopRegisters(Inst&: Insts[1], Reg1: AArch64::X0, Reg2: AArch64::X1);
2327	// Here we load address of the next function which should be called in the
2328	// original binary to X16 register. Writing to X16 is permitted without
2329	// needing to restore.
2330	loadReg(Inst&: Insts[2], To: AArch64::X16, From: AArch64::SP);
2331	createPopRegisters(Inst&: Insts[3], Reg1: AArch64::X0, Reg2: AArch64::X1);
2332	createIndirectBranch(Inst&: Insts[4], MemBaseReg: AArch64::X16, Disp: 0);
2333	return Insts;
2334	}
2335
2336	InstructionListType
2337	createInstrumentedIndTailCallHandlerExitBB() const override {
2338	return createInstrumentedIndCallHandlerExitBB();
2339	}
2340
2341	InstructionListType createGetter(MCContext *Ctx, const char *name) const {
2342	InstructionListType Insts(4);
2343	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: name);
2344	InstructionListType Addr = materializeAddress(Target: Locs, Ctx, RegName: AArch64::X0);
2345	std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.begin());
2346	assert(Addr.size() == 2 && "Invalid Addr size");
2347	loadReg(Inst&: Insts[2], To: AArch64::X0, From: AArch64::X0);
2348	createReturn(Inst&: Insts[3]);
2349	return Insts;
2350	}
2351
2352	InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
2353	return createGetter(Ctx, name: "__bolt_num_counters");
2354	}
2355
2356	InstructionListType
2357	createInstrLocationsGetter(MCContext *Ctx) const override {
2358	return createGetter(Ctx, name: "__bolt_instr_locations");
2359	}
2360
2361	InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
2362	return createGetter(Ctx, name: "__bolt_instr_tables");
2363	}
2364
2365	InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
2366	return createGetter(Ctx, name: "__bolt_instr_num_funcs");
2367	}
2368
2369	void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
2370	bool IsTailCall = isTailCall(Inst);
2371	if (IsTailCall)
2372	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
2373	if (Inst.getOpcode() == AArch64::BR || Inst.getOpcode() == AArch64::BLR) {
2374	Inst.setOpcode(AArch64::ORRXrs);
2375	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
2376	Inst.insert(I: Inst.begin() + 1, Op: MCOperand::createReg(Reg: AArch64::XZR));
2377	Inst.insert(I: Inst.begin() + 3, Op: MCOperand::createImm(Val: 0));
2378	return;
2379	}
2380	llvm_unreachable("not implemented");
2381	}
2382
2383	InstructionListType createLoadImmediate(const MCPhysReg Dest,
2384	uint64_t Imm) const override {
2385	InstructionListType Insts(4);
2386	int Shift = 48;
2387	for (int I = 0; I < 4; I++, Shift -= 16) {
2388	Insts[I].setOpcode(AArch64::MOVKXi);
2389	Insts[I].addOperand(Op: MCOperand::createReg(Reg: Dest));
2390	Insts[I].addOperand(Op: MCOperand::createReg(Reg: Dest));
2391	Insts[I].addOperand(Op: MCOperand::createImm(Val: (Imm >> Shift) & 0xFFFF));
2392	Insts[I].addOperand(Op: MCOperand::createImm(Val: Shift));
2393	}
2394	return Insts;
2395	}
2396
2397	void createIndirectCallInst(MCInst &Inst, bool IsTailCall,
2398	MCPhysReg Reg) const {
2399	Inst.clear();
2400	Inst.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);
2401	Inst.addOperand(Op: MCOperand::createReg(Reg));
2402	}
2403
2404	InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
2405	MCSymbol *HandlerFuncAddr,
2406	int CallSiteID,
2407	MCContext *Ctx) override {
2408	InstructionListType Insts;
2409	// Code sequence used to enter indirect call instrumentation helper:
2410	// stp x0, x1, [sp, #-16]! createPushRegisters
2411	// mov target x0 convertIndirectCallToLoad -> orr x0 target xzr
2412	// mov x1 CallSiteID createLoadImmediate ->
2413	// movk x1, #0x0, lsl #48
2414	// movk x1, #0x0, lsl #32
2415	// movk x1, #0x0, lsl #16
2416	// movk x1, #0x0
2417	// stp x0, x1, [sp, #-16]!
2418	// bl *HandlerFuncAddr createIndirectCall ->
2419	// adr x0 *HandlerFuncAddr -> adrp + add
2420	// blr x0
2421	Insts.emplace_back();
2422	createPushRegisters(Inst&: Insts.back(), Reg1: AArch64::X0, Reg2: AArch64::X1);
2423	Insts.emplace_back(args&: CallInst);
2424	convertIndirectCallToLoad(Inst&: Insts.back(), Reg: AArch64::X0);
2425	InstructionListType LoadImm =
2426	createLoadImmediate(Dest: getIntArgRegister(ArgNo: 1), Imm: CallSiteID);
2427	Insts.insert(position: Insts.end(), first: LoadImm.begin(), last: LoadImm.end());
2428	Insts.emplace_back();
2429	createPushRegisters(Inst&: Insts.back(), Reg1: AArch64::X0, Reg2: AArch64::X1);
2430	Insts.resize(new_size: Insts.size() + 2);
2431	InstructionListType Addr =
2432	materializeAddress(Target: HandlerFuncAddr, Ctx, RegName: AArch64::X0);
2433	assert(Addr.size() == 2 && "Invalid Addr size");
2434	std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.end() - Addr.size());
2435	Insts.emplace_back();
2436	createIndirectCallInst(Inst&: Insts.back(), IsTailCall: isTailCall(Inst: CallInst), Reg: AArch64::X0);
2437
2438	// Carry over metadata including tail call marker if present.
2439	stripAnnotations(Inst&: Insts.back());
2440	moveAnnotations(SrcInst: std::move(CallInst), DstInst&: Insts.back());
2441
2442	return Insts;
2443	}
2444
2445	InstructionListType
2446	createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
2447	const MCSymbol *IndCallHandler,
2448	MCContext *Ctx) override {
2449	// Code sequence used to check whether InstrTampoline was initialized
2450	// and call it if so, returns via IndCallHandler
2451	// stp x0, x1, [sp, #-16]!
2452	// mrs x1, nzcv
2453	// adr x0, InstrTrampoline -> adrp + add
2454	// ldr x0, [x0]
2455	// subs x0, x0, #0x0
2456	// b.eq IndCallHandler
2457	// str x30, [sp, #-16]!
2458	// blr x0
2459	// ldr x30, [sp], #16
2460	// b IndCallHandler
2461	InstructionListType Insts;
2462	Insts.emplace_back();
2463	createPushRegisters(Inst&: Insts.back(), Reg1: AArch64::X0, Reg2: AArch64::X1);
2464	Insts.emplace_back();
2465	getSystemFlag(Inst&: Insts.back(), RegName: getIntArgRegister(ArgNo: 1));
2466	Insts.emplace_back();
2467	Insts.emplace_back();
2468	InstructionListType Addr =
2469	materializeAddress(Target: InstrTrampoline, Ctx, RegName: AArch64::X0);
2470	std::copy(first: Addr.begin(), last: Addr.end(), result: Insts.end() - Addr.size());
2471	assert(Addr.size() == 2 && "Invalid Addr size");
2472	Insts.emplace_back();
2473	loadReg(Inst&: Insts.back(), To: AArch64::X0, From: AArch64::X0);
2474	InstructionListType cmpJmp =
2475	createCmpJE(RegNo: AArch64::X0, Imm: 0, Target: IndCallHandler, Ctx);
2476	Insts.insert(position: Insts.end(), first: cmpJmp.begin(), last: cmpJmp.end());
2477	Insts.emplace_back();
2478	storeReg(Inst&: Insts.back(), From: AArch64::LR, To: AArch64::SP);
2479	Insts.emplace_back();
2480	Insts.back().setOpcode(AArch64::BLR);
2481	Insts.back().addOperand(Op: MCOperand::createReg(Reg: AArch64::X0));
2482	Insts.emplace_back();
2483	loadReg(Inst&: Insts.back(), To: AArch64::LR, From: AArch64::SP);
2484	Insts.emplace_back();
2485	createDirectCall(Inst&: Insts.back(), Target: IndCallHandler, Ctx, /*IsTailCall*/ true);
2486	return Insts;
2487	}
2488
2489	InstructionListType
2490	createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
2491	unsigned CodePointerSize) const override {
2492	unsigned int I = 0;
2493	InstructionListType Instrs(IsLeaf ? 12 : 10);
2494
2495	if (IsLeaf)
2496	createStackPointerIncrement(Inst&: Instrs[I++], Size: 128);
2497	createPushRegisters(Inst&: Instrs[I++], Reg1: AArch64::X0, Reg2: AArch64::X1);
2498	getSystemFlag(Inst&: Instrs[I++], RegName: AArch64::X1);
2499	InstructionListType Addr = materializeAddress(Target, Ctx, RegName: AArch64::X0);
2500	assert(Addr.size() == 2 && "Invalid Addr size");
2501	std::copy(first: Addr.begin(), last: Addr.end(), result: Instrs.begin() + I);
2502	I += Addr.size();
2503	storeReg(Inst&: Instrs[I++], From: AArch64::X2, To: AArch64::SP);
2504	InstructionListType Insts = createIncMemory(RegTo: AArch64::X0, RegTmp: AArch64::X2);
2505	assert(Insts.size() == 2 && "Invalid Insts size");
2506	std::copy(first: Insts.begin(), last: Insts.end(), result: Instrs.begin() + I);
2507	I += Insts.size();
2508	loadReg(Inst&: Instrs[I++], To: AArch64::X2, From: AArch64::SP);
2509	setSystemFlag(Inst&: Instrs[I++], RegName: AArch64::X1);
2510	createPopRegisters(Inst&: Instrs[I++], Reg1: AArch64::X0, Reg2: AArch64::X1);
2511	if (IsLeaf)
2512	createStackPointerDecrement(Inst&: Instrs[I++], Size: 128);
2513	return Instrs;
2514	}
2515
2516	std::vector<MCInst> createSymbolTrampoline(const MCSymbol *TgtSym,
2517	MCContext *Ctx) override {
2518	std::vector<MCInst> Insts;
2519	createShortJmp(Seq&: Insts, Target: TgtSym, Ctx, /*IsTailCall*/ true);
2520	return Insts;
2521	}
2522
2523	InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx,
2524	MCPhysReg RegName,
2525	int64_t Addend = 0) const override {
2526	// Get page-aligned address and add page offset
2527	InstructionListType Insts(2);
2528	Insts[0].setOpcode(AArch64::ADRP);
2529	Insts[0].clear();
2530	Insts[0].addOperand(Op: MCOperand::createReg(Reg: RegName));
2531	Insts[0].addOperand(Op: MCOperand::createImm(Val: 0));
2532	setOperandToSymbolRef(Inst&: Insts[0], /* OpNum */ 1, Symbol: Target, Addend, Ctx,
2533	RelType: ELF::R_AARCH64_NONE);
2534	Insts[1].setOpcode(AArch64::ADDXri);
2535	Insts[1].clear();
2536	Insts[1].addOperand(Op: MCOperand::createReg(Reg: RegName));
2537	Insts[1].addOperand(Op: MCOperand::createReg(Reg: RegName));
2538	Insts[1].addOperand(Op: MCOperand::createImm(Val: 0));
2539	Insts[1].addOperand(Op: MCOperand::createImm(Val: 0));
2540	setOperandToSymbolRef(Inst&: Insts[1], /* OpNum */ 2, Symbol: Target, Addend, Ctx,
2541	RelType: ELF::R_AARCH64_ADD_ABS_LO12_NC);
2542	return Insts;
2543	}
2544
2545	std::optional<Relocation>
2546	createRelocation(const MCFixup &Fixup,
2547	const MCAsmBackend &MAB) const override {
2548	MCFixupKindInfo FKI = MAB.getFixupKindInfo(Kind: Fixup.getKind());
2549
2550	assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2551	const uint64_t RelOffset = Fixup.getOffset();
2552
2553	uint32_t RelType;
2554	if (Fixup.getKind() == MCFixupKind(AArch64::fixup_aarch64_pcrel_call26))
2555	RelType = ELF::R_AARCH64_CALL26;
2556	else if (Fixup.getKind() ==
2557	MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26))
2558	RelType = ELF::R_AARCH64_JUMP26;
2559	else if (Fixup.isPCRel()) {
2560	switch (FKI.TargetSize) {
2561	default:
2562	return std::nullopt;
2563	case 16:
2564	RelType = ELF::R_AARCH64_PREL16;
2565	break;
2566	case 32:
2567	RelType = ELF::R_AARCH64_PREL32;
2568	break;
2569	case 64:
2570	RelType = ELF::R_AARCH64_PREL64;
2571	break;
2572	}
2573	} else {
2574	switch (FKI.TargetSize) {
2575	default:
2576	return std::nullopt;
2577	case 16:
2578	RelType = ELF::R_AARCH64_ABS16;
2579	break;
2580	case 32:
2581	RelType = ELF::R_AARCH64_ABS32;
2582	break;
2583	case 64:
2584	RelType = ELF::R_AARCH64_ABS64;
2585	break;
2586	}
2587	}
2588
2589	auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
2590
2591	return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
2592	}
2593
2594	uint16_t getMinFunctionAlignment() const override { return 4; }
2595
2596	std::optional<uint32_t>
2597	getInstructionSize(const MCInst &Inst) const override {
2598	return 4;
2599	}
2600	};
2601
2602	} // end anonymous namespace
2603
2604	namespace llvm {
2605	namespace bolt {
2606
2607	MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis,
2608	const MCInstrInfo *Info,
2609	const MCRegisterInfo *RegInfo,
2610	const MCSubtargetInfo *STI) {
2611	return new AArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
2612	}
2613
2614	} // namespace bolt
2615	} // namespace llvm
2616