1	//===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file provides X86-specific MCPlus builder.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "MCTargetDesc/X86BaseInfo.h"
14	#include "MCTargetDesc/X86EncodingOptimization.h"
15	#include "MCTargetDesc/X86MCTargetDesc.h"
16	#include "X86MCSymbolizer.h"
17	#include "bolt/Core/MCPlus.h"
18	#include "bolt/Core/MCPlusBuilder.h"
19	#include "llvm/BinaryFormat/ELF.h"
20	#include "llvm/MC/MCContext.h"
21	#include "llvm/MC/MCFixupKindInfo.h"
22	#include "llvm/MC/MCInst.h"
23	#include "llvm/MC/MCInstBuilder.h"
24	#include "llvm/MC/MCInstrInfo.h"
25	#include "llvm/MC/MCRegister.h"
26	#include "llvm/MC/MCRegisterInfo.h"
27	#include "llvm/Support/CommandLine.h"
28	#include "llvm/Support/DataExtractor.h"
29	#include "llvm/Support/Debug.h"
30	#include "llvm/Support/Errc.h"
31	#include "llvm/Support/ErrorHandling.h"
32	#include "llvm/Support/ErrorOr.h"
33	#include <set>
34
35	#define DEBUG_TYPE "mcplus"
36
37	using namespace llvm;
38	using namespace bolt;
39
40	namespace opts {
41
42	extern cl::OptionCategory BoltOptCategory;
43
44	static cl::opt<bool> X86StripRedundantAddressSize(
45	"x86-strip-redundant-address-size",
46	cl::desc("Remove redundant Address-Size override prefix"), cl::init(Val: true),
47	cl::cat(BoltOptCategory));
48
49	} // namespace opts
50
51	namespace {
52
53	bool isMOVSX64rm32(const MCInst &Inst) {
54	return Inst.getOpcode() == X86::MOVSX64rm32;
55	}
56
57	bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; }
58
59	bool isADDri(const MCInst &Inst) {
60	return Inst.getOpcode() == X86::ADD64ri32 ||
61	Inst.getOpcode() == X86::ADD64ri8;
62	}
63
64	// Create instruction to increment contents of target by 1
65	static InstructionListType createIncMemory(const MCSymbol *Target,
66	MCContext *Ctx) {
67	InstructionListType Insts;
68	Insts.emplace_back();
69	Insts.back().setOpcode(X86::LOCK_INC64m);
70	Insts.back().clear();
71	Insts.back().addOperand(Op: MCOperand::createReg(X86::Reg: RIP)); // BaseReg
72	Insts.back().addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
73	Insts.back().addOperand(Op: MCOperand::createReg(X86::Reg: NoRegister)); // IndexReg
74
75	Insts.back().addOperand(Op: MCOperand::createExpr(
76	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None,
77	Ctx&: *Ctx))); // Displacement
78	Insts.back().addOperand(
79	Op: MCOperand::createReg(X86::Reg: NoRegister)); // AddrSegmentReg
80	return Insts;
81	}
82
83	#define GET_INSTRINFO_OPERAND_TYPES_ENUM
84	#define GET_INSTRINFO_OPERAND_TYPE
85	#define GET_INSTRINFO_MEM_OPERAND_SIZE
86	#include "X86GenInstrInfo.inc"
87
88	class X86MCPlusBuilder : public MCPlusBuilder {
89	public:
90	using MCPlusBuilder::MCPlusBuilder;
91
92	std::unique_ptr<MCSymbolizer>
93	createTargetSymbolizer(BinaryFunction &Function,
94	bool CreateNewSymbols) const override {
95	return std::make_unique<X86MCSymbolizer>(Function, CreateNewSymbols);
96	}
97
98	bool isBranch(const MCInst &Inst) const override {
99	return Analysis->isBranch(Inst) && !isTailCall(Inst);
100	}
101
102	bool isNoop(const MCInst &Inst) const override {
103	return X86::isNOP(Inst.getOpcode());
104	}
105
106	unsigned getCondCode(const MCInst &Inst) const override {
107	unsigned Opcode = Inst.getOpcode();
108	if (X86::isJCC(Opcode))
109	return Inst.getOperand(i: Info->get(Opcode).NumOperands - 1).getImm();
110	return X86::COND_INVALID;
111	}
112
113	unsigned getInvertedCondCode(unsigned CC) const override {
114	switch (CC) {
115	default: return X86::COND_INVALID;
116	case X86::COND_E: return X86::COND_NE;
117	case X86::COND_NE: return X86::COND_E;
118	case X86::COND_L: return X86::COND_GE;
119	case X86::COND_LE: return X86::COND_G;
120	case X86::COND_G: return X86::COND_LE;
121	case X86::COND_GE: return X86::COND_L;
122	case X86::COND_B: return X86::COND_AE;
123	case X86::COND_BE: return X86::COND_A;
124	case X86::COND_A: return X86::COND_BE;
125	case X86::COND_AE: return X86::COND_B;
126	case X86::COND_S: return X86::COND_NS;
127	case X86::COND_NS: return X86::COND_S;
128	case X86::COND_P: return X86::COND_NP;
129	case X86::COND_NP: return X86::COND_P;
130	case X86::COND_O: return X86::COND_NO;
131	case X86::COND_NO: return X86::COND_O;
132	}
133	}
134
135	unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
136	enum DecodedCondCode : uint8_t {
137	DCC_EQUAL = 0x1,
138	DCC_GREATER = 0x2,
139	DCC_LESSER = 0x4,
140	DCC_GREATER_OR_LESSER = 0x6,
141	DCC_UNSIGNED = 0x8,
142	DCC_SIGNED = 0x10,
143	DCC_INVALID = 0x20,
144	};
145
146	auto decodeCondCode = [&](unsigned CC) -> uint8_t {
147	switch (CC) {
148	default: return DCC_INVALID;
149	case X86::COND_E: return DCC_EQUAL;
150	case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
151	case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
152	case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
153	case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
154	case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
155	case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
156	case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
157	case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
158	case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
159	}
160	};
161
162	uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
163
164	if (DCC & DCC_INVALID)
165	return X86::COND_INVALID;
166
167	if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
168	return X86::COND_INVALID;
169
170	switch (DCC) {
171	default: return X86::COND_INVALID;
172	case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
173	case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
174	case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
175	case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
176	case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
177	case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
178	case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
179	case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
180	case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
181	case DCC_EQUAL: return X86::COND_E;
182	case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
183	case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
184	case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
185	case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
186	}
187	}
188
189	bool isValidCondCode(unsigned CC) const override {
190	return (CC != X86::COND_INVALID);
191	}
192
193	bool isBreakpoint(const MCInst &Inst) const override {
194	return Inst.getOpcode() == X86::INT3;
195	}
196
197	bool isPrefix(const MCInst &Inst) const override {
198	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
199	return X86II::isPrefix(TSFlags: Desc.TSFlags);
200	}
201
202	bool isRep(const MCInst &Inst) const override {
203	return Inst.getFlags() == X86::IP_HAS_REPEAT;
204	}
205
206	bool deleteREPPrefix(MCInst &Inst) const override {
207	if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
208	Inst.setFlags(0);
209	return true;
210	}
211	return false;
212	}
213
214	bool isIndirectCall(const MCInst &Inst) const override {
215	return isCall(Inst) &&
216	((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(i: 0).isReg());
217	}
218
219	bool isPop(const MCInst &Inst) const override {
220	return getPopSize(Inst) == 0 ? false : true;
221	}
222
223	bool isTerminateBranch(const MCInst &Inst) const override {
224	return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
225	}
226
227	int getPopSize(const MCInst &Inst) const override {
228	switch (Inst.getOpcode()) {
229	case X86::POP16r:
230	case X86::POP16rmm:
231	case X86::POP16rmr:
232	case X86::POPF16:
233	case X86::POPA16:
234	case X86::POPDS16:
235	case X86::POPES16:
236	case X86::POPFS16:
237	case X86::POPGS16:
238	case X86::POPSS16:
239	return 2;
240	case X86::POP32r:
241	case X86::POP32rmm:
242	case X86::POP32rmr:
243	case X86::POPA32:
244	case X86::POPDS32:
245	case X86::POPES32:
246	case X86::POPF32:
247	case X86::POPFS32:
248	case X86::POPGS32:
249	case X86::POPSS32:
250	return 4;
251	case X86::POP64r:
252	case X86::POP64rmm:
253	case X86::POP64rmr:
254	case X86::POPF64:
255	case X86::POPFS64:
256	case X86::POPGS64:
257	return 8;
258	}
259	return 0;
260	}
261
262	bool isPush(const MCInst &Inst) const override {
263	return getPushSize(Inst) == 0 ? false : true;
264	}
265
266	int getPushSize(const MCInst &Inst) const override {
267	switch (Inst.getOpcode()) {
268	case X86::PUSH16i8:
269	case X86::PUSH16r:
270	case X86::PUSH16rmm:
271	case X86::PUSH16rmr:
272	case X86::PUSHA16:
273	case X86::PUSHCS16:
274	case X86::PUSHDS16:
275	case X86::PUSHES16:
276	case X86::PUSHF16:
277	case X86::PUSHFS16:
278	case X86::PUSHGS16:
279	case X86::PUSHSS16:
280	case X86::PUSH16i:
281	return 2;
282	case X86::PUSH32i8:
283	case X86::PUSH32r:
284	case X86::PUSH32rmm:
285	case X86::PUSH32rmr:
286	case X86::PUSHA32:
287	case X86::PUSHCS32:
288	case X86::PUSHDS32:
289	case X86::PUSHES32:
290	case X86::PUSHF32:
291	case X86::PUSHFS32:
292	case X86::PUSHGS32:
293	case X86::PUSHSS32:
294	case X86::PUSH32i:
295	return 4;
296	case X86::PUSH64i32:
297	case X86::PUSH64i8:
298	case X86::PUSH64r:
299	case X86::PUSH64rmm:
300	case X86::PUSH64rmr:
301	case X86::PUSHF64:
302	case X86::PUSHFS64:
303	case X86::PUSHGS64:
304	return 8;
305	}
306	return 0;
307	}
308
309	bool isSUB(const MCInst &Inst) const override {
310	return X86::isSUB(Inst.getOpcode());
311	}
312
313	bool isLEA64r(const MCInst &Inst) const override {
314	return Inst.getOpcode() == X86::LEA64r;
315	}
316
317	bool isLeave(const MCInst &Inst) const override {
318	return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
319	}
320
321	bool isMoveMem2Reg(const MCInst &Inst) const override {
322	switch (Inst.getOpcode()) {
323	case X86::MOV16rm:
324	case X86::MOV32rm:
325	case X86::MOV64rm:
326	return true;
327	}
328	return false;
329	}
330
331	bool isUnsupportedInstruction(const MCInst &Inst) const override {
332	switch (Inst.getOpcode()) {
333	default:
334	return false;
335
336	case X86::LOOP:
337	case X86::LOOPE:
338	case X86::LOOPNE:
339	case X86::JECXZ:
340	case X86::JRCXZ:
341	// These have a short displacement, and therefore (often) break after
342	// basic block relayout.
343	return true;
344	}
345	}
346
347	bool mayLoad(const MCInst &Inst) const override {
348	if (isPop(Inst))
349	return true;
350
351	int MemOpNo = getMemoryOperandNo(Inst);
352	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
353
354	if (MemOpNo == -1)
355	return false;
356
357	return MCII.mayLoad();
358	}
359
360	bool mayStore(const MCInst &Inst) const override {
361	if (isPush(Inst))
362	return true;
363
364	int MemOpNo = getMemoryOperandNo(Inst);
365	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
366
367	if (MemOpNo == -1)
368	return false;
369
370	return MCII.mayStore();
371	}
372
373	bool isCleanRegXOR(const MCInst &Inst) const override {
374	switch (Inst.getOpcode()) {
375	case X86::XOR16rr:
376	case X86::XOR32rr:
377	case X86::XOR64rr:
378	break;
379	default:
380	return false;
381	}
382	return (Inst.getOperand(i: 0).getReg() == Inst.getOperand(i: 2).getReg());
383	}
384
385	bool isPacked(const MCInst &Inst) const override {
386	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
387	return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
388	}
389
390	bool shouldRecordCodeRelocation(uint32_t RelType) const override {
391	switch (RelType) {
392	case ELF::R_X86_64_8:
393	case ELF::R_X86_64_16:
394	case ELF::R_X86_64_32:
395	case ELF::R_X86_64_32S:
396	case ELF::R_X86_64_64:
397	case ELF::R_X86_64_PC8:
398	case ELF::R_X86_64_PC32:
399	case ELF::R_X86_64_PC64:
400	case ELF::R_X86_64_GOTPC64:
401	case ELF::R_X86_64_GOTPCRELX:
402	case ELF::R_X86_64_REX_GOTPCRELX:
403	return true;
404	case ELF::R_X86_64_PLT32:
405	case ELF::R_X86_64_GOTPCREL:
406	case ELF::R_X86_64_TPOFF32:
407	case ELF::R_X86_64_GOTTPOFF:
408	return false;
409	default:
410	llvm_unreachable("Unexpected x86 relocation type in code");
411	}
412	}
413
414	StringRef getTrapFillValue() const override { return StringRef("\314", 1); }
415
416	struct IndJmpMatcherFrag1 : MCInstMatcher {
417	std::unique_ptr<MCInstMatcher> Base;
418	std::unique_ptr<MCInstMatcher> Scale;
419	std::unique_ptr<MCInstMatcher> Index;
420	std::unique_ptr<MCInstMatcher> Offset;
421
422	IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
423	std::unique_ptr<MCInstMatcher> Scale,
424	std::unique_ptr<MCInstMatcher> Index,
425	std::unique_ptr<MCInstMatcher> Offset)
426	: Base(std::move(Base)), Scale(std::move(Scale)),
427	Index(std::move(Index)), Offset(std::move(Offset)) {}
428
429	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
430	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
431	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
432	return false;
433
434	if (CurInst->getOpcode() != X86::JMP64m)
435	return false;
436
437	int MemOpNo = MIB.getMemoryOperandNo(Inst: *CurInst);
438	if (MemOpNo == -1)
439	return false;
440
441	if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
442	return false;
443	if (!Scale->match(MRI, MIB, this->InstrWindow,
444	MemOpNo + X86::AddrScaleAmt))
445	return false;
446	if (!Index->match(MRI, MIB, this->InstrWindow,
447	MemOpNo + X86::AddrIndexReg))
448	return false;
449	if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
450	return false;
451	return true;
452	}
453
454	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
455	MIB.addAnnotation(*CurInst, Annotation, true);
456	Base->annotate(MIB, Annotation);
457	Scale->annotate(MIB, Annotation);
458	Index->annotate(MIB, Annotation);
459	Offset->annotate(MIB, Annotation);
460	}
461	};
462
463	std::unique_ptr<MCInstMatcher>
464	matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
465	std::unique_ptr<MCInstMatcher> Scale,
466	std::unique_ptr<MCInstMatcher> Index,
467	std::unique_ptr<MCInstMatcher> Offset) const override {
468	return std::unique_ptr<MCInstMatcher>(
469	new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
470	std::move(Index), std::move(Offset)));
471	}
472
473	struct IndJmpMatcherFrag2 : MCInstMatcher {
474	std::unique_ptr<MCInstMatcher> Reg;
475
476	IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
477	: Reg(std::move(Reg)) {}
478
479	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
480	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
481	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
482	return false;
483
484	if (CurInst->getOpcode() != X86::JMP64r)
485	return false;
486
487	return Reg->match(MRI, MIB, this->InstrWindow, 0);
488	}
489
490	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
491	MIB.addAnnotation(*CurInst, Annotation, true);
492	Reg->annotate(MIB, Annotation);
493	}
494	};
495
496	std::unique_ptr<MCInstMatcher>
497	matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
498	return std::unique_ptr<MCInstMatcher>(
499	new IndJmpMatcherFrag2(std::move(Target)));
500	}
501
502	struct LoadMatcherFrag1 : MCInstMatcher {
503	std::unique_ptr<MCInstMatcher> Base;
504	std::unique_ptr<MCInstMatcher> Scale;
505	std::unique_ptr<MCInstMatcher> Index;
506	std::unique_ptr<MCInstMatcher> Offset;
507
508	LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
509	std::unique_ptr<MCInstMatcher> Scale,
510	std::unique_ptr<MCInstMatcher> Index,
511	std::unique_ptr<MCInstMatcher> Offset)
512	: Base(std::move(Base)), Scale(std::move(Scale)),
513	Index(std::move(Index)), Offset(std::move(Offset)) {}
514
515	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
516	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
517	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
518	return false;
519
520	if (CurInst->getOpcode() != X86::MOV64rm &&
521	CurInst->getOpcode() != X86::MOVSX64rm32)
522	return false;
523
524	int MemOpNo = MIB.getMemoryOperandNo(Inst: *CurInst);
525	if (MemOpNo == -1)
526	return false;
527
528	if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
529	return false;
530	if (!Scale->match(MRI, MIB, this->InstrWindow,
531	MemOpNo + X86::AddrScaleAmt))
532	return false;
533	if (!Index->match(MRI, MIB, this->InstrWindow,
534	MemOpNo + X86::AddrIndexReg))
535	return false;
536	if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
537	return false;
538	return true;
539	}
540
541	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
542	MIB.addAnnotation(*CurInst, Annotation, true);
543	Base->annotate(MIB, Annotation);
544	Scale->annotate(MIB, Annotation);
545	Index->annotate(MIB, Annotation);
546	Offset->annotate(MIB, Annotation);
547	}
548	};
549
550	std::unique_ptr<MCInstMatcher>
551	matchLoad(std::unique_ptr<MCInstMatcher> Base,
552	std::unique_ptr<MCInstMatcher> Scale,
553	std::unique_ptr<MCInstMatcher> Index,
554	std::unique_ptr<MCInstMatcher> Offset) const override {
555	return std::unique_ptr<MCInstMatcher>(
556	new LoadMatcherFrag1(std::move(Base), std::move(Scale),
557	std::move(Index), std::move(Offset)));
558	}
559
560	struct AddMatcher : MCInstMatcher {
561	std::unique_ptr<MCInstMatcher> A;
562	std::unique_ptr<MCInstMatcher> B;
563
564	AddMatcher(std::unique_ptr<MCInstMatcher> A,
565	std::unique_ptr<MCInstMatcher> B)
566	: A(std::move(A)), B(std::move(B)) {}
567
568	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
569	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
570	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
571	return false;
572
573	if (CurInst->getOpcode() == X86::ADD64rr ||
574	CurInst->getOpcode() == X86::ADD64rr_DB ||
575	CurInst->getOpcode() == X86::ADD64rr_REV) {
576	if (!A->match(MRI, MIB, this->InstrWindow, 1)) {
577	if (!B->match(MRI, MIB, this->InstrWindow, 1))
578	return false;
579	return A->match(MRI, MIB, this->InstrWindow, 2);
580	}
581
582	if (B->match(MRI, MIB, this->InstrWindow, 2))
583	return true;
584
585	if (!B->match(MRI, MIB, this->InstrWindow, 1))
586	return false;
587	return A->match(MRI, MIB, this->InstrWindow, 2);
588	}
589
590	return false;
591	}
592
593	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
594	MIB.addAnnotation(*CurInst, Annotation, true);
595	A->annotate(MIB, Annotation);
596	B->annotate(MIB, Annotation);
597	}
598	};
599
600	std::unique_ptr<MCInstMatcher>
601	matchAdd(std::unique_ptr<MCInstMatcher> A,
602	std::unique_ptr<MCInstMatcher> B) const override {
603	return std::unique_ptr<MCInstMatcher>(
604	new AddMatcher(std::move(A), std::move(B)));
605	}
606
607	struct LEAMatcher : MCInstMatcher {
608	std::unique_ptr<MCInstMatcher> Target;
609
610	LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
611	: Target(std::move(Target)) {}
612
613	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
614	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
615	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
616	return false;
617
618	if (CurInst->getOpcode() != X86::LEA64r)
619	return false;
620
621	if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 ||
622	CurInst->getOperand(1 + X86::AddrIndexReg).getReg() !=
623	X86::NoRegister ||
624	(CurInst->getOperand(1 + X86::AddrBaseReg).getReg() !=
625	X86::NoRegister &&
626	CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP))
627	return false;
628
629	return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp);
630	}
631
632	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
633	MIB.addAnnotation(*CurInst, Annotation, true);
634	Target->annotate(MIB, Annotation);
635	}
636	};
637
638	std::unique_ptr<MCInstMatcher>
639	matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
640	return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
641	}
642
643	bool hasPCRelOperand(const MCInst &Inst) const override {
644	for (const MCOperand &Operand : Inst)
645	if (Operand.isReg() && Operand.getReg() == X86::RIP)
646	return true;
647	return false;
648	}
649
650	int getMemoryOperandNo(const MCInst &Inst) const override {
651	unsigned Opcode = Inst.getOpcode();
652	const MCInstrDesc &Desc = Info->get(Opcode);
653	int MemOpNo = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
654	if (MemOpNo >= 0)
655	MemOpNo += X86II::getOperandBias(Desc);
656	return MemOpNo;
657	}
658
659	bool hasEVEXEncoding(const MCInst &Inst) const override {
660	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
661	return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
662	}
663
664	std::optional<X86MemOperand>
665	evaluateX86MemoryOperand(const MCInst &Inst) const override {
666	int MemOpNo = getMemoryOperandNo(Inst);
667	if (MemOpNo < 0)
668	return std::nullopt;
669	unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
670
671	if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
672	return std::nullopt;
673
674	const MCOperand &Base = Inst.getOperand(i: MemOpOffset + X86::AddrBaseReg);
675	const MCOperand &Scale = Inst.getOperand(i: MemOpOffset + X86::AddrScaleAmt);
676	const MCOperand &Index = Inst.getOperand(i: MemOpOffset + X86::AddrIndexReg);
677	const MCOperand &Disp = Inst.getOperand(i: MemOpOffset + X86::AddrDisp);
678	const MCOperand &Segment =
679	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg);
680
681	// Make sure it is a well-formed memory operand.
682	if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
683	(!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
684	return std::nullopt;
685
686	X86MemOperand MO;
687	MO.BaseRegNum = Base.getReg();
688	MO.ScaleImm = Scale.getImm();
689	MO.IndexRegNum = Index.getReg();
690	MO.DispImm = Disp.isImm() ? Disp.getImm() : 0;
691	MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr;
692	MO.SegRegNum = Segment.getReg();
693	return MO;
694	}
695
696	bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
697	uint64_t Address,
698	uint64_t Size) const override {
699	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
700	if (!MO)
701	return false;
702
703	// Make sure it's a well-formed addressing we can statically evaluate.
704	if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) ||
705	MO->IndexRegNum != X86::NoRegister ||
706	MO->SegRegNum != X86::NoRegister || MO->DispExpr)
707	return false;
708
709	Target = MO->DispImm;
710	if (MO->BaseRegNum == X86::RIP) {
711	assert(Size != 0 && "instruction size required in order to statically "
712	"evaluate RIP-relative address");
713	Target += Address + Size;
714	}
715	return true;
716	}
717
718	MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
719	int MemOpNo = getMemoryOperandNo(Inst);
720	if (MemOpNo < 0)
721	return Inst.end();
722	return Inst.begin() + (MemOpNo + X86::AddrDisp);
723	}
724
725	bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
726	MCOperand *OI = getMemOperandDisp(Inst);
727	if (OI == Inst.end())
728	return false;
729	*OI = Operand;
730	return true;
731	}
732
733	/// Get the registers used as function parameters.
734	/// This function is specific to the x86_64 abi on Linux.
735	BitVector getRegsUsedAsParams() const override {
736	BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
737	Regs |= getAliases(X86::RSI);
738	Regs |= getAliases(X86::RDI);
739	Regs |= getAliases(X86::RDX);
740	Regs |= getAliases(X86::RCX);
741	Regs |= getAliases(X86::R8);
742	Regs |= getAliases(X86::R9);
743	return Regs;
744	}
745
746	void getCalleeSavedRegs(BitVector &Regs) const override {
747	Regs |= getAliases(X86::RBX);
748	Regs |= getAliases(X86::RBP);
749	Regs |= getAliases(X86::R12);
750	Regs |= getAliases(X86::R13);
751	Regs |= getAliases(X86::R14);
752	Regs |= getAliases(X86::R15);
753	}
754
755	void getDefaultDefIn(BitVector &Regs) const override {
756	assert(Regs.size() >= RegInfo->getNumRegs() &&
757	"The size of BitVector is less than RegInfo->getNumRegs().");
758	Regs.set(X86::RAX);
759	Regs.set(X86::RCX);
760	Regs.set(X86::RDX);
761	Regs.set(X86::RSI);
762	Regs.set(X86::RDI);
763	Regs.set(X86::R8);
764	Regs.set(X86::R9);
765	Regs.set(X86::XMM0);
766	Regs.set(X86::XMM1);
767	Regs.set(X86::XMM2);
768	Regs.set(X86::XMM3);
769	Regs.set(X86::XMM4);
770	Regs.set(X86::XMM5);
771	Regs.set(X86::XMM6);
772	Regs.set(X86::XMM7);
773	}
774
775	void getDefaultLiveOut(BitVector &Regs) const override {
776	assert(Regs.size() >= RegInfo->getNumRegs() &&
777	"The size of BitVector is less than RegInfo->getNumRegs().");
778	Regs |= getAliases(X86::RAX);
779	Regs |= getAliases(X86::RDX);
780	Regs |= getAliases(X86::RCX);
781	Regs |= getAliases(X86::XMM0);
782	Regs |= getAliases(X86::XMM1);
783	}
784
785	void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
786	if (IncludeAlias) {
787	Regs |= getAliases(X86::RAX);
788	Regs |= getAliases(X86::RBX);
789	Regs |= getAliases(X86::RBP);
790	Regs |= getAliases(X86::RSI);
791	Regs |= getAliases(X86::RDI);
792	Regs |= getAliases(X86::RDX);
793	Regs |= getAliases(X86::RCX);
794	Regs |= getAliases(X86::R8);
795	Regs |= getAliases(X86::R9);
796	Regs |= getAliases(X86::R10);
797	Regs |= getAliases(X86::R11);
798	Regs |= getAliases(X86::R12);
799	Regs |= getAliases(X86::R13);
800	Regs |= getAliases(X86::R14);
801	Regs |= getAliases(X86::R15);
802	return;
803	}
804	Regs.set(X86::RAX);
805	Regs.set(X86::RBX);
806	Regs.set(X86::RBP);
807	Regs.set(X86::RSI);
808	Regs.set(X86::RDI);
809	Regs.set(X86::RDX);
810	Regs.set(X86::RCX);
811	Regs.set(X86::R8);
812	Regs.set(X86::R9);
813	Regs.set(X86::R10);
814	Regs.set(X86::R11);
815	Regs.set(X86::R12);
816	Regs.set(X86::R13);
817	Regs.set(X86::R14);
818	Regs.set(X86::R15);
819	}
820
821	void getClassicGPRegs(BitVector &Regs) const override {
822	Regs |= getAliases(X86::RAX);
823	Regs |= getAliases(X86::RBX);
824	Regs |= getAliases(X86::RBP);
825	Regs |= getAliases(X86::RSI);
826	Regs |= getAliases(X86::RDI);
827	Regs |= getAliases(X86::RDX);
828	Regs |= getAliases(X86::RCX);
829	}
830
831	void getRepRegs(BitVector &Regs) const override {
832	Regs |= getAliases(X86::RCX);
833	}
834
835	MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
836	Reg = getX86SubSuperRegister(Reg, Size: Size * 8);
837	assert((Reg != X86::NoRegister) && "Invalid register");
838	return Reg;
839	}
840
841	bool isUpper8BitReg(MCPhysReg Reg) const override {
842	switch (Reg) {
843	case X86::AH:
844	case X86::BH:
845	case X86::CH:
846	case X86::DH:
847	return true;
848	default:
849	return false;
850	}
851	}
852
853	bool cannotUseREX(const MCInst &Inst) const override {
854	switch (Inst.getOpcode()) {
855	case X86::MOV8mr_NOREX:
856	case X86::MOV8rm_NOREX:
857	case X86::MOV8rr_NOREX:
858	case X86::MOVSX32rm8_NOREX:
859	case X86::MOVSX32rr8_NOREX:
860	case X86::MOVZX32rm8_NOREX:
861	case X86::MOVZX32rr8_NOREX:
862	case X86::MOV8mr:
863	case X86::MOV8rm:
864	case X86::MOV8rr:
865	case X86::MOVSX32rm8:
866	case X86::MOVSX32rr8:
867	case X86::MOVZX32rm8:
868	case X86::MOVZX32rr8:
869	case X86::TEST8ri:
870	for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) {
871	if (!Operand.isReg())
872	continue;
873	if (isUpper8BitReg(Reg: Operand.getReg()))
874	return true;
875	}
876	[[fallthrough]];
877	default:
878	return false;
879	}
880	}
881
882	static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) {
883	using namespace llvm::X86;
884	int OpType = getOperandType(Inst.getOpcode(), MemOpNo);
885	return getMemOperandSize(OpType) / 8;
886	}
887
888	/// Classifying a stack access as *not* "SIMPLE" here means we don't know how
889	/// to change this instruction memory access. It will disable any changes to
890	/// the stack layout, so we can't do the most aggressive form of shrink
891	/// wrapping. We must do so in a way that keeps the original stack layout.
892	/// Otherwise you need to adjust the offset of all instructions accessing the
893	/// stack: we can't do that anymore because there is one instruction that is
894	/// not simple. There are other implications as well. We have heuristics to
895	/// detect when a register is callee-saved and thus eligible for shrink
896	/// wrapping. If you are restoring a register using a non-simple stack access,
897	/// then it is classified as NOT callee-saved, and it disables shrink wrapping
898	/// for *that* register (but not for others).
899	///
900	/// Classifying a stack access as "size 0" or detecting an indexed memory
901	/// access (to address a vector, for example) here means we know there is a
902	/// stack access, but we can't quite understand how wide is the access in
903	/// bytes. This is very serious because we can't understand how memory
904	/// accesses alias with each other for this function. This will essentially
905	/// disable not only shrink wrapping but all frame analysis, it will fail it
906	/// as "we don't understand this function and we give up on it".
907	bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
908	bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
909	uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
910	bool &IsSimple, bool &IsIndexed) const override {
911	// Detect simple push/pop cases first
912	if (int Sz = getPushSize(Inst)) {
913	IsLoad = false;
914	IsStore = true;
915	IsStoreFromReg = true;
916	StackPtrReg = X86::RSP;
917	StackOffset = -Sz;
918	Size = Sz;
919	IsSimple = true;
920	if (Inst.getOperand(i: 0).isImm())
921	SrcImm = Inst.getOperand(i: 0).getImm();
922	else if (Inst.getOperand(i: 0).isReg())
923	Reg = Inst.getOperand(i: 0).getReg();
924	else
925	IsSimple = false;
926
927	return true;
928	}
929	if (int Sz = getPopSize(Inst)) {
930	IsLoad = true;
931	IsStore = false;
932	if (Inst.getNumOperands() == 0 || !Inst.getOperand(i: 0).isReg()) {
933	IsSimple = false;
934	} else {
935	Reg = Inst.getOperand(i: 0).getReg();
936	IsSimple = true;
937	}
938	StackPtrReg = X86::RSP;
939	StackOffset = 0;
940	Size = Sz;
941	return true;
942	}
943
944	struct InstInfo {
945	// Size in bytes that Inst loads from memory.
946	uint8_t DataSize;
947	bool IsLoad;
948	bool IsStore;
949	bool StoreFromReg;
950	bool Simple;
951	};
952
953	InstInfo I;
954	int MemOpNo = getMemoryOperandNo(Inst);
955	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
956	// If it is not dealing with a memory operand, we discard it
957	if (MemOpNo == -1 || MCII.isCall())
958	return false;
959
960	switch (Inst.getOpcode()) {
961	default: {
962	bool IsLoad = MCII.mayLoad();
963	bool IsStore = MCII.mayStore();
964	// Is it LEA? (deals with memory but is not loading nor storing)
965	if (!IsLoad && !IsStore) {
966	I = {.DataSize: 0, .IsLoad: IsLoad, .IsStore: IsStore, .StoreFromReg: false, .Simple: false};
967	break;
968	}
969	uint8_t Sz = getMemDataSize(Inst, MemOpNo);
970	I = {.DataSize: Sz, .IsLoad: IsLoad, .IsStore: IsStore, .StoreFromReg: false, .Simple: false};
971	break;
972	}
973	// Report simple stack accesses
974	case X86::MOV8rm: I = {.DataSize: 1, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
975	case X86::MOV16rm: I = {.DataSize: 2, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
976	case X86::MOV32rm: I = {.DataSize: 4, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
977	case X86::MOV64rm: I = {.DataSize: 8, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
978	case X86::MOV8mr: I = {.DataSize: 1, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
979	case X86::MOV16mr: I = {.DataSize: 2, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
980	case X86::MOV32mr: I = {.DataSize: 4, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
981	case X86::MOV64mr: I = {.DataSize: 8, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
982	case X86::MOV8mi: I = {.DataSize: 1, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
983	case X86::MOV16mi: I = {.DataSize: 2, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
984	case X86::MOV32mi: I = {.DataSize: 4, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
985	} // end switch (Inst.getOpcode())
986
987	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
988	if (!MO) {
989	LLVM_DEBUG(dbgs() << "Evaluate failed on ");
990	LLVM_DEBUG(Inst.dump());
991	return false;
992	}
993
994	// Make sure it's a stack access
995	if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP)
996	return false;
997
998	IsLoad = I.IsLoad;
999	IsStore = I.IsStore;
1000	IsStoreFromReg = I.StoreFromReg;
1001	Size = I.DataSize;
1002	IsSimple = I.Simple;
1003	StackPtrReg = MO->BaseRegNum;
1004	StackOffset = MO->DispImm;
1005	IsIndexed =
1006	MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister;
1007
1008	if (!I.Simple)
1009	return true;
1010
1011	// Retrieve related register in simple MOV from/to stack operations.
1012	unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1013	if (I.IsLoad) {
1014	MCOperand RegOpnd = Inst.getOperand(i: 0);
1015	assert(RegOpnd.isReg() && "unexpected destination operand");
1016	Reg = RegOpnd.getReg();
1017	} else if (I.IsStore) {
1018	MCOperand SrcOpnd =
1019	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg + 1);
1020	if (I.StoreFromReg) {
1021	assert(SrcOpnd.isReg() && "unexpected source operand");
1022	Reg = SrcOpnd.getReg();
1023	} else {
1024	assert(SrcOpnd.isImm() && "unexpected source operand");
1025	SrcImm = SrcOpnd.getImm();
1026	}
1027	}
1028
1029	return true;
1030	}
1031
1032	void changeToPushOrPop(MCInst &Inst) const override {
1033	assert(!isPush(Inst) && !isPop(Inst));
1034
1035	struct InstInfo {
1036	// Size in bytes that Inst loads from memory.
1037	uint8_t DataSize;
1038	bool IsLoad;
1039	bool StoreFromReg;
1040	};
1041
1042	InstInfo I;
1043	switch (Inst.getOpcode()) {
1044	default: {
1045	llvm_unreachable("Unhandled opcode");
1046	return;
1047	}
1048	case X86::MOV16rm: I = {.DataSize: 2, .IsLoad: true, .StoreFromReg: false}; break;
1049	case X86::MOV32rm: I = {.DataSize: 4, .IsLoad: true, .StoreFromReg: false}; break;
1050	case X86::MOV64rm: I = {.DataSize: 8, .IsLoad: true, .StoreFromReg: false}; break;
1051	case X86::MOV16mr: I = {.DataSize: 2, .IsLoad: false, .StoreFromReg: true}; break;
1052	case X86::MOV32mr: I = {.DataSize: 4, .IsLoad: false, .StoreFromReg: true}; break;
1053	case X86::MOV64mr: I = {.DataSize: 8, .IsLoad: false, .StoreFromReg: true}; break;
1054	case X86::MOV16mi: I = {.DataSize: 2, .IsLoad: false, .StoreFromReg: false}; break;
1055	case X86::MOV32mi: I = {.DataSize: 4, .IsLoad: false, .StoreFromReg: false}; break;
1056	} // end switch (Inst.getOpcode())
1057
1058	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1059	if (!MO) {
1060	llvm_unreachable("Evaluate failed");
1061	return;
1062	}
1063	// Make sure it's a stack access
1064	if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) {
1065	llvm_unreachable("Not a stack access");
1066	return;
1067	}
1068
1069	unsigned MemOpOffset = getMemoryOperandNo(Inst);
1070	unsigned NewOpcode = 0;
1071	if (I.IsLoad) {
1072	switch (I.DataSize) {
1073	case 2: NewOpcode = X86::POP16r; break;
1074	case 4: NewOpcode = X86::POP32r; break;
1075	case 8: NewOpcode = X86::POP64r; break;
1076	default:
1077	llvm_unreachable("Unexpected size");
1078	}
1079	unsigned RegOpndNum = Inst.getOperand(i: 0).getReg();
1080	Inst.clear();
1081	Inst.setOpcode(NewOpcode);
1082	Inst.addOperand(Op: MCOperand::createReg(Reg: RegOpndNum));
1083	} else {
1084	MCOperand SrcOpnd =
1085	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg + 1);
1086	if (I.StoreFromReg) {
1087	switch (I.DataSize) {
1088	case 2: NewOpcode = X86::PUSH16r; break;
1089	case 4: NewOpcode = X86::PUSH32r; break;
1090	case 8: NewOpcode = X86::PUSH64r; break;
1091	default:
1092	llvm_unreachable("Unexpected size");
1093	}
1094	assert(SrcOpnd.isReg() && "Unexpected source operand");
1095	unsigned RegOpndNum = SrcOpnd.getReg();
1096	Inst.clear();
1097	Inst.setOpcode(NewOpcode);
1098	Inst.addOperand(Op: MCOperand::createReg(Reg: RegOpndNum));
1099	} else {
1100	switch (I.DataSize) {
1101	case 2: NewOpcode = X86::PUSH16i8; break;
1102	case 4: NewOpcode = X86::PUSH32i8; break;
1103	case 8: NewOpcode = X86::PUSH64i32; break;
1104	default:
1105	llvm_unreachable("Unexpected size");
1106	}
1107	assert(SrcOpnd.isImm() && "Unexpected source operand");
1108	int64_t SrcImm = SrcOpnd.getImm();
1109	Inst.clear();
1110	Inst.setOpcode(NewOpcode);
1111	Inst.addOperand(Op: MCOperand::createImm(Val: SrcImm));
1112	}
1113	}
1114	}
1115
1116	bool isStackAdjustment(const MCInst &Inst) const override {
1117	switch (Inst.getOpcode()) {
1118	default:
1119	return false;
1120	case X86::SUB64ri32:
1121	case X86::SUB64ri8:
1122	case X86::ADD64ri32:
1123	case X86::ADD64ri8:
1124	case X86::LEA64r:
1125	break;
1126	}
1127
1128	return any_of(defOperands(Inst), [](const MCOperand &Op) {
1129	return Op.isReg() && Op.getReg() == X86::RSP;
1130	});
1131	}
1132
1133	bool
1134	evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output,
1135	std::pair<MCPhysReg, int64_t> Input1,
1136	std::pair<MCPhysReg, int64_t> Input2) const override {
1137
1138	auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1139	if (Reg == Input1.first)
1140	return Input1.second;
1141	if (Reg == Input2.first)
1142	return Input2.second;
1143	return make_error_code(errc::result_out_of_range);
1144	};
1145
1146	switch (Inst.getOpcode()) {
1147	default:
1148	return false;
1149
1150	case X86::SUB64ri32:
1151	case X86::SUB64ri8:
1152	if (!Inst.getOperand(i: 2).isImm())
1153	return false;
1154	if (ErrorOr<int64_t> InputVal =
1155	getOperandVal(Inst.getOperand(1).getReg()))
1156	Output = *InputVal - Inst.getOperand(i: 2).getImm();
1157	else
1158	return false;
1159	break;
1160	case X86::ADD64ri32:
1161	case X86::ADD64ri8:
1162	if (!Inst.getOperand(i: 2).isImm())
1163	return false;
1164	if (ErrorOr<int64_t> InputVal =
1165	getOperandVal(Inst.getOperand(1).getReg()))
1166	Output = *InputVal + Inst.getOperand(i: 2).getImm();
1167	else
1168	return false;
1169	break;
1170	case X86::ADD64i32:
1171	if (!Inst.getOperand(i: 0).isImm())
1172	return false;
1173	if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1174	Output = *InputVal + Inst.getOperand(i: 0).getImm();
1175	else
1176	return false;
1177	break;
1178
1179	case X86::LEA64r: {
1180	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1181	if (!MO)
1182	return false;
1183
1184	if (MO->BaseRegNum == X86::NoRegister ||
1185	MO->IndexRegNum != X86::NoRegister ||
1186	MO->SegRegNum != X86::NoRegister || MO->DispExpr)
1187	return false;
1188
1189	if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum))
1190	Output = *InputVal + MO->DispImm;
1191	else
1192	return false;
1193
1194	break;
1195	}
1196	}
1197	return true;
1198	}
1199
1200	bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1201	MCPhysReg &To) const override {
1202	switch (Inst.getOpcode()) {
1203	default:
1204	return false;
1205	case X86::LEAVE:
1206	case X86::LEAVE64:
1207	To = getStackPointer();
1208	From = getFramePointer();
1209	return true;
1210	case X86::MOV64rr:
1211	To = Inst.getOperand(i: 0).getReg();
1212	From = Inst.getOperand(i: 1).getReg();
1213	return true;
1214	}
1215	}
1216
1217	MCPhysReg getStackPointer() const override { return X86::RSP; }
1218	MCPhysReg getFramePointer() const override { return X86::RBP; }
1219	MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1220
1221	bool escapesVariable(const MCInst &Inst,
1222	bool HasFramePointer) const override {
1223	int MemOpNo = getMemoryOperandNo(Inst);
1224	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1225	const unsigned NumDefs = MCII.getNumDefs();
1226	static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
1227	getAliases(X86::RBP));
1228	static BitVector SPAliases(getAliases(X86::RSP));
1229
1230	// FIXME: PUSH can be technically a leak, but let's ignore this for now
1231	// because a lot of harmless prologue code will spill SP to the stack.
1232	// Unless push is clearly pushing an object address to the stack as
1233	// demonstrated by having a MemOp.
1234	bool IsPush = isPush(Inst);
1235	if (IsPush && MemOpNo == -1)
1236	return false;
1237
1238	// We use this to detect LEA (has memop but does not access mem)
1239	bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1240	bool DoesLeak = false;
1241	for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1242	// Ignore if SP/BP is used to dereference memory -- that's fine
1243	if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1244	I <= MemOpNo + 5)
1245	continue;
1246	// Ignore if someone is writing to SP/BP
1247	if (I < static_cast<int>(NumDefs))
1248	continue;
1249
1250	const MCOperand &Operand = Inst.getOperand(i: I);
1251	if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1252	DoesLeak = true;
1253	break;
1254	}
1255	if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1256	DoesLeak = true;
1257	break;
1258	}
1259	}
1260
1261	// If potential leak, check if it is not just writing to itself/sp/bp
1262	if (DoesLeak) {
1263	DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) {
1264	assert(Operand.isReg());
1265	MCPhysReg Reg = Operand.getReg();
1266	return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg];
1267	});
1268	}
1269	return DoesLeak;
1270	}
1271
1272	bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1273	unsigned ImmOpNo = -1U;
1274	int MemOpNo = getMemoryOperandNo(Inst);
1275	if (MemOpNo != -1)
1276	ImmOpNo = MemOpNo + X86::AddrDisp;
1277	else
1278	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1279	++Index)
1280	if (Inst.getOperand(i: Index).isImm())
1281	ImmOpNo = Index;
1282	if (ImmOpNo == -1U)
1283	return false;
1284
1285	MCOperand &Operand = Inst.getOperand(i: ImmOpNo);
1286	Amt += Operand.getImm();
1287	Operand.setImm(Amt);
1288	// Check for the need for relaxation
1289	if (int64_t(Amt) == int64_t(int8_t(Amt)))
1290	return true;
1291
1292	// Relax instruction
1293	switch (Inst.getOpcode()) {
1294	case X86::SUB64ri8:
1295	Inst.setOpcode(X86::SUB64ri32);
1296	break;
1297	case X86::ADD64ri8:
1298	Inst.setOpcode(X86::ADD64ri32);
1299	break;
1300	default:
1301	// No need for relaxation
1302	break;
1303	}
1304	return true;
1305	}
1306
1307	/// TODO: this implementation currently works for the most common opcodes that
1308	/// load from memory. It can be extended to work with memory store opcodes as
1309	/// well as more memory load opcodes.
1310	bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1311	uint64_t Offset) const override {
1312	enum CheckSignExt : uint8_t {
1313	NOCHECK = 0,
1314	CHECK8,
1315	CHECK32,
1316	};
1317
1318	using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1319	struct InstInfo {
1320	// Size in bytes that Inst loads from memory.
1321	uint8_t DataSize;
1322
1323	// True when the target operand has to be duplicated because the opcode
1324	// expects a LHS operand.
1325	bool HasLHS;
1326
1327	// List of checks and corresponding opcodes to be used. We try to use the
1328	// smallest possible immediate value when various sizes are available,
1329	// hence we may need to check whether a larger constant fits in a smaller
1330	// immediate.
1331	CheckList Checks;
1332	};
1333
1334	InstInfo I;
1335
1336	switch (Inst.getOpcode()) {
1337	default: {
1338	switch (getPopSize(Inst)) {
1339	case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
1340	case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
1341	case 8: I = {8, false, {{CHECK32, X86::MOV64ri32},
1342	{NOCHECK, X86::MOV64rm}}}; break;
1343	default: return false;
1344	}
1345	break;
1346	}
1347
1348	// MOV
1349	case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
1350	case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
1351	case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
1352	case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32},
1353	{NOCHECK, X86::MOV64rm}}}; break;
1354
1355	// MOVZX
1356	case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
1357	case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
1358	case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
1359
1360	// CMP
1361	case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
1362	case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8},
1363	{NOCHECK, X86::CMP16ri}}}; break;
1364	case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8},
1365	{NOCHECK, X86::CMP32ri}}}; break;
1366	case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8},
1367	{CHECK32, X86::CMP64ri32},
1368	{NOCHECK, X86::CMP64rm}}}; break;
1369
1370	// TEST
1371	case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
1372	case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
1373	case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
1374	case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32},
1375	{NOCHECK, X86::TEST64mr}}}; break;
1376
1377	// ADD
1378	case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
1379	case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8},
1380	{NOCHECK, X86::ADD16ri}}}; break;
1381	case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8},
1382	{NOCHECK, X86::ADD32ri}}}; break;
1383	case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8},
1384	{CHECK32, X86::ADD64ri32},
1385	{NOCHECK, X86::ADD64rm}}}; break;
1386
1387	// SUB
1388	case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
1389	case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8},
1390	{NOCHECK, X86::SUB16ri}}}; break;
1391	case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8},
1392	{NOCHECK, X86::SUB32ri}}}; break;
1393	case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8},
1394	{CHECK32, X86::SUB64ri32},
1395	{NOCHECK, X86::SUB64rm}}}; break;
1396
1397	// AND
1398	case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
1399	case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8},
1400	{NOCHECK, X86::AND16ri}}}; break;
1401	case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8},
1402	{NOCHECK, X86::AND32ri}}}; break;
1403	case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8},
1404	{CHECK32, X86::AND64ri32},
1405	{NOCHECK, X86::AND64rm}}}; break;
1406
1407	// OR
1408	case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
1409	case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8},
1410	{NOCHECK, X86::OR16ri}}}; break;
1411	case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8},
1412	{NOCHECK, X86::OR32ri}}}; break;
1413	case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8},
1414	{CHECK32, X86::OR64ri32},
1415	{NOCHECK, X86::OR64rm}}}; break;
1416
1417	// XOR
1418	case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
1419	case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8},
1420	{NOCHECK, X86::XOR16ri}}}; break;
1421	case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8},
1422	{NOCHECK, X86::XOR32ri}}}; break;
1423	case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8},
1424	{CHECK32, X86::XOR64ri32},
1425	{NOCHECK, X86::XOR64rm}}}; break;
1426	}
1427
1428	// Compute the immediate value.
1429	assert(Offset + I.DataSize <= ConstantData.size() &&
1430	"invalid offset for given constant data");
1431	int64_t ImmVal =
1432	DataExtractor(ConstantData, true, 8).getSigned(offset_ptr: &Offset, size: I.DataSize);
1433
1434	// Compute the new opcode.
1435	unsigned NewOpcode = 0;
1436	for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1437	NewOpcode = Check.second;
1438	if (Check.first == NOCHECK)
1439	break;
1440	if (Check.first == CHECK8 && isInt<8>(ImmVal))
1441	break;
1442	if (Check.first == CHECK32 && isInt<32>(ImmVal))
1443	break;
1444	}
1445	if (NewOpcode == Inst.getOpcode())
1446	return false;
1447
1448	// Modify the instruction.
1449	MCOperand ImmOp = MCOperand::createImm(Val: ImmVal);
1450	uint32_t TargetOpNum = 0;
1451	// Test instruction does not follow the regular pattern of putting the
1452	// memory reference of a load (5 MCOperands) last in the list of operands.
1453	// Since it is not modifying the register operand, it is not treated as
1454	// a destination operand and it is not the first operand as it is in the
1455	// other instructions we treat here.
1456	if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1457	NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1458	TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1459
1460	MCOperand TargetOp = Inst.getOperand(i: TargetOpNum);
1461	Inst.clear();
1462	Inst.setOpcode(NewOpcode);
1463	Inst.addOperand(Op: TargetOp);
1464	if (I.HasLHS)
1465	Inst.addOperand(Op: TargetOp);
1466	Inst.addOperand(Op: ImmOp);
1467
1468	return true;
1469	}
1470
1471	/// TODO: this implementation currently works for the most common opcodes that
1472	/// load from memory. It can be extended to work with memory store opcodes as
1473	/// well as more memory load opcodes.
1474	bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1475	unsigned NewOpcode;
1476
1477	switch (Inst.getOpcode()) {
1478	default: {
1479	switch (getPopSize(Inst)) {
1480	case 2: NewOpcode = X86::MOV16rr; break;
1481	case 4: NewOpcode = X86::MOV32rr; break;
1482	case 8: NewOpcode = X86::MOV64rr; break;
1483	default: return false;
1484	}
1485	break;
1486	}
1487
1488	// MOV
1489	case X86::MOV8rm: NewOpcode = X86::MOV8rr; break;
1490	case X86::MOV16rm: NewOpcode = X86::MOV16rr; break;
1491	case X86::MOV32rm: NewOpcode = X86::MOV32rr; break;
1492	case X86::MOV64rm: NewOpcode = X86::MOV64rr; break;
1493	}
1494
1495	// Modify the instruction.
1496	MCOperand RegOp = MCOperand::createReg(Reg: RegNum);
1497	MCOperand TargetOp = Inst.getOperand(i: 0);
1498	Inst.clear();
1499	Inst.setOpcode(NewOpcode);
1500	Inst.addOperand(Op: TargetOp);
1501	Inst.addOperand(Op: RegOp);
1502
1503	return true;
1504	}
1505
1506	bool isRedundantMove(const MCInst &Inst) const override {
1507	switch (Inst.getOpcode()) {
1508	default:
1509	return false;
1510
1511	// MOV
1512	case X86::MOV8rr:
1513	case X86::MOV16rr:
1514	case X86::MOV32rr:
1515	case X86::MOV64rr:
1516	break;
1517	}
1518
1519	assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1520	return Inst.getOperand(i: 0).getReg() == Inst.getOperand(i: 1).getReg();
1521	}
1522
1523	bool requiresAlignedAddress(const MCInst &Inst) const override {
1524	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
1525	for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1526	const MCOperandInfo &Op = Desc.operands()[I];
1527	if (Op.OperandType != MCOI::OPERAND_REGISTER)
1528	continue;
1529	if (Op.RegClass == X86::VR128RegClassID)
1530	return true;
1531	}
1532	return false;
1533	}
1534
1535	bool convertJmpToTailCall(MCInst &Inst) override {
1536	if (isTailCall(Inst))
1537	return false;
1538
1539	int NewOpcode;
1540	switch (Inst.getOpcode()) {
1541	default:
1542	return false;
1543	case X86::JMP_1:
1544	case X86::JMP_2:
1545	case X86::JMP_4:
1546	NewOpcode = X86::JMP_4;
1547	break;
1548	case X86::JMP16m:
1549	case X86::JMP32m:
1550	case X86::JMP64m:
1551	NewOpcode = X86::JMP32m;
1552	break;
1553	case X86::JMP16r:
1554	case X86::JMP32r:
1555	case X86::JMP64r:
1556	NewOpcode = X86::JMP32r;
1557	break;
1558	}
1559
1560	Inst.setOpcode(NewOpcode);
1561	setTailCall(Inst);
1562	return true;
1563	}
1564
1565	bool convertTailCallToJmp(MCInst &Inst) override {
1566	int NewOpcode;
1567	switch (Inst.getOpcode()) {
1568	default:
1569	return false;
1570	case X86::JMP_4:
1571	NewOpcode = X86::JMP_1;
1572	break;
1573	case X86::JMP32m:
1574	NewOpcode = X86::JMP64m;
1575	break;
1576	case X86::JMP32r:
1577	NewOpcode = X86::JMP64r;
1578	break;
1579	}
1580
1581	Inst.setOpcode(NewOpcode);
1582	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1583	clearOffset(Inst);
1584	return true;
1585	}
1586
1587	bool convertTailCallToCall(MCInst &Inst) override {
1588	int NewOpcode;
1589	switch (Inst.getOpcode()) {
1590	default:
1591	return false;
1592	case X86::JMP_4:
1593	NewOpcode = X86::CALL64pcrel32;
1594	break;
1595	case X86::JMP32m:
1596	NewOpcode = X86::CALL64m;
1597	break;
1598	case X86::JMP32r:
1599	NewOpcode = X86::CALL64r;
1600	break;
1601	}
1602
1603	Inst.setOpcode(NewOpcode);
1604	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1605	return true;
1606	}
1607
1608	InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
1609	const MCSymbol *TargetLocation,
1610	MCContext *Ctx) override {
1611	assert((DirectCall.getOpcode() == X86::CALL64pcrel32 ||
1612	(DirectCall.getOpcode() == X86::JMP_4 && isTailCall(DirectCall))) &&
1613	"64-bit direct (tail) call instruction expected");
1614
1615	InstructionListType Code;
1616	// Create a new indirect call by converting the previous direct call.
1617	MCInst Inst = DirectCall;
1618	const auto NewOpcode =
1619	(Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1620	Inst.setOpcode(NewOpcode);
1621
1622	// Replace the first operand and preserve auxiliary operands of
1623	// the instruction.
1624	Inst.erase(I: Inst.begin());
1625	Inst.insert(Inst.begin(),
1626	MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
1627	Inst.insert(I: Inst.begin(),
1628	Op: MCOperand::createExpr( // Displacement
1629	Val: MCSymbolRefExpr::create(Symbol: TargetLocation,
1630	Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
1631	Inst.insert(Inst.begin(),
1632	MCOperand::createReg(X86::NoRegister)); // IndexReg
1633	Inst.insert(I: Inst.begin(),
1634	Op: MCOperand::createImm(Val: 1)); // ScaleAmt
1635	Inst.insert(Inst.begin(),
1636	MCOperand::createReg(X86::RIP)); // BaseReg
1637
1638	Code.emplace_back(Inst);
1639	return Code;
1640	}
1641
1642	void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1643	bool IsTailCall = isTailCall(Inst);
1644	if (IsTailCall)
1645	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1646	if (Inst.getOpcode() == X86::CALL64m ||
1647	(Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1648	Inst.setOpcode(X86::MOV64rm);
1649	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1650	return;
1651	}
1652	if (Inst.getOpcode() == X86::CALL64r ||
1653	(Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1654	Inst.setOpcode(X86::MOV64rr);
1655	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1656	return;
1657	}
1658	LLVM_DEBUG(Inst.dump());
1659	llvm_unreachable("not implemented");
1660	}
1661
1662	bool shortenInstruction(MCInst &Inst,
1663	const MCSubtargetInfo &STI) const override {
1664	unsigned OldOpcode = Inst.getOpcode();
1665	unsigned NewOpcode = OldOpcode;
1666
1667	int MemOpNo = getMemoryOperandNo(Inst);
1668
1669	// Check and remove redundant Address-Size override prefix.
1670	if (opts::X86StripRedundantAddressSize) {
1671	uint64_t TSFlags = Info->get(Opcode: OldOpcode).TSFlags;
1672	unsigned Flags = Inst.getFlags();
1673
1674	if (!X86_MC::needsAddressSizeOverride(MI: Inst, STI, MemoryOperand: MemOpNo, TSFlags) &&
1675	Flags & X86::IP_HAS_AD_SIZE)
1676	Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1677	}
1678
1679	// Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1680	// SIB byte is present, but no index is used and modrm alone should have
1681	// been enough. Converting to NoRegister effectively removes the SIB byte.
1682	if (MemOpNo >= 0) {
1683	MCOperand &IndexOp =
1684	Inst.getOperand(i: static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1685	if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1686	IndexOp = MCOperand::createReg(X86::NoRegister);
1687	}
1688
1689	if (isBranch(Inst)) {
1690	NewOpcode = getShortBranchOpcode(Opcode: OldOpcode);
1691	} else if (OldOpcode == X86::MOV64ri) {
1692	if (Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1693	const int64_t Imm =
1694	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1695	if (int64_t(Imm) == int64_t(int32_t(Imm)))
1696	NewOpcode = X86::MOV64ri32;
1697	}
1698	} else {
1699	// If it's arithmetic instruction check if signed operand fits in 1 byte.
1700	const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(Opcode: OldOpcode);
1701	if (ShortOpcode != OldOpcode &&
1702	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1703	int64_t Imm =
1704	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1705	if (int64_t(Imm) == int64_t(int8_t(Imm)))
1706	NewOpcode = ShortOpcode;
1707	}
1708	}
1709
1710	if (NewOpcode == OldOpcode)
1711	return false;
1712
1713	Inst.setOpcode(NewOpcode);
1714	return true;
1715	}
1716
1717	bool
1718	convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1719	bool AllowBasePtrStackMemOp) const override {
1720	// - Register-register moves are OK
1721	// - Stores are filtered out by opcode (no store CMOV)
1722	// - Non-stack loads are prohibited (generally unsafe)
1723	// - Stack loads are OK if AllowStackMemOp is true
1724	// - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1725	if (mayLoad(Inst)) {
1726	// If stack memory operands are not allowed, no loads are allowed
1727	if (!AllowStackMemOp)
1728	return false;
1729
1730	// If stack memory operands are allowed, check if it's a load from stack
1731	bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1732	MCPhysReg Reg;
1733	int32_t SrcImm;
1734	uint16_t StackPtrReg;
1735	int64_t StackOffset;
1736	uint8_t Size;
1737	bool IsStackAccess =
1738	isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1739	StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1740	// Prohibit non-stack-based loads
1741	if (!IsStackAccess)
1742	return false;
1743	// If stack memory operands are allowed, check if it's RBP-based
1744	if (!AllowBasePtrStackMemOp &&
1745	RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg))
1746	return false;
1747	}
1748
1749	unsigned NewOpcode = 0;
1750	switch (Inst.getOpcode()) {
1751	case X86::MOV16rr:
1752	NewOpcode = X86::CMOV16rr;
1753	break;
1754	case X86::MOV16rm:
1755	NewOpcode = X86::CMOV16rm;
1756	break;
1757	case X86::MOV32rr:
1758	NewOpcode = X86::CMOV32rr;
1759	break;
1760	case X86::MOV32rm:
1761	NewOpcode = X86::CMOV32rm;
1762	break;
1763	case X86::MOV64rr:
1764	NewOpcode = X86::CMOV64rr;
1765	break;
1766	case X86::MOV64rm:
1767	NewOpcode = X86::CMOV64rm;
1768	break;
1769	default:
1770	return false;
1771	}
1772	Inst.setOpcode(NewOpcode);
1773	// Insert CC at the end of prime operands, before annotations
1774	Inst.insert(I: Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1775	Op: MCOperand::createImm(Val: CC));
1776	// CMOV is a 3-operand MCInst, so duplicate the destination as src1
1777	Inst.insert(I: Inst.begin(), Op: Inst.getOperand(i: 0));
1778	return true;
1779	}
1780
1781	bool lowerTailCall(MCInst &Inst) override {
1782	if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1783	Inst.setOpcode(X86::JMP_1);
1784	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1785	return true;
1786	}
1787	return false;
1788	}
1789
1790	const MCSymbol *getTargetSymbol(const MCInst &Inst,
1791	unsigned OpNum = 0) const override {
1792	if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1793	return nullptr;
1794
1795	const MCOperand &Op = Inst.getOperand(i: OpNum);
1796	if (!Op.isExpr())
1797	return nullptr;
1798
1799	return MCPlusBuilder::getTargetSymbol(Expr: Op.getExpr());
1800	}
1801
1802	bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1803	const MCSymbol *&TBB, const MCSymbol *&FBB,
1804	MCInst *&CondBranch,
1805	MCInst *&UncondBranch) const override {
1806	auto I = End;
1807
1808	// Bottom-up analysis
1809	while (I != Begin) {
1810	--I;
1811
1812	// Ignore nops and CFIs
1813	if (isPseudo(Inst: *I))
1814	continue;
1815
1816	// Stop when we find the first non-terminator
1817	if (!isTerminator(Inst: *I))
1818	break;
1819
1820	if (!isBranch(Inst: *I))
1821	break;
1822
1823	// Handle unconditional branches.
1824	if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1825	I->getOpcode() == X86::JMP_4) &&
1826	!isTailCall(*I)) {
1827	// If any code was seen after this unconditional branch, we've seen
1828	// unreachable code. Ignore them.
1829	CondBranch = nullptr;
1830	UncondBranch = &*I;
1831	const MCSymbol *Sym = getTargetSymbol(Inst: *I);
1832	assert(Sym != nullptr &&
1833	"Couldn't extract BB symbol from jump operand");
1834	TBB = Sym;
1835	continue;
1836	}
1837
1838	// Ignore indirect branches
1839	if (getCondCode(Inst: *I) == X86::COND_INVALID)
1840	return false;
1841
1842	if (CondBranch == nullptr) {
1843	const MCSymbol *TargetBB = getTargetSymbol(Inst: *I);
1844	if (TargetBB == nullptr) {
1845	// Unrecognized branch target
1846	return false;
1847	}
1848	FBB = TBB;
1849	TBB = TargetBB;
1850	CondBranch = &*I;
1851	continue;
1852	}
1853
1854	llvm_unreachable("multiple conditional branches in one BB");
1855	}
1856	return true;
1857	}
1858
1859	/// Analyzes PIC-style jump table code template and return identified
1860	/// IndirectBranchType, MemLocInstr (all cases) and FixedEntryLoadInstr
1861	/// (POSSIBLE_PIC_FIXED_BRANCH case).
1862	template <typename Itr>
1863	std::tuple<IndirectBranchType, MCInst *, MCInst *>
1864	analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
1865	// Analyze PIC-style jump table code template:
1866	//
1867	// lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr
1868	// mov ({%r1|%r2}, %index, 4), {%r2|%r1}
1869	// add %r2, %r1
1870	// jmp *%r1
1871	//
1872	// or a fixed indirect jump template:
1873	//
1874	// movslq En(%rip), {%r2|%r1} <- FixedEntryLoadInstr
1875	// lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr
1876	// add %r2, %r1
1877	// jmp *%r1
1878	//
1879	// (with any irrelevant instructions in-between)
1880	//
1881	// When we call this helper we've already determined %r1 and %r2, and
1882	// reverse instruction iterator \p II is pointing to the ADD instruction.
1883	//
1884	// PIC jump table looks like following:
1885	//
1886	// JT: ----------
1887	// E1:| L1 - JT |
1888	// |----------|
1889	// E2:| L2 - JT |
1890	// |----------|
1891	// | |
1892	// ......
1893	// En:| Ln - JT |
1894	// ----------
1895	//
1896	// Where L1, L2, ..., Ln represent labels in the function.
1897	//
1898	// The actual relocations in the table will be of the form:
1899	//
1900	// Ln - JT
1901	// = (Ln - En) + (En - JT)
1902	// = R_X86_64_PC32(Ln) + En - JT
1903	// = R_X86_64_PC32(Ln + offsetof(En))
1904	//
1905	auto isRIPRel = [&](X86MemOperand &MO) {
1906	// NB: DispExpr should be set
1907	return MO.DispExpr != nullptr &&
1908	MO.BaseRegNum == RegInfo->getProgramCounter() &&
1909	MO.IndexRegNum == X86::NoRegister &&
1910	MO.SegRegNum == X86::NoRegister;
1911	};
1912	auto isIndexed = [](X86MemOperand &MO, MCPhysReg R) {
1913	// NB: IndexRegNum should be set.
1914	return MO.IndexRegNum != X86::NoRegister && MO.BaseRegNum == R &&
1915	MO.ScaleImm == 4 && MO.DispImm == 0 &&
1916	MO.SegRegNum == X86::NoRegister;
1917	};
1918	LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
1919	MCInst *FirstInstr = nullptr;
1920	MCInst *SecondInstr = nullptr;
1921	enum {
1922	NOMATCH = 0,
1923	MATCH_JUMP_TABLE,
1924	MATCH_FIXED_BRANCH,
1925	} MatchingState = NOMATCH;
1926	while (++II != IE) {
1927	MCInst &Instr = *II;
1928	const MCInstrDesc &InstrDesc = Info->get(Opcode: Instr.getOpcode());
1929	if (!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R1, RI: *RegInfo) &&
1930	!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R2, RI: *RegInfo)) {
1931	// Ignore instructions that don't affect R1, R2 registers.
1932	continue;
1933	}
1934	const bool IsMOVSXInstr = isMOVSX64rm32(Inst: Instr);
1935	const bool IsLEAInstr = isLEA64r(Inst: Instr);
1936	if (MatchingState == NOMATCH) {
1937	if (IsMOVSXInstr)
1938	MatchingState = MATCH_JUMP_TABLE;
1939	else if (IsLEAInstr)
1940	MatchingState = MATCH_FIXED_BRANCH;
1941	else
1942	break;
1943
1944	// Check if the first instruction is setting %r1 or %r2. In canonical
1945	// form lea sets %r1 and mov sets %r2. If it's the opposite - rename so
1946	// we have to only check a single form.
1947	unsigned DestReg = Instr.getOperand(i: 0).getReg();
1948	MCPhysReg &ExpectReg = MatchingState == MATCH_JUMP_TABLE ? R2 : R1;
1949	if (DestReg != ExpectReg)
1950	std::swap(R1, R2);
1951	if (DestReg != ExpectReg)
1952	break;
1953
1954	// Verify operands
1955	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
1956	if (!MO)
1957	break;
1958	if ((MatchingState == MATCH_JUMP_TABLE && isIndexed(*MO, R1)) ||
1959	(MatchingState == MATCH_FIXED_BRANCH && isRIPRel(*MO)))
1960	FirstInstr = &Instr;
1961	else
1962	break;
1963	} else {
1964	unsigned ExpectReg = MatchingState == MATCH_JUMP_TABLE ? R1 : R2;
1965	if (!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: ExpectReg, RI: *RegInfo))
1966	continue;
1967	if ((MatchingState == MATCH_JUMP_TABLE && !IsLEAInstr) ||
1968	(MatchingState == MATCH_FIXED_BRANCH && !IsMOVSXInstr))
1969	break;
1970	if (Instr.getOperand(i: 0).getReg() != ExpectReg)
1971	break;
1972
1973	// Verify operands.
1974	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
1975	if (!MO)
1976	break;
1977	if (!isRIPRel(*MO))
1978	break;
1979	SecondInstr = &Instr;
1980	break;
1981	}
1982	}
1983
1984	if (!SecondInstr)
1985	return std::make_tuple(IndirectBranchType::UNKNOWN, nullptr, nullptr);
1986
1987	if (MatchingState == MATCH_FIXED_BRANCH) {
1988	LLVM_DEBUG(dbgs() << "checking potential fixed indirect branch\n");
1989	return std::make_tuple(IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH,
1990	FirstInstr, SecondInstr);
1991	}
1992	LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
1993	return std::make_tuple(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
1994	SecondInstr, nullptr);
1995	}
1996
1997	IndirectBranchType
1998	analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
1999	InstructionIterator End, const unsigned PtrSize,
2000	MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
2001	unsigned &IndexRegNumOut, int64_t &DispValueOut,
2002	const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
2003	MCInst *&FixedEntryLoadInst) const override {
2004	// Try to find a (base) memory location from where the address for
2005	// the indirect branch is loaded. For X86-64 the memory will be specified
2006	// in the following format:
2007	//
2008	// {%rip}/{%basereg} + Imm + IndexReg * Scale
2009	//
2010	// We are interested in the cases where Scale == sizeof(uintptr_t) and
2011	// the contents of the memory are presumably an array of pointers to code.
2012	//
2013	// Normal jump table:
2014	//
2015	// jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr
2016	//
2017	// or
2018	//
2019	// mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr
2020	// ...
2021	// jmp %r1
2022	//
2023	// We handle PIC-style jump tables separately.
2024	//
2025	MemLocInstrOut = nullptr;
2026	BaseRegNumOut = X86::NoRegister;
2027	IndexRegNumOut = X86::NoRegister;
2028	DispValueOut = 0;
2029	DispExprOut = nullptr;
2030	FixedEntryLoadInst = nullptr;
2031
2032	std::reverse_iterator<InstructionIterator> II(End);
2033	std::reverse_iterator<InstructionIterator> IE(Begin);
2034
2035	IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2036
2037	// An instruction referencing memory used by jump instruction (directly or
2038	// via register). This location could be an array of function pointers
2039	// in case of indirect tail call, or a jump table.
2040	MCInst *MemLocInstr = nullptr;
2041
2042	if (MCPlus::getNumPrimeOperands(Inst: Instruction) == 1) {
2043	// If the indirect jump is on register - try to detect if the
2044	// register value is loaded from a memory location.
2045	assert(Instruction.getOperand(0).isReg() && "register operand expected");
2046	const unsigned R1 = Instruction.getOperand(i: 0).getReg();
2047	// Check if one of the previous instructions defines the jump-on register.
2048	for (auto PrevII = II; PrevII != IE; ++PrevII) {
2049	MCInst &PrevInstr = *PrevII;
2050	const MCInstrDesc &PrevInstrDesc = Info->get(Opcode: PrevInstr.getOpcode());
2051
2052	if (!PrevInstrDesc.hasDefOfPhysReg(MI: PrevInstr, Reg: R1, RI: *RegInfo))
2053	continue;
2054
2055	if (isMoveMem2Reg(Inst: PrevInstr)) {
2056	MemLocInstr = &PrevInstr;
2057	break;
2058	}
2059	if (isADD64rr(Inst: PrevInstr)) {
2060	unsigned R2 = PrevInstr.getOperand(i: 2).getReg();
2061	if (R1 == R2)
2062	return IndirectBranchType::UNKNOWN;
2063	std::tie(Type, MemLocInstr, FixedEntryLoadInst) =
2064	analyzePICJumpTable(PrevII, IE, R1, R2);
2065	break;
2066	}
2067	return IndirectBranchType::UNKNOWN;
2068	}
2069	if (!MemLocInstr) {
2070	// No definition seen for the register in this function so far. Could be
2071	// an input parameter - which means it is an external code reference.
2072	// It also could be that the definition happens to be in the code that
2073	// we haven't processed yet. Since we have to be conservative, return
2074	// as UNKNOWN case.
2075	return IndirectBranchType::UNKNOWN;
2076	}
2077	} else {
2078	MemLocInstr = &Instruction;
2079	}
2080
2081	const MCRegister RIPRegister = RegInfo->getProgramCounter();
2082
2083	// Analyze the memory location.
2084	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr);
2085	if (!MO)
2086	return IndirectBranchType::UNKNOWN;
2087
2088	BaseRegNumOut = MO->BaseRegNum;
2089	IndexRegNumOut = MO->IndexRegNum;
2090	DispValueOut = MO->DispImm;
2091	DispExprOut = MO->DispExpr;
2092
2093	if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) ||
2094	MO->SegRegNum != X86::NoRegister)
2095	return IndirectBranchType::UNKNOWN;
2096
2097	if (MemLocInstr == &Instruction &&
2098	(!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) {
2099	MemLocInstrOut = MemLocInstr;
2100	return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2101	}
2102
2103	switch (Type) {
2104	case IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE:
2105	if (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)
2106	return IndirectBranchType::UNKNOWN;
2107	break;
2108	case IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH:
2109	break;
2110	default:
2111	if (MO->ScaleImm != PtrSize)
2112	return IndirectBranchType::UNKNOWN;
2113	}
2114
2115	MemLocInstrOut = MemLocInstr;
2116
2117	return Type;
2118	}
2119
2120	/// Analyze a callsite to see if it could be a virtual method call. This only
2121	/// checks to see if the overall pattern is satisfied, it does not guarantee
2122	/// that the callsite is a true virtual method call.
2123	/// The format of virtual method calls that are recognized is one of the
2124	/// following:
2125	///
2126	/// Form 1: (found in debug code)
2127	/// add METHOD_OFFSET, %VtableReg
2128	/// mov (%VtableReg), %MethodReg
2129	/// ...
2130	/// call or jmp *%MethodReg
2131	///
2132	/// Form 2:
2133	/// mov METHOD_OFFSET(%VtableReg), %MethodReg
2134	/// ...
2135	/// call or jmp *%MethodReg
2136	///
2137	/// Form 3:
2138	/// ...
2139	/// call or jmp *METHOD_OFFSET(%VtableReg)
2140	///
2141	bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2142	InstructionIterator ForwardEnd,
2143	std::vector<MCInst *> &MethodFetchInsns,
2144	unsigned &VtableRegNum, unsigned &MethodRegNum,
2145	uint64_t &MethodOffset) const override {
2146	VtableRegNum = X86::NoRegister;
2147	MethodRegNum = X86::NoRegister;
2148	MethodOffset = 0;
2149
2150	std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2151	std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2152
2153	MCInst &CallInst = *Itr++;
2154	assert(isIndirectBranch(CallInst) || isCall(CallInst));
2155
2156	// The call can just be jmp offset(reg)
2157	if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) {
2158	if (!MO->DispExpr && MO->BaseRegNum != X86::RIP &&
2159	MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) {
2160	MethodRegNum = MO->BaseRegNum;
2161	if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister &&
2162	MO->SegRegNum == X86::NoRegister) {
2163	VtableRegNum = MethodRegNum;
2164	MethodOffset = MO->DispImm;
2165	MethodFetchInsns.push_back(&CallInst);
2166	return true;
2167	}
2168	}
2169	return false;
2170	}
2171	if (CallInst.getOperand(i: 0).isReg())
2172	MethodRegNum = CallInst.getOperand(i: 0).getReg();
2173	else
2174	return false;
2175
2176	if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2177	VtableRegNum = X86::NoRegister;
2178	MethodRegNum = X86::NoRegister;
2179	return false;
2180	}
2181
2182	// find load from vtable, this may or may not include the method offset
2183	while (Itr != End) {
2184	MCInst &CurInst = *Itr++;
2185	const MCInstrDesc &Desc = Info->get(Opcode: CurInst.getOpcode());
2186	if (Desc.hasDefOfPhysReg(MI: CurInst, Reg: MethodRegNum, RI: *RegInfo)) {
2187	if (!mayLoad(Inst: CurInst))
2188	return false;
2189	if (std::optional<X86MemOperand> MO =
2190	evaluateX86MemoryOperand(CurInst)) {
2191	if (!MO->DispExpr && MO->ScaleImm == 1 &&
2192	MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP &&
2193	MO->BaseRegNum != X86::NoRegister &&
2194	MO->IndexRegNum == X86::NoRegister &&
2195	MO->SegRegNum == X86::NoRegister) {
2196	VtableRegNum = MO->BaseRegNum;
2197	MethodOffset = MO->DispImm;
2198	MethodFetchInsns.push_back(&CurInst);
2199	if (MethodOffset != 0)
2200	return true;
2201	break;
2202	}
2203	}
2204	return false;
2205	}
2206	}
2207
2208	if (!VtableRegNum)
2209	return false;
2210
2211	// look for any adds affecting the method register.
2212	while (Itr != End) {
2213	MCInst &CurInst = *Itr++;
2214	const MCInstrDesc &Desc = Info->get(Opcode: CurInst.getOpcode());
2215	if (Desc.hasDefOfPhysReg(MI: CurInst, Reg: VtableRegNum, RI: *RegInfo)) {
2216	if (isADDri(Inst: CurInst)) {
2217	assert(!MethodOffset);
2218	MethodOffset = CurInst.getOperand(i: 2).getImm();
2219	MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst);
2220	break;
2221	}
2222	}
2223	}
2224
2225	return true;
2226	}
2227
2228	void createStackPointerIncrement(MCInst &Inst, int Size,
2229	bool NoFlagsClobber) const override {
2230	if (NoFlagsClobber) {
2231	Inst.setOpcode(X86::LEA64r);
2232	Inst.clear();
2233	Inst.addOperand(MCOperand::createReg(X86::RSP));
2234	Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg
2235	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2236	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2237	Inst.addOperand(Op: MCOperand::createImm(Val: -Size)); // Displacement
2238	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2239	return;
2240	}
2241	Inst.setOpcode(X86::SUB64ri8);
2242	Inst.clear();
2243	Inst.addOperand(MCOperand::createReg(X86::RSP));
2244	Inst.addOperand(MCOperand::createReg(X86::RSP));
2245	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2246	}
2247
2248	void createStackPointerDecrement(MCInst &Inst, int Size,
2249	bool NoFlagsClobber) const override {
2250	if (NoFlagsClobber) {
2251	Inst.setOpcode(X86::LEA64r);
2252	Inst.clear();
2253	Inst.addOperand(MCOperand::createReg(X86::RSP));
2254	Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg
2255	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2256	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2257	Inst.addOperand(Op: MCOperand::createImm(Val: Size)); // Displacement
2258	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2259	return;
2260	}
2261	Inst.setOpcode(X86::ADD64ri8);
2262	Inst.clear();
2263	Inst.addOperand(MCOperand::createReg(X86::RSP));
2264	Inst.addOperand(MCOperand::createReg(X86::RSP));
2265	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2266	}
2267
2268	void createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2269	const MCPhysReg &SrcReg, int Size) const override {
2270	unsigned NewOpcode;
2271	switch (Size) {
2272	default:
2273	llvm_unreachable("Invalid operand size");
2274	return;
2275	case 2: NewOpcode = X86::MOV16mr; break;
2276	case 4: NewOpcode = X86::MOV32mr; break;
2277	case 8: NewOpcode = X86::MOV64mr; break;
2278	}
2279	Inst.setOpcode(NewOpcode);
2280	Inst.clear();
2281	Inst.addOperand(Op: MCOperand::createReg(Reg: StackReg)); // BaseReg
2282	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2283	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2284	Inst.addOperand(Op: MCOperand::createImm(Val: Offset)); // Displacement
2285	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2286	Inst.addOperand(Op: MCOperand::createReg(Reg: SrcReg));
2287	}
2288
2289	void createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2290	int Offset, const MCPhysReg &DstReg,
2291	int Size) const override {
2292	return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2293	Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2294	DstReg, Size);
2295	}
2296
2297	void createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2298	const MCPhysReg &IndexReg, int64_t Offset,
2299	const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2300	const MCPhysReg &DstReg, int Size) const override {
2301	unsigned NewOpcode;
2302	switch (Size) {
2303	default:
2304	llvm_unreachable("Invalid operand size");
2305	return;
2306	case 2: NewOpcode = X86::MOV16rm; break;
2307	case 4: NewOpcode = X86::MOV32rm; break;
2308	case 8: NewOpcode = X86::MOV64rm; break;
2309	}
2310	Inst.setOpcode(NewOpcode);
2311	Inst.clear();
2312	Inst.addOperand(Op: MCOperand::createReg(Reg: DstReg));
2313	Inst.addOperand(Op: MCOperand::createReg(Reg: BaseReg));
2314	Inst.addOperand(Op: MCOperand::createImm(Val: Scale));
2315	Inst.addOperand(Op: MCOperand::createReg(Reg: IndexReg));
2316	if (OffsetExpr)
2317	Inst.addOperand(Op: MCOperand::createExpr(Val: OffsetExpr)); // Displacement
2318	else
2319	Inst.addOperand(Op: MCOperand::createImm(Val: Offset)); // Displacement
2320	Inst.addOperand(Op: MCOperand::createReg(Reg: AddrSegmentReg)); // AddrSegmentReg
2321	}
2322
2323	InstructionListType createLoadImmediate(const MCPhysReg Dest,
2324	uint64_t Imm) const override {
2325	InstructionListType Insts;
2326	Insts.emplace_back();
2327	Insts.back().setOpcode(X86::MOV64ri32);
2328	Insts.back().clear();
2329	Insts.back().addOperand(Op: MCOperand::createReg(Reg: Dest));
2330	Insts.back().addOperand(Op: MCOperand::createImm(Val: Imm));
2331	return Insts;
2332	}
2333
2334	void createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2335	const MCOperand &BaseReg, const MCOperand &Scale,
2336	const MCOperand &IndexReg, const MCOperand &Offset,
2337	const MCOperand &TmpReg) const override {
2338	// The code fragment we emit here is:
2339	//
2340	// mov32 (%base, %index, scale), %tmpreg
2341	// ijmp *(%tmpreg)
2342	//
2343	MCInst IJmp;
2344	IJmp.setOpcode(X86::JMP64r);
2345	IJmp.addOperand(Op: TmpReg);
2346
2347	MCInst Load;
2348	Load.setOpcode(X86::MOV32rm);
2349	Load.addOperand(Op: TmpReg);
2350	Load.addOperand(Op: BaseReg);
2351	Load.addOperand(Op: Scale);
2352	Load.addOperand(Op: IndexReg);
2353	Load.addOperand(Op: Offset);
2354	Load.addOperand(MCOperand::createReg(X86::NoRegister));
2355
2356	Insts.push_back(Load);
2357	Insts.push_back(IJmp);
2358	}
2359
2360	void createNoop(MCInst &Inst) const override {
2361	Inst.setOpcode(X86::NOOP);
2362	Inst.clear();
2363	}
2364
2365	void createReturn(MCInst &Inst) const override {
2366	Inst.setOpcode(X86::RET64);
2367	Inst.clear();
2368	}
2369
2370	InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2371	InstructionListType Code;
2372	if (ReturnEnd)
2373	Code.emplace_back(MCInstBuilder(X86::LEA64r)
2374	.addReg(X86::RAX)
2375	.addReg(X86::RDI)
2376	.addImm(1)
2377	.addReg(X86::RDX)
2378	.addImm(0)
2379	.addReg(X86::NoRegister));
2380	else
2381	Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2382	.addReg(X86::RAX)
2383	.addReg(X86::RDI));
2384
2385	Code.emplace_back(MCInstBuilder(X86::MOV32rr)
2386	.addReg(X86::ECX)
2387	.addReg(X86::EDX));
2388	Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64));
2389
2390	return Code;
2391	}
2392
2393	InstructionListType createOneByteMemcpy() const override {
2394	InstructionListType Code;
2395	Code.emplace_back(MCInstBuilder(X86::MOV8rm)
2396	.addReg(X86::CL)
2397	.addReg(X86::RSI)
2398	.addImm(0)
2399	.addReg(X86::NoRegister)
2400	.addImm(0)
2401	.addReg(X86::NoRegister));
2402	Code.emplace_back(MCInstBuilder(X86::MOV8mr)
2403	.addReg(X86::RDI)
2404	.addImm(0)
2405	.addReg(X86::NoRegister)
2406	.addImm(0)
2407	.addReg(X86::NoRegister)
2408	.addReg(X86::CL));
2409	Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2410	.addReg(X86::RAX)
2411	.addReg(X86::RDI));
2412	return Code;
2413	}
2414
2415	InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2416	const MCSymbol *Target,
2417	MCContext *Ctx) const override {
2418	InstructionListType Code;
2419	Code.emplace_back(MCInstBuilder(X86::CMP64ri8)
2420	.addReg(RegNo)
2421	.addImm(Imm));
2422	Code.emplace_back(MCInstBuilder(X86::JCC_1)
2423	.addExpr(MCSymbolRefExpr::create(
2424	Target, MCSymbolRefExpr::VK_None, *Ctx))
2425	.addImm(X86::COND_E));
2426	return Code;
2427	}
2428
2429	InstructionListType createCmpJNE(MCPhysReg RegNo, int64_t Imm,
2430	const MCSymbol *Target,
2431	MCContext *Ctx) const override {
2432	InstructionListType Code;
2433	Code.emplace_back(MCInstBuilder(X86::CMP64ri8).addReg(RegNo).addImm(Imm));
2434	Code.emplace_back(MCInstBuilder(X86::JCC_1)
2435	.addExpr(MCSymbolRefExpr::create(
2436	Target, MCSymbolRefExpr::VK_None, *Ctx))
2437	.addImm(X86::COND_NE));
2438	return Code;
2439	}
2440
2441	std::optional<Relocation>
2442	createRelocation(const MCFixup &Fixup,
2443	const MCAsmBackend &MAB) const override {
2444	MCFixupKindInfo FKI = MAB.getFixupKindInfo(Kind: Fixup.getKind());
2445
2446	assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2447	const uint64_t RelOffset = Fixup.getOffset();
2448
2449	uint32_t RelType;
2450	if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2451	switch (FKI.TargetSize) {
2452	default:
2453	return std::nullopt;
2454	case 8: RelType = ELF::R_X86_64_PC8; break;
2455	case 16: RelType = ELF::R_X86_64_PC16; break;
2456	case 32: RelType = ELF::R_X86_64_PC32; break;
2457	case 64: RelType = ELF::R_X86_64_PC64; break;
2458	}
2459	} else {
2460	switch (FKI.TargetSize) {
2461	default:
2462	return std::nullopt;
2463	case 8: RelType = ELF::R_X86_64_8; break;
2464	case 16: RelType = ELF::R_X86_64_16; break;
2465	case 32: RelType = ELF::R_X86_64_32; break;
2466	case 64: RelType = ELF::R_X86_64_64; break;
2467	}
2468	}
2469
2470	auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
2471
2472	return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
2473	}
2474
2475	bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2476	int64_t Addend, MCContext *Ctx, int64_t &Value,
2477	uint32_t RelType) const override {
2478	unsigned ImmOpNo = -1U;
2479
2480	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2481	++Index) {
2482	if (Inst.getOperand(i: Index).isImm()) {
2483	ImmOpNo = Index;
2484	// TODO: this is a bit hacky. It finds the correct operand by
2485	// searching for a specific immediate value. If no value is
2486	// provided it defaults to the last immediate operand found.
2487	// This could lead to unexpected results if the instruction
2488	// has more than one immediate with the same value.
2489	if (Inst.getOperand(i: ImmOpNo).getImm() == Value)
2490	break;
2491	}
2492	}
2493
2494	if (ImmOpNo == -1U)
2495	return false;
2496
2497	Value = Inst.getOperand(i: ImmOpNo).getImm();
2498
2499	setOperandToSymbolRef(Inst, OpNum: ImmOpNo, Symbol, Addend, Ctx, RelType);
2500
2501	return true;
2502	}
2503
2504	bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2505	int64_t Imm) const override {
2506
2507	enum CheckSignExt : uint8_t {
2508	NOCHECK = 0,
2509	CHECK8,
2510	CHECK32,
2511	};
2512
2513	using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2514	struct InstInfo {
2515	// Size in bytes that Inst loads from memory.
2516	uint8_t DataSize;
2517
2518	// True when the target operand has to be duplicated because the opcode
2519	// expects a LHS operand.
2520	bool HasLHS;
2521
2522	// List of checks and corresponding opcodes to be used. We try to use the
2523	// smallest possible immediate value when various sizes are available,
2524	// hence we may need to check whether a larger constant fits in a smaller
2525	// immediate.
2526	CheckList Checks;
2527	};
2528
2529	InstInfo I;
2530
2531	switch (Inst.getOpcode()) {
2532	default: {
2533	switch (getPushSize(Inst)) {
2534
2535	case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break;
2536	case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break;
2537	case 8: I = {8, false, {{CHECK8, X86::PUSH64i8},
2538	{CHECK32, X86::PUSH64i32},
2539	{NOCHECK, Inst.getOpcode()}}}; break;
2540	default: return false;
2541	}
2542	break;
2543	}
2544
2545	// MOV
2546	case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
2547	case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
2548	case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
2549	case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32},
2550	{NOCHECK, X86::MOV64ri}}}; break;
2551
2552	case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break;
2553	case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break;
2554	case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break;
2555	case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32},
2556	{NOCHECK, X86::MOV64mr}}}; break;
2557
2558	// MOVZX
2559	case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
2560	case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
2561	case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
2562
2563	// CMP
2564	case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
2565	case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8},
2566	{NOCHECK, X86::CMP16ri}}}; break;
2567	case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8},
2568	{NOCHECK, X86::CMP32ri}}}; break;
2569	case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8},
2570	{CHECK32, X86::CMP64ri32},
2571	{NOCHECK, X86::CMP64rr}}}; break;
2572
2573	// TEST
2574	case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
2575	case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
2576	case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
2577	case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32},
2578	{NOCHECK, X86::TEST64rr}}}; break;
2579
2580	// ADD
2581	case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
2582	case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8},
2583	{NOCHECK, X86::ADD16ri}}}; break;
2584	case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8},
2585	{NOCHECK, X86::ADD32ri}}}; break;
2586	case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8},
2587	{CHECK32, X86::ADD64ri32},
2588	{NOCHECK, X86::ADD64rr}}}; break;
2589
2590	// SUB
2591	case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
2592	case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8},
2593	{NOCHECK, X86::SUB16ri}}}; break;
2594	case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8},
2595	{NOCHECK, X86::SUB32ri}}}; break;
2596	case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8},
2597	{CHECK32, X86::SUB64ri32},
2598	{NOCHECK, X86::SUB64rr}}}; break;
2599
2600	// AND
2601	case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
2602	case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8},
2603	{NOCHECK, X86::AND16ri}}}; break;
2604	case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8},
2605	{NOCHECK, X86::AND32ri}}}; break;
2606	case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8},
2607	{CHECK32, X86::AND64ri32},
2608	{NOCHECK, X86::AND64rr}}}; break;
2609
2610	// OR
2611	case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
2612	case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8},
2613	{NOCHECK, X86::OR16ri}}}; break;
2614	case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8},
2615	{NOCHECK, X86::OR32ri}}}; break;
2616	case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8},
2617	{CHECK32, X86::OR64ri32},
2618	{NOCHECK, X86::OR64rr}}}; break;
2619
2620	// XOR
2621	case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
2622	case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8},
2623	{NOCHECK, X86::XOR16ri}}}; break;
2624	case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8},
2625	{NOCHECK, X86::XOR32ri}}}; break;
2626	case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8},
2627	{CHECK32, X86::XOR64ri32},
2628	{NOCHECK, X86::XOR64rr}}}; break;
2629	}
2630
2631	// Compute the new opcode.
2632	unsigned NewOpcode = 0;
2633	for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2634	NewOpcode = Check.second;
2635	if (Check.first == NOCHECK)
2636	break;
2637	if (Check.first == CHECK8 && isInt<8>(Imm))
2638	break;
2639	if (Check.first == CHECK32 && isInt<32>(Imm))
2640	break;
2641	}
2642	if (NewOpcode == Inst.getOpcode())
2643	return false;
2644
2645	const MCInstrDesc &InstDesc = Info->get(Opcode: Inst.getOpcode());
2646
2647	unsigned NumFound = 0;
2648	for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2649	E = InstDesc.getNumOperands();
2650	Index != E; ++Index)
2651	if (Inst.getOperand(i: Index).isReg() &&
2652	Inst.getOperand(i: Index).getReg() == Register)
2653	NumFound++;
2654
2655	if (NumFound != 1)
2656	return false;
2657
2658	MCOperand TargetOp = Inst.getOperand(i: 0);
2659	Inst.clear();
2660	Inst.setOpcode(NewOpcode);
2661	Inst.addOperand(Op: TargetOp);
2662	if (I.HasLHS)
2663	Inst.addOperand(Op: TargetOp);
2664	Inst.addOperand(Op: MCOperand::createImm(Val: Imm));
2665
2666	return true;
2667	}
2668
2669	bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2670	unsigned ReplaceWith) const override {
2671
2672	// Get the HasLHS value so that iteration can be done
2673	bool HasLHS;
2674	if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) ||
2675	X86::isSUB(Inst.getOpcode())) {
2676	HasLHS = true;
2677	} else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) ||
2678	X86::isTEST(Inst.getOpcode())) {
2679	HasLHS = false;
2680	} else {
2681	switch (Inst.getOpcode()) {
2682	case X86::MOV8rr:
2683	case X86::MOV8rm:
2684	case X86::MOV8mr:
2685	case X86::MOV8ri:
2686	case X86::MOV16rr:
2687	case X86::MOV16rm:
2688	case X86::MOV16mr:
2689	case X86::MOV16ri:
2690	case X86::MOV32rr:
2691	case X86::MOV32rm:
2692	case X86::MOV32mr:
2693	case X86::MOV32ri:
2694	case X86::MOV64rr:
2695	case X86::MOV64rm:
2696	case X86::MOV64mr:
2697	case X86::MOV64ri:
2698	case X86::MOVZX16rr8:
2699	case X86::MOVZX32rr8:
2700	case X86::MOVZX32rr16:
2701	case X86::MOVSX32rm8:
2702	case X86::MOVSX32rr8:
2703	case X86::MOVSX64rm32:
2704	case X86::LEA64r:
2705	HasLHS = false;
2706	break;
2707	default:
2708	return false;
2709	}
2710	}
2711
2712	const MCInstrDesc &InstDesc = Info->get(Opcode: Inst.getOpcode());
2713
2714	bool FoundOne = false;
2715
2716	// Iterate only through src operands that arent also dest operands
2717	for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2718	E = InstDesc.getNumOperands();
2719	Index != E; ++Index) {
2720	BitVector RegAliases = getAliases(Reg: ToReplace, OnlySmaller: true);
2721	if (!Inst.getOperand(i: Index).isReg() ||
2722	!RegAliases.test(Idx: Inst.getOperand(i: Index).getReg()))
2723	continue;
2724	// Resize register if needed
2725	unsigned SizedReplaceWith = getAliasSized(
2726	Reg: ReplaceWith, Size: getRegSize(Reg: Inst.getOperand(i: Index).getReg()));
2727	MCOperand NewOperand = MCOperand::createReg(Reg: SizedReplaceWith);
2728	Inst.getOperand(i: Index) = NewOperand;
2729	FoundOne = true;
2730	}
2731
2732	// Return true if at least one operand was replaced
2733	return FoundOne;
2734	}
2735
2736	void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2737	MCContext *Ctx) const override {
2738	Inst.clear();
2739	Inst.setOpcode(X86::JMP_1);
2740	Inst.clear();
2741	Inst.addOperand(Op: MCOperand::createExpr(
2742	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2743	}
2744
2745	void createLongUncondBranch(MCInst &Inst, const MCSymbol *Target,
2746	MCContext *Ctx) const override {
2747	Inst.setOpcode(X86::JMP_4);
2748	Inst.clear();
2749	Inst.addOperand(Op: MCOperand::createExpr(
2750	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2751	}
2752
2753	void createCall(MCInst &Inst, const MCSymbol *Target,
2754	MCContext *Ctx) override {
2755	Inst.setOpcode(X86::CALL64pcrel32);
2756	Inst.clear();
2757	Inst.addOperand(Op: MCOperand::createExpr(
2758	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2759	}
2760
2761	void createTailCall(MCInst &Inst, const MCSymbol *Target,
2762	MCContext *Ctx) override {
2763	return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2764	}
2765
2766	void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2767	MCContext *Ctx) override {
2768	Seq.clear();
2769	Seq.emplace_back();
2770	createDirectCall(Inst&: Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2771	}
2772
2773	void createTrap(MCInst &Inst) const override {
2774	Inst.clear();
2775	Inst.setOpcode(X86::TRAP);
2776	}
2777
2778	void createCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC,
2779	MCContext *Ctx) const override {
2780	Inst.setOpcode(X86::JCC_1);
2781	Inst.clear();
2782	Inst.addOperand(Op: MCOperand::createExpr(
2783	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2784	Inst.addOperand(Op: MCOperand::createImm(Val: CC));
2785	}
2786
2787	void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC,
2788	MCContext *Ctx) const override {
2789	Inst.setOpcode(X86::JCC_4);
2790	Inst.clear();
2791	Inst.addOperand(Op: MCOperand::createExpr(
2792	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2793	Inst.addOperand(Op: MCOperand::createImm(Val: CC));
2794	}
2795
2796	void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2797	MCContext *Ctx) const override {
2798	unsigned InvCC = getInvertedCondCode(CC: getCondCode(Inst));
2799	assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2800	Inst.getOperand(i: Info->get(Opcode: Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2801	Inst.getOperand(i: 0) = MCOperand::createExpr(
2802	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
2803	}
2804
2805	bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2806	unsigned CC) const override {
2807	if (CC == X86::COND_INVALID)
2808	return false;
2809	Inst.getOperand(i: Info->get(Opcode: Inst.getOpcode()).NumOperands - 1).setImm(CC);
2810	Inst.getOperand(i: 0) = MCOperand::createExpr(
2811	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
2812	return true;
2813	}
2814
2815	unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2816	switch (CC) {
2817	default: return X86::COND_INVALID;
2818
2819	case X86::COND_E: return X86::COND_E;
2820	case X86::COND_NE: return X86::COND_E;
2821
2822	case X86::COND_L: return X86::COND_L;
2823	case X86::COND_GE: return X86::COND_L;
2824
2825	case X86::COND_LE: return X86::COND_G;
2826	case X86::COND_G: return X86::COND_G;
2827
2828	case X86::COND_B: return X86::COND_B;
2829	case X86::COND_AE: return X86::COND_B;
2830
2831	case X86::COND_BE: return X86::COND_A;
2832	case X86::COND_A: return X86::COND_A;
2833
2834	case X86::COND_S: return X86::COND_S;
2835	case X86::COND_NS: return X86::COND_S;
2836
2837	case X86::COND_P: return X86::COND_P;
2838	case X86::COND_NP: return X86::COND_P;
2839
2840	case X86::COND_O: return X86::COND_O;
2841	case X86::COND_NO: return X86::COND_O;
2842	}
2843	}
2844
2845	void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
2846	MCContext *Ctx) const override {
2847	assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
2848	"Invalid instruction");
2849	Inst.getOperand(i: 0) = MCOperand::createExpr(
2850	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
2851	}
2852
2853	MCPhysReg getX86R11() const override { return X86::R11; }
2854
2855	unsigned getShortBranchOpcode(unsigned Opcode) const override {
2856	switch (Opcode) {
2857	default:
2858	return Opcode;
2859	case X86::JMP_2:
2860	return X86::JMP_1;
2861	case X86::JMP_4:
2862	return X86::JMP_1;
2863	case X86::JCC_2:
2864	return X86::JCC_1;
2865	case X86::JCC_4:
2866	return X86::JCC_1;
2867	}
2868	}
2869
2870	MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
2871	// FIXME: this should depend on the calling convention.
2872	switch (ArgNo) {
2873	case 0: return X86::RDI;
2874	case 1: return X86::RSI;
2875	case 2: return X86::RDX;
2876	case 3: return X86::RCX;
2877	case 4: return X86::R8;
2878	case 5: return X86::R9;
2879	default: return getNoRegister();
2880	}
2881	}
2882
2883	void createPause(MCInst &Inst) const override {
2884	Inst.clear();
2885	Inst.setOpcode(X86::PAUSE);
2886	}
2887
2888	void createLfence(MCInst &Inst) const override {
2889	Inst.clear();
2890	Inst.setOpcode(X86::LFENCE);
2891	}
2892
2893	void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
2894	bool IsTailCall) override {
2895	Inst.clear();
2896	Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
2897	Inst.addOperand(Op: MCOperand::createExpr(
2898	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2899	if (IsTailCall)
2900	setTailCall(Inst);
2901	}
2902
2903	void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
2904	MCContext *Ctx, bool IsTailCall) override {
2905	Seq.clear();
2906	MCInst Inst;
2907	Inst.setOpcode(X86::JMP_1);
2908	Inst.addOperand(Op: MCOperand::createExpr(
2909	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2910	if (IsTailCall)
2911	setTailCall(Inst);
2912	Seq.emplace_back(Inst);
2913	}
2914
2915	bool isConditionalMove(const MCInst &Inst) const override {
2916	unsigned OpCode = Inst.getOpcode();
2917	return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
2918	OpCode == X86::CMOV64rr);
2919	}
2920
2921	bool isBranchOnMem(const MCInst &Inst) const override {
2922	unsigned OpCode = Inst.getOpcode();
2923	if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
2924	OpCode == X86::JMP64m)
2925	return true;
2926
2927	return false;
2928	}
2929
2930	bool isBranchOnReg(const MCInst &Inst) const override {
2931	unsigned OpCode = Inst.getOpcode();
2932	if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
2933	OpCode == X86::JMP64r)
2934	return true;
2935
2936	return false;
2937	}
2938
2939	void createPushRegister(MCInst &Inst, MCPhysReg Reg,
2940	unsigned Size) const override {
2941	Inst.clear();
2942	unsigned NewOpcode = 0;
2943	if (Reg == X86::EFLAGS) {
2944	switch (Size) {
2945	case 2: NewOpcode = X86::PUSHF16; break;
2946	case 4: NewOpcode = X86::PUSHF32; break;
2947	case 8: NewOpcode = X86::PUSHF64; break;
2948	default:
2949	llvm_unreachable("Unexpected size");
2950	}
2951	Inst.setOpcode(NewOpcode);
2952	return;
2953	}
2954	switch (Size) {
2955	case 2: NewOpcode = X86::PUSH16r; break;
2956	case 4: NewOpcode = X86::PUSH32r; break;
2957	case 8: NewOpcode = X86::PUSH64r; break;
2958	default:
2959	llvm_unreachable("Unexpected size");
2960	}
2961	Inst.setOpcode(NewOpcode);
2962	Inst.addOperand(Op: MCOperand::createReg(Reg));
2963	}
2964
2965	void createPopRegister(MCInst &Inst, MCPhysReg Reg,
2966	unsigned Size) const override {
2967	Inst.clear();
2968	unsigned NewOpcode = 0;
2969	if (Reg == X86::EFLAGS) {
2970	switch (Size) {
2971	case 2: NewOpcode = X86::POPF16; break;
2972	case 4: NewOpcode = X86::POPF32; break;
2973	case 8: NewOpcode = X86::POPF64; break;
2974	default:
2975	llvm_unreachable("Unexpected size");
2976	}
2977	Inst.setOpcode(NewOpcode);
2978	return;
2979	}
2980	switch (Size) {
2981	case 2: NewOpcode = X86::POP16r; break;
2982	case 4: NewOpcode = X86::POP32r; break;
2983	case 8: NewOpcode = X86::POP64r; break;
2984	default:
2985	llvm_unreachable("Unexpected size");
2986	}
2987	Inst.setOpcode(NewOpcode);
2988	Inst.addOperand(Op: MCOperand::createReg(Reg));
2989	}
2990
2991	void createPushFlags(MCInst &Inst, unsigned Size) const override {
2992	return createPushRegister(Inst, X86::EFLAGS, Size);
2993	}
2994
2995	void createPopFlags(MCInst &Inst, unsigned Size) const override {
2996	return createPopRegister(Inst, X86::EFLAGS, Size);
2997	}
2998
2999	void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
3000	unsigned Size) const {
3001	unsigned int Opcode;
3002	switch (Size) {
3003	case 1: Opcode = X86::ADD8ri; break;
3004	case 2: Opcode = X86::ADD16ri; break;
3005	case 4: Opcode = X86::ADD32ri; break;
3006	default:
3007	llvm_unreachable("Unexpected size");
3008	}
3009	Inst.setOpcode(Opcode);
3010	Inst.clear();
3011	Inst.addOperand(Op: MCOperand::createReg(Reg));
3012	Inst.addOperand(Op: MCOperand::createReg(Reg));
3013	Inst.addOperand(Op: MCOperand::createImm(Val: Value));
3014	}
3015
3016	void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3017	unsigned Size) const {
3018	unsigned int Opcode;
3019	switch (Size) {
3020	case 1: Opcode = X86::MOV8ri; break;
3021	case 2: Opcode = X86::MOV16ri; break;
3022	case 4: Opcode = X86::MOV32ri; break;
3023	// Writing to a 32-bit register always zeros the upper 32 bits of the
3024	// full-width register
3025	case 8:
3026	Opcode = X86::MOV32ri;
3027	Reg = getAliasSized(Reg, Size: 4);
3028	break;
3029	default:
3030	llvm_unreachable("Unexpected size");
3031	}
3032	Inst.setOpcode(Opcode);
3033	Inst.clear();
3034	Inst.addOperand(Op: MCOperand::createReg(Reg));
3035	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
3036	}
3037
3038	void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3039	Inst.setOpcode(X86::SETCCr);
3040	Inst.clear();
3041	Inst.addOperand(Op: MCOperand::createReg(Reg));
3042	Inst.addOperand(Op: MCOperand::createImm(Val: X86::COND_O));
3043	}
3044
3045	void createX86Lahf(MCInst &Inst) const {
3046	Inst.setOpcode(X86::LAHF);
3047	Inst.clear();
3048	}
3049
3050	void createX86Sahf(MCInst &Inst) const {
3051	Inst.setOpcode(X86::SAHF);
3052	Inst.clear();
3053	}
3054
3055	InstructionListType
3056	createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
3057	unsigned CodePointerSize) const override {
3058	InstructionListType Instrs(IsLeaf ? 13 : 11);
3059	unsigned int I = 0;
3060
3061	// Don't clobber application red zone (ABI dependent)
3062	if (IsLeaf)
3063	createStackPointerIncrement(Inst&: Instrs[I++], Size: 128,
3064	/*NoFlagsClobber=*/true);
3065
3066	// Performance improvements based on the optimization discussed at
3067	// https://reviews.llvm.org/D6629
3068	// LAHF/SAHF are used instead of PUSHF/POPF
3069	// PUSHF
3070	createPushRegister(Instrs[I++], X86::RAX, 8);
3071	createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3072	createX86Lahf(Inst&: Instrs[I++]);
3073	createPushRegister(Instrs[I++], X86::RAX, 8);
3074	createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3075	createX86SaveOVFlagToRegister(Instrs[I++], X86::AL);
3076	// LOCK INC
3077	InstructionListType IncMem = createIncMemory(Target, Ctx);
3078	assert(IncMem.size() == 1 && "Invalid IncMem size");
3079	std::copy(IncMem.begin(), IncMem.end(), Instrs.begin() + I);
3080	I += IncMem.size();
3081	// POPF
3082	createAddRegImm(Instrs[I++], X86::AL, 127, 1);
3083	createPopRegister(Instrs[I++], X86::RAX, 8);
3084	createX86Sahf(Inst&: Instrs[I++]);
3085	createPopRegister(Instrs[I++], X86::RAX, 8);
3086
3087	if (IsLeaf)
3088	createStackPointerDecrement(Inst&: Instrs[I], Size: 128,
3089	/*NoFlagsClobber=*/true);
3090	return Instrs;
3091	}
3092
3093	void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3094	int64_t Disp) const {
3095	Inst.setOpcode(X86::XCHG64rm);
3096	Inst.clear();
3097	Inst.addOperand(Op: MCOperand::createReg(Reg: Source));
3098	Inst.addOperand(Op: MCOperand::createReg(Reg: Source));
3099	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg)); // BaseReg
3100	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3101	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3102	Inst.addOperand(Op: MCOperand::createImm(Val: Disp)); // Displacement
3103	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3104	}
3105
3106	void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3107	int64_t Disp) const {
3108	Inst.setOpcode(X86::JMP64m);
3109	Inst.clear();
3110	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg)); // BaseReg
3111	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3112	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3113	Inst.addOperand(Op: MCOperand::createImm(Val: Disp)); // Displacement
3114	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3115	}
3116
3117	InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
3118	MCSymbol *HandlerFuncAddr,
3119	int CallSiteID,
3120	MCContext *Ctx) override {
3121	// Check if the target address expression used in the original indirect call
3122	// uses the stack pointer, which we are going to clobber.
3123	static BitVector SPAliases(getAliases(X86::RSP));
3124	bool UsesSP = any_of(useOperands(Inst&: CallInst), [&](const MCOperand &Op) {
3125	return Op.isReg() && SPAliases[Op.getReg()];
3126	});
3127
3128	InstructionListType Insts;
3129	MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3130	// Code sequence used to enter indirect call instrumentation helper:
3131	// push %rdi
3132	// add $8, %rsp ;; $rsp may be used in target, so fix it to prev val
3133	// movq target, %rdi ;; via convertIndirectCallTargetToLoad
3134	// sub $8, %rsp ;; restore correct stack value
3135	// push %rdi
3136	// movq $CallSiteID, %rdi
3137	// push %rdi
3138	// callq/jmp HandlerFuncAddr
3139	Insts.emplace_back();
3140	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3141	if (UsesSP) { // Only adjust SP if we really need to
3142	Insts.emplace_back();
3143	createStackPointerDecrement(Inst&: Insts.back(), Size: 8, /*NoFlagsClobber=*/false);
3144	}
3145	Insts.emplace_back(CallInst);
3146	// Insts.back() and CallInst now share the same annotation instruction.
3147	// Strip it from Insts.back(), only preserving tail call annotation.
3148	stripAnnotations(Inst&: Insts.back(), /*KeepTC=*/true);
3149	convertIndirectCallToLoad(Inst&: Insts.back(), Reg: TempReg);
3150	if (UsesSP) {
3151	Insts.emplace_back();
3152	createStackPointerIncrement(Inst&: Insts.back(), Size: 8, /*NoFlagsClobber=*/false);
3153	}
3154	Insts.emplace_back();
3155	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3156	InstructionListType LoadImm = createLoadImmediate(Dest: TempReg, Imm: CallSiteID);
3157	Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end());
3158	Insts.emplace_back();
3159	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3160
3161	MCInst &NewCallInst = Insts.emplace_back();
3162	createDirectCall(Inst&: NewCallInst, Target: HandlerFuncAddr, Ctx, IsTailCall: isTailCall(Inst: CallInst));
3163
3164	// Carry over metadata including tail call marker if present.
3165	stripAnnotations(Inst&: NewCallInst);
3166	moveAnnotations(SrcInst: std::move(CallInst), DstInst&: NewCallInst);
3167
3168	return Insts;
3169	}
3170
3171	InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3172	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3173	// We just need to undo the sequence created for every ind call in
3174	// instrumentIndirectTarget(), which can be accomplished minimally with:
3175	// popfq
3176	// pop %rdi
3177	// add $16, %rsp
3178	// xchg (%rsp), %rdi
3179	// jmp *-8(%rsp)
3180	InstructionListType Insts(5);
3181	createPopFlags(Inst&: Insts[0], Size: 8);
3182	createPopRegister(Inst&: Insts[1], Reg: TempReg, Size: 8);
3183	createStackPointerDecrement(Inst&: Insts[2], Size: 16, /*NoFlagsClobber=*/false);
3184	createSwap(Insts[3], TempReg, X86::RSP, 0);
3185	createIndirectBranch(Insts[4], X86::RSP, -8);
3186	return Insts;
3187	}
3188
3189	InstructionListType
3190	createInstrumentedIndTailCallHandlerExitBB() const override {
3191	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3192	// Same thing as above, but for tail calls
3193	// popfq
3194	// add $16, %rsp
3195	// pop %rdi
3196	// jmp *-16(%rsp)
3197	InstructionListType Insts(4);
3198	createPopFlags(Inst&: Insts[0], Size: 8);
3199	createStackPointerDecrement(Inst&: Insts[1], Size: 16, /*NoFlagsClobber=*/false);
3200	createPopRegister(Inst&: Insts[2], Reg: TempReg, Size: 8);
3201	createIndirectBranch(Insts[3], X86::RSP, -16);
3202	return Insts;
3203	}
3204
3205	InstructionListType
3206	createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3207	const MCSymbol *IndCallHandler,
3208	MCContext *Ctx) override {
3209	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3210	// Code sequence used to check whether InstrTampoline was initialized
3211	// and call it if so, returns via IndCallHandler.
3212	// pushfq
3213	// mov InstrTrampoline,%rdi
3214	// cmp $0x0,%rdi
3215	// je IndCallHandler
3216	// callq *%rdi
3217	// jmpq IndCallHandler
3218	InstructionListType Insts;
3219	Insts.emplace_back();
3220	createPushFlags(Inst&: Insts.back(), Size: 8);
3221	Insts.emplace_back();
3222	createMove(Inst&: Insts.back(), Src: InstrTrampoline, Reg: TempReg, Ctx);
3223	InstructionListType cmpJmp = createCmpJE(RegNo: TempReg, Imm: 0, Target: IndCallHandler, Ctx);
3224	Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
3225	Insts.emplace_back();
3226	Insts.back().setOpcode(X86::CALL64r);
3227	Insts.back().addOperand(Op: MCOperand::createReg(Reg: TempReg));
3228	Insts.emplace_back();
3229	createDirectCall(Inst&: Insts.back(), Target: IndCallHandler, Ctx, /*IsTailCall*/ true);
3230	return Insts;
3231	}
3232
3233	InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3234	InstructionListType Insts(2);
3235	MCSymbol *NumLocs = Ctx->getOrCreateSymbol(Name: "__bolt_num_counters");
3236	createMove(Insts[0], NumLocs, X86::EAX, Ctx);
3237	createReturn(Inst&: Insts[1]);
3238	return Insts;
3239	}
3240
3241	InstructionListType
3242	createInstrLocationsGetter(MCContext *Ctx) const override {
3243	InstructionListType Insts(2);
3244	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_locations");
3245	createLea(Insts[0], Locs, X86::EAX, Ctx);
3246	createReturn(Inst&: Insts[1]);
3247	return Insts;
3248	}
3249
3250	InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3251	InstructionListType Insts(2);
3252	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_tables");
3253	createLea(Insts[0], Locs, X86::EAX, Ctx);
3254	createReturn(Inst&: Insts[1]);
3255	return Insts;
3256	}
3257
3258	InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3259	InstructionListType Insts(2);
3260	MCSymbol *NumFuncs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_num_funcs");
3261	createMove(Insts[0], NumFuncs, X86::EAX, Ctx);
3262	createReturn(Inst&: Insts[1]);
3263	return Insts;
3264	}
3265
3266	InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3267	MCContext *Ctx) override {
3268	InstructionListType Insts(1);
3269	createUncondBranch(Inst&: Insts[0], TBB: TgtSym, Ctx);
3270	return Insts;
3271	}
3272
3273	BlocksVectorTy indirectCallPromotion(
3274	const MCInst &CallInst,
3275	const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3276	const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3277	const std::vector<MCInst *> &MethodFetchInsns,
3278	const bool MinimizeCodeSize, MCContext *Ctx) override {
3279	const bool IsTailCall = isTailCall(Inst: CallInst);
3280	const bool IsJumpTable = getJumpTable(Inst: CallInst) != 0;
3281	BlocksVectorTy Results;
3282
3283	// Label for the current code block.
3284	MCSymbol *NextTarget = nullptr;
3285
3286	// The join block which contains all the instructions following CallInst.
3287	// MergeBlock remains null if CallInst is a tail call.
3288	MCSymbol *MergeBlock = nullptr;
3289
3290	unsigned FuncAddrReg = X86::R10;
3291
3292	const bool LoadElim = !VtableSyms.empty();
3293	assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3294	"There must be a vtable entry for every method "
3295	"in the targets vector.");
3296
3297	if (MinimizeCodeSize && !LoadElim) {
3298	std::set<unsigned> UsedRegs;
3299
3300	for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(Inst: CallInst); ++I) {
3301	const MCOperand &Op = CallInst.getOperand(i: I);
3302	if (Op.isReg())
3303	UsedRegs.insert(Op.getReg());
3304	}
3305
3306	if (UsedRegs.count(X86::R10) == 0)
3307	FuncAddrReg = X86::R10;
3308	else if (UsedRegs.count(X86::R11) == 0)
3309	FuncAddrReg = X86::R11;
3310	else
3311	return Results;
3312	}
3313
3314	const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3315	assert(MergeBlock);
3316	NewCall.push_back(x: CallInst);
3317	MCInst &Merge = NewCall.back();
3318	Merge.clear();
3319	createUncondBranch(Inst&: Merge, TBB: MergeBlock, Ctx);
3320	};
3321
3322	for (unsigned int i = 0; i < Targets.size(); ++i) {
3323	Results.emplace_back(NextTarget, InstructionListType());
3324	InstructionListType *NewCall = &Results.back().second;
3325
3326	if (MinimizeCodeSize && !LoadElim) {
3327	// Load the call target into FuncAddrReg.
3328	NewCall->push_back(x: CallInst); // Copy CallInst in order to get SMLoc
3329	MCInst &Target = NewCall->back();
3330	Target.clear();
3331	Target.setOpcode(X86::MOV64ri32);
3332	Target.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3333	if (Targets[i].first) {
3334	// Is this OK?
3335	Target.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3336	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3337	} else {
3338	const uint64_t Addr = Targets[i].second;
3339	// Immediate address is out of sign extended 32 bit range.
3340	if (int64_t(Addr) != int64_t(int32_t(Addr)))
3341	return BlocksVectorTy();
3342
3343	Target.addOperand(Op: MCOperand::createImm(Val: Addr));
3344	}
3345
3346	// Compare current call target to a specific address.
3347	NewCall->push_back(x: CallInst);
3348	MCInst &Compare = NewCall->back();
3349	Compare.clear();
3350	if (isBranchOnReg(CallInst))
3351	Compare.setOpcode(X86::CMP64rr);
3352	else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3353	Compare.setOpcode(X86::CMP64ri32);
3354	else
3355	Compare.setOpcode(X86::CMP64rm);
3356
3357	Compare.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3358
3359	// TODO: Would be preferable to only load this value once.
3360	for (unsigned i = 0;
3361	i < Info->get(Opcode: CallInst.getOpcode()).getNumOperands(); ++i)
3362	if (!CallInst.getOperand(i).isInst())
3363	Compare.addOperand(Op: CallInst.getOperand(i));
3364	} else {
3365	// Compare current call target to a specific address.
3366	NewCall->push_back(x: CallInst);
3367	MCInst &Compare = NewCall->back();
3368	Compare.clear();
3369	if (isBranchOnReg(CallInst))
3370	Compare.setOpcode(X86::CMP64ri32);
3371	else
3372	Compare.setOpcode(X86::CMP64mi32);
3373
3374	// Original call address.
3375	for (unsigned i = 0;
3376	i < Info->get(Opcode: CallInst.getOpcode()).getNumOperands(); ++i)
3377	if (!CallInst.getOperand(i).isInst())
3378	Compare.addOperand(Op: CallInst.getOperand(i));
3379
3380	// Target address.
3381	if (Targets[i].first || LoadElim) {
3382	const MCSymbol *Sym =
3383	LoadElim ? VtableSyms[i].first : Targets[i].first;
3384	const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3385	const MCExpr *Expr = MCSymbolRefExpr::create(Symbol: Sym, Ctx&: *Ctx);
3386	if (Addend)
3387	Expr = MCBinaryExpr::createAdd(
3388	LHS: Expr, RHS: MCConstantExpr::create(Value: Addend, Ctx&: *Ctx), Ctx&: *Ctx);
3389	Compare.addOperand(Op: MCOperand::createExpr(Val: Expr));
3390	} else {
3391	const uint64_t Addr = Targets[i].second;
3392	// Immediate address is out of sign extended 32 bit range.
3393	if (int64_t(Addr) != int64_t(int32_t(Addr)))
3394	return BlocksVectorTy();
3395
3396	Compare.addOperand(Op: MCOperand::createImm(Val: Addr));
3397	}
3398	}
3399
3400	// jump to next target compare.
3401	NextTarget =
3402	Ctx->createNamedTempSymbol(); // generate label for the next block
3403	NewCall->push_back(x: CallInst);
3404
3405	if (IsJumpTable) {
3406	MCInst &Je = NewCall->back();
3407
3408	// Jump to next compare if target addresses don't match.
3409	Je.clear();
3410	Je.setOpcode(X86::JCC_1);
3411	if (Targets[i].first)
3412	Je.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3413	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3414	else
3415	Je.addOperand(Op: MCOperand::createImm(Val: Targets[i].second));
3416
3417	Je.addOperand(Op: MCOperand::createImm(Val: X86::COND_E));
3418	assert(!isInvoke(CallInst));
3419	} else {
3420	MCInst &Jne = NewCall->back();
3421
3422	// Jump to next compare if target addresses don't match.
3423	Jne.clear();
3424	Jne.setOpcode(X86::JCC_1);
3425	Jne.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3426	Symbol: NextTarget, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
3427	Jne.addOperand(Op: MCOperand::createImm(Val: X86::COND_NE));
3428
3429	// Call specific target directly.
3430	Results.emplace_back(Ctx->createNamedTempSymbol(),
3431	InstructionListType());
3432	NewCall = &Results.back().second;
3433	NewCall->push_back(x: CallInst);
3434	MCInst &CallOrJmp = NewCall->back();
3435
3436	CallOrJmp.clear();
3437
3438	if (MinimizeCodeSize && !LoadElim) {
3439	CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3440	CallOrJmp.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3441	} else {
3442	CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3443
3444	if (Targets[i].first)
3445	CallOrJmp.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3446	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3447	else
3448	CallOrJmp.addOperand(Op: MCOperand::createImm(Val: Targets[i].second));
3449	}
3450	if (IsTailCall)
3451	setTailCall(CallOrJmp);
3452
3453	if (CallOrJmp.getOpcode() == X86::CALL64r ||
3454	CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3455	if (std::optional<uint32_t> Offset = getOffset(Inst: CallInst))
3456	// Annotated as duplicated call
3457	setOffset(Inst&: CallOrJmp, Offset: *Offset);
3458	}
3459
3460	if (isInvoke(Inst: CallInst) && !isInvoke(Inst: CallOrJmp)) {
3461	// Copy over any EH or GNU args size information from the original
3462	// call.
3463	std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
3464	if (EHInfo)
3465	addEHInfo(Inst&: CallOrJmp, LP: *EHInfo);
3466	int64_t GnuArgsSize = getGnuArgsSize(Inst: CallInst);
3467	if (GnuArgsSize >= 0)
3468	addGnuArgsSize(Inst&: CallOrJmp, GnuArgsSize);
3469	}
3470
3471	if (!IsTailCall) {
3472	// The fallthrough block for the most common target should be
3473	// the merge block.
3474	if (i == 0) {
3475	// Fallthrough to merge block.
3476	MergeBlock = Ctx->createNamedTempSymbol();
3477	} else {
3478	// Insert jump to the merge block if we are not doing a fallthrough.
3479	jumpToMergeBlock(*NewCall);
3480	}
3481	}
3482	}
3483	}
3484
3485	// Cold call block.
3486	Results.emplace_back(NextTarget, InstructionListType());
3487	InstructionListType &NewCall = Results.back().second;
3488	for (const MCInst *Inst : MethodFetchInsns)
3489	if (Inst != &CallInst)
3490	NewCall.push_back(*Inst);
3491	NewCall.push_back(x: CallInst);
3492
3493	// Jump to merge block from cold call block
3494	if (!IsTailCall && !IsJumpTable) {
3495	jumpToMergeBlock(NewCall);
3496
3497	// Record merge block
3498	Results.emplace_back(MergeBlock, InstructionListType());
3499	}
3500
3501	return Results;
3502	}
3503
3504	BlocksVectorTy jumpTablePromotion(
3505	const MCInst &IJmpInst,
3506	const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3507	const std::vector<MCInst *> &TargetFetchInsns,
3508	MCContext *Ctx) const override {
3509	assert(getJumpTable(IJmpInst) != 0);
3510	uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg");
3511	if (IndexReg == 0)
3512	return BlocksVectorTy();
3513
3514	BlocksVectorTy Results;
3515
3516	// Label for the current code block.
3517	MCSymbol *NextTarget = nullptr;
3518
3519	for (unsigned int i = 0; i < Targets.size(); ++i) {
3520	Results.emplace_back(NextTarget, InstructionListType());
3521	InstructionListType *CurBB = &Results.back().second;
3522
3523	// Compare current index to a specific index.
3524	CurBB->emplace_back(MCInst());
3525	MCInst &CompareInst = CurBB->back();
3526	CompareInst.setLoc(IJmpInst.getLoc());
3527	CompareInst.setOpcode(X86::CMP64ri32);
3528	CompareInst.addOperand(Op: MCOperand::createReg(Reg: IndexReg));
3529
3530	const uint64_t CaseIdx = Targets[i].second;
3531	// Immediate address is out of sign extended 32 bit range.
3532	if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3533	return BlocksVectorTy();
3534
3535	CompareInst.addOperand(Op: MCOperand::createImm(Val: CaseIdx));
3536	shortenInstruction(Inst&: CompareInst, STI: *Ctx->getSubtargetInfo());
3537
3538	// jump to next target compare.
3539	NextTarget =
3540	Ctx->createNamedTempSymbol(); // generate label for the next block
3541	CurBB->push_back(x: MCInst());
3542
3543	MCInst &JEInst = CurBB->back();
3544	JEInst.setLoc(IJmpInst.getLoc());
3545
3546	// Jump to target if indices match
3547	JEInst.setOpcode(X86::JCC_1);
3548	JEInst.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3549	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3550	JEInst.addOperand(Op: MCOperand::createImm(Val: X86::COND_E));
3551	}
3552
3553	// Cold call block.
3554	Results.emplace_back(NextTarget, InstructionListType());
3555	InstructionListType &CurBB = Results.back().second;
3556	for (const MCInst *Inst : TargetFetchInsns)
3557	if (Inst != &IJmpInst)
3558	CurBB.push_back(*Inst);
3559
3560	CurBB.push_back(x: IJmpInst);
3561
3562	return Results;
3563	}
3564
3565	private:
3566	void createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3567	MCContext *Ctx) const {
3568	Inst.setOpcode(X86::MOV64rm);
3569	Inst.clear();
3570	Inst.addOperand(Op: MCOperand::createReg(Reg));
3571	Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg
3572	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3573	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3574	Inst.addOperand(Op: MCOperand::createExpr(
3575	Val: MCSymbolRefExpr::create(Symbol: Src, Kind: MCSymbolRefExpr::VK_None,
3576	Ctx&: *Ctx))); // Displacement
3577	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3578	}
3579
3580	void createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3581	MCContext *Ctx) const {
3582	Inst.setOpcode(X86::LEA64r);
3583	Inst.clear();
3584	Inst.addOperand(Op: MCOperand::createReg(Reg));
3585	Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg
3586	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3587	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3588	Inst.addOperand(Op: MCOperand::createExpr(
3589	Val: MCSymbolRefExpr::create(Symbol: Src, Kind: MCSymbolRefExpr::VK_None,
3590	Ctx&: *Ctx))); // Displacement
3591	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3592	}
3593	};
3594
3595	} // namespace
3596
3597	namespace llvm {
3598	namespace bolt {
3599
3600	MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3601	const MCInstrInfo *Info,
3602	const MCRegisterInfo *RegInfo,
3603	const MCSubtargetInfo *STI) {
3604	return new X86MCPlusBuilder(Analysis, Info, RegInfo, STI);
3605	}
3606
3607	} // namespace bolt
3608	} // namespace llvm
3609