1	//===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file provides X86-specific MCPlus builder.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "MCTargetDesc/X86BaseInfo.h"
14	#include "MCTargetDesc/X86EncodingOptimization.h"
15	#include "MCTargetDesc/X86MCTargetDesc.h"
16	#include "X86MCSymbolizer.h"
17	#include "bolt/Core/MCPlus.h"
18	#include "bolt/Core/MCPlusBuilder.h"
19	#include "llvm/BinaryFormat/ELF.h"
20	#include "llvm/MC/MCContext.h"
21	#include "llvm/MC/MCInst.h"
22	#include "llvm/MC/MCInstBuilder.h"
23	#include "llvm/MC/MCInstrInfo.h"
24	#include "llvm/MC/MCRegister.h"
25	#include "llvm/MC/MCRegisterInfo.h"
26	#include "llvm/Support/CommandLine.h"
27	#include "llvm/Support/DataExtractor.h"
28	#include "llvm/Support/Debug.h"
29	#include "llvm/Support/Errc.h"
30	#include "llvm/Support/ErrorHandling.h"
31	#include "llvm/Support/ErrorOr.h"
32	#include <set>
33
34	#define DEBUG_TYPE "mcplus"
35
36	using namespace llvm;
37	using namespace bolt;
38
39	namespace opts {
40
41	extern cl::OptionCategory BoltOptCategory;
42
43	static cl::opt<bool> X86StripRedundantAddressSize(
44	"x86-strip-redundant-address-size",
45	cl::desc("Remove redundant Address-Size override prefix"), cl::init(Val: true),
46	cl::cat(BoltOptCategory));
47
48	} // namespace opts
49
50	namespace {
51
52	bool isMOVSX64rm32(const MCInst &Inst) {
53	return Inst.getOpcode() == X86::MOVSX64rm32;
54	}
55
56	bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; }
57
58	bool isADDri(const MCInst &Inst) {
59	return Inst.getOpcode() == X86::ADD64ri32 ||
60	Inst.getOpcode() == X86::ADD64ri8;
61	}
62
63	// Create instruction to increment contents of target by 1
64	static InstructionListType createIncMemory(const MCSymbol *Target,
65	MCContext *Ctx) {
66	InstructionListType Insts;
67	Insts.emplace_back();
68	Insts.back().setOpcode(X86::LOCK_INC64m);
69	Insts.back().clear();
70	Insts.back().addOperand(Op: MCOperand::createReg(Reg: X86::RIP)); // BaseReg
71	Insts.back().addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
72	Insts.back().addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
73
74	Insts.back().addOperand(
75	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target,
76	Ctx&: *Ctx))); // Displacement
77	Insts.back().addOperand(
78	Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
79	return Insts;
80	}
81
82	#define GET_INSTRINFO_OPERAND_TYPES_ENUM
83	#define GET_INSTRINFO_OPERAND_TYPE
84	#define GET_INSTRINFO_MEM_OPERAND_SIZE
85	#include "X86GenInstrInfo.inc"
86
87	class X86MCPlusBuilder : public MCPlusBuilder {
88	public:
89	using MCPlusBuilder::MCPlusBuilder;
90
91	std::unique_ptr<MCSymbolizer>
92	createTargetSymbolizer(BinaryFunction &Function,
93	bool CreateNewSymbols) const override {
94	return std::make_unique<X86MCSymbolizer>(args&: Function, args&: CreateNewSymbols);
95	}
96
97	bool isBranch(const MCInst &Inst) const override {
98	return Analysis->isBranch(Inst) && !isTailCall(Inst);
99	}
100
101	bool isNoop(const MCInst &Inst) const override {
102	return X86::isNOP(Opcode: Inst.getOpcode());
103	}
104
105	unsigned getCondCode(const MCInst &Inst) const override {
106	unsigned Opcode = Inst.getOpcode();
107	if (X86::isJCC(Opcode))
108	return Inst.getOperand(i: Info->get(Opcode).NumOperands - 1).getImm();
109	return X86::COND_INVALID;
110	}
111
112	unsigned getInvertedCondCode(unsigned CC) const override {
113	switch (CC) {
114	default: return X86::COND_INVALID;
115	case X86::COND_E: return X86::COND_NE;
116	case X86::COND_NE: return X86::COND_E;
117	case X86::COND_L: return X86::COND_GE;
118	case X86::COND_LE: return X86::COND_G;
119	case X86::COND_G: return X86::COND_LE;
120	case X86::COND_GE: return X86::COND_L;
121	case X86::COND_B: return X86::COND_AE;
122	case X86::COND_BE: return X86::COND_A;
123	case X86::COND_A: return X86::COND_BE;
124	case X86::COND_AE: return X86::COND_B;
125	case X86::COND_S: return X86::COND_NS;
126	case X86::COND_NS: return X86::COND_S;
127	case X86::COND_P: return X86::COND_NP;
128	case X86::COND_NP: return X86::COND_P;
129	case X86::COND_O: return X86::COND_NO;
130	case X86::COND_NO: return X86::COND_O;
131	}
132	}
133
134	unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
135	enum DecodedCondCode : uint8_t {
136	DCC_EQUAL = 0x1,
137	DCC_GREATER = 0x2,
138	DCC_LESSER = 0x4,
139	DCC_GREATER_OR_LESSER = 0x6,
140	DCC_UNSIGNED = 0x8,
141	DCC_SIGNED = 0x10,
142	DCC_INVALID = 0x20,
143	};
144
145	auto decodeCondCode = [&](unsigned CC) -> uint8_t {
146	switch (CC) {
147	default: return DCC_INVALID;
148	case X86::COND_E: return DCC_EQUAL;
149	case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
150	case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
151	case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
152	case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
153	case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
154	case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
155	case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
156	case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
157	case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
158	}
159	};
160
161	uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
162
163	if (DCC & DCC_INVALID)
164	return X86::COND_INVALID;
165
166	if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
167	return X86::COND_INVALID;
168
169	switch (DCC) {
170	default: return X86::COND_INVALID;
171	case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
172	case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
173	case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
174	case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
175	case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
176	case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
177	case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
178	case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
179	case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
180	case DCC_EQUAL: return X86::COND_E;
181	case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
182	case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
183	case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
184	case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
185	}
186	}
187
188	bool isValidCondCode(unsigned CC) const override {
189	return (CC != X86::COND_INVALID);
190	}
191
192	bool isBreakpoint(const MCInst &Inst) const override {
193	return Inst.getOpcode() == X86::INT3;
194	}
195
196	bool isPrefix(const MCInst &Inst) const override {
197	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
198	return X86II::isPrefix(TSFlags: Desc.TSFlags);
199	}
200
201	bool isRep(const MCInst &Inst) const override {
202	return Inst.getFlags() == X86::IP_HAS_REPEAT;
203	}
204
205	bool deleteREPPrefix(MCInst &Inst) const override {
206	if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
207	Inst.setFlags(0);
208	return true;
209	}
210	return false;
211	}
212
213	bool isIndirectCall(const MCInst &Inst) const override {
214	return isCall(Inst) &&
215	((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(i: 0).isReg());
216	}
217
218	bool isPop(const MCInst &Inst) const override {
219	return getPopSize(Inst) == 0 ? false : true;
220	}
221
222	bool isTerminateBranch(const MCInst &Inst) const override {
223	return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
224	}
225
226	int getPopSize(const MCInst &Inst) const override {
227	switch (Inst.getOpcode()) {
228	case X86::POP16r:
229	case X86::POP16rmm:
230	case X86::POP16rmr:
231	case X86::POPF16:
232	case X86::POPA16:
233	case X86::POPDS16:
234	case X86::POPES16:
235	case X86::POPFS16:
236	case X86::POPGS16:
237	case X86::POPSS16:
238	return 2;
239	case X86::POP32r:
240	case X86::POP32rmm:
241	case X86::POP32rmr:
242	case X86::POPA32:
243	case X86::POPDS32:
244	case X86::POPES32:
245	case X86::POPF32:
246	case X86::POPFS32:
247	case X86::POPGS32:
248	case X86::POPSS32:
249	return 4;
250	case X86::POP64r:
251	case X86::POP64rmm:
252	case X86::POP64rmr:
253	case X86::POPF64:
254	case X86::POPFS64:
255	case X86::POPGS64:
256	return 8;
257	}
258	return 0;
259	}
260
261	bool isPush(const MCInst &Inst) const override {
262	return getPushSize(Inst) == 0 ? false : true;
263	}
264
265	int getPushSize(const MCInst &Inst) const override {
266	switch (Inst.getOpcode()) {
267	case X86::PUSH16i8:
268	case X86::PUSH16r:
269	case X86::PUSH16rmm:
270	case X86::PUSH16rmr:
271	case X86::PUSHA16:
272	case X86::PUSHCS16:
273	case X86::PUSHDS16:
274	case X86::PUSHES16:
275	case X86::PUSHF16:
276	case X86::PUSHFS16:
277	case X86::PUSHGS16:
278	case X86::PUSHSS16:
279	case X86::PUSH16i:
280	return 2;
281	case X86::PUSH32i8:
282	case X86::PUSH32r:
283	case X86::PUSH32rmm:
284	case X86::PUSH32rmr:
285	case X86::PUSHA32:
286	case X86::PUSHCS32:
287	case X86::PUSHDS32:
288	case X86::PUSHES32:
289	case X86::PUSHF32:
290	case X86::PUSHFS32:
291	case X86::PUSHGS32:
292	case X86::PUSHSS32:
293	case X86::PUSH32i:
294	return 4;
295	case X86::PUSH64i32:
296	case X86::PUSH64i8:
297	case X86::PUSH64r:
298	case X86::PUSH64rmm:
299	case X86::PUSH64rmr:
300	case X86::PUSHF64:
301	case X86::PUSHFS64:
302	case X86::PUSHGS64:
303	return 8;
304	}
305	return 0;
306	}
307
308	bool isSUB(const MCInst &Inst) const override {
309	return X86::isSUB(Opcode: Inst.getOpcode());
310	}
311
312	bool isLEA64r(const MCInst &Inst) const override {
313	return Inst.getOpcode() == X86::LEA64r;
314	}
315
316	bool isLeave(const MCInst &Inst) const override {
317	return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
318	}
319
320	bool isMoveMem2Reg(const MCInst &Inst) const override {
321	switch (Inst.getOpcode()) {
322	case X86::MOV16rm:
323	case X86::MOV32rm:
324	case X86::MOV64rm:
325	return true;
326	}
327	return false;
328	}
329
330	bool isUnsupportedInstruction(const MCInst &Inst) const override {
331	switch (Inst.getOpcode()) {
332	default:
333	return false;
334
335	case X86::LOOP:
336	case X86::LOOPE:
337	case X86::LOOPNE:
338	case X86::JECXZ:
339	case X86::JRCXZ:
340	// These have a short displacement, and therefore (often) break after
341	// basic block relayout.
342	return true;
343	}
344	}
345
346	bool mayLoad(const MCInst &Inst) const override {
347	if (isPop(Inst))
348	return true;
349
350	int MemOpNo = getMemoryOperandNo(Inst);
351	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
352
353	if (MemOpNo == -1)
354	return false;
355
356	return MCII.mayLoad();
357	}
358
359	bool mayStore(const MCInst &Inst) const override {
360	if (isPush(Inst))
361	return true;
362
363	int MemOpNo = getMemoryOperandNo(Inst);
364	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
365
366	if (MemOpNo == -1)
367	return false;
368
369	return MCII.mayStore();
370	}
371
372	bool isCleanRegXOR(const MCInst &Inst) const override {
373	switch (Inst.getOpcode()) {
374	case X86::XOR16rr:
375	case X86::XOR32rr:
376	case X86::XOR64rr:
377	break;
378	default:
379	return false;
380	}
381	return (Inst.getOperand(i: 0).getReg() == Inst.getOperand(i: 2).getReg());
382	}
383
384	bool isPacked(const MCInst &Inst) const override {
385	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
386	return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
387	}
388
389	bool shouldRecordCodeRelocation(uint32_t RelType) const override {
390	switch (RelType) {
391	case ELF::R_X86_64_8:
392	case ELF::R_X86_64_16:
393	case ELF::R_X86_64_32:
394	case ELF::R_X86_64_32S:
395	case ELF::R_X86_64_64:
396	case ELF::R_X86_64_PC8:
397	case ELF::R_X86_64_PC32:
398	case ELF::R_X86_64_PC64:
399	case ELF::R_X86_64_GOTPC64:
400	case ELF::R_X86_64_GOTPCRELX:
401	case ELF::R_X86_64_REX_GOTPCRELX:
402	return true;
403	case ELF::R_X86_64_PLT32:
404	case ELF::R_X86_64_GOTPCREL:
405	case ELF::R_X86_64_TPOFF32:
406	case ELF::R_X86_64_GOTTPOFF:
407	return false;
408	default:
409	llvm_unreachable("Unexpected x86 relocation type in code");
410	}
411	}
412
413	StringRef getTrapFillValue() const override { return StringRef("\314", 1); }
414
415	struct IndJmpMatcherFrag1 : MCInstMatcher {
416	std::unique_ptr<MCInstMatcher> Base;
417	std::unique_ptr<MCInstMatcher> Scale;
418	std::unique_ptr<MCInstMatcher> Index;
419	std::unique_ptr<MCInstMatcher> Offset;
420
421	IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
422	std::unique_ptr<MCInstMatcher> Scale,
423	std::unique_ptr<MCInstMatcher> Index,
424	std::unique_ptr<MCInstMatcher> Offset)
425	: Base(std::move(Base)), Scale(std::move(Scale)),
426	Index(std::move(Index)), Offset(std::move(Offset)) {}
427
428	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
429	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
430	if (!MCInstMatcher::match(MRI, MIA&: MIB, InInstrWindow, OpNum))
431	return false;
432
433	if (CurInst->getOpcode() != X86::JMP64m)
434	return false;
435
436	int MemOpNo = MIB.getMemoryOperandNo(Inst: *CurInst);
437	if (MemOpNo == -1)
438	return false;
439
440	if (!Base->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: MemOpNo + X86::AddrBaseReg))
441	return false;
442	if (!Scale->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow,
443	OpNum: MemOpNo + X86::AddrScaleAmt))
444	return false;
445	if (!Index->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow,
446	OpNum: MemOpNo + X86::AddrIndexReg))
447	return false;
448	if (!Offset->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: MemOpNo + X86::AddrDisp))
449	return false;
450	return true;
451	}
452
453	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
454	MIB.addAnnotation(Inst&: *CurInst, Name: Annotation, Val: true);
455	Base->annotate(MIA&: MIB, Annotation);
456	Scale->annotate(MIA&: MIB, Annotation);
457	Index->annotate(MIA&: MIB, Annotation);
458	Offset->annotate(MIA&: MIB, Annotation);
459	}
460	};
461
462	std::unique_ptr<MCInstMatcher>
463	matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
464	std::unique_ptr<MCInstMatcher> Scale,
465	std::unique_ptr<MCInstMatcher> Index,
466	std::unique_ptr<MCInstMatcher> Offset) const override {
467	return std::unique_ptr<MCInstMatcher>(
468	new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
469	std::move(Index), std::move(Offset)));
470	}
471
472	struct IndJmpMatcherFrag2 : MCInstMatcher {
473	std::unique_ptr<MCInstMatcher> Reg;
474
475	IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
476	: Reg(std::move(Reg)) {}
477
478	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
479	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
480	if (!MCInstMatcher::match(MRI, MIA&: MIB, InInstrWindow, OpNum))
481	return false;
482
483	if (CurInst->getOpcode() != X86::JMP64r)
484	return false;
485
486	return Reg->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 0);
487	}
488
489	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
490	MIB.addAnnotation(Inst&: *CurInst, Name: Annotation, Val: true);
491	Reg->annotate(MIA&: MIB, Annotation);
492	}
493	};
494
495	std::unique_ptr<MCInstMatcher>
496	matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
497	return std::unique_ptr<MCInstMatcher>(
498	new IndJmpMatcherFrag2(std::move(Target)));
499	}
500
501	struct LoadMatcherFrag1 : MCInstMatcher {
502	std::unique_ptr<MCInstMatcher> Base;
503	std::unique_ptr<MCInstMatcher> Scale;
504	std::unique_ptr<MCInstMatcher> Index;
505	std::unique_ptr<MCInstMatcher> Offset;
506
507	LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
508	std::unique_ptr<MCInstMatcher> Scale,
509	std::unique_ptr<MCInstMatcher> Index,
510	std::unique_ptr<MCInstMatcher> Offset)
511	: Base(std::move(Base)), Scale(std::move(Scale)),
512	Index(std::move(Index)), Offset(std::move(Offset)) {}
513
514	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
515	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
516	if (!MCInstMatcher::match(MRI, MIA&: MIB, InInstrWindow, OpNum))
517	return false;
518
519	if (CurInst->getOpcode() != X86::MOV64rm &&
520	CurInst->getOpcode() != X86::MOVSX64rm32)
521	return false;
522
523	int MemOpNo = MIB.getMemoryOperandNo(Inst: *CurInst);
524	if (MemOpNo == -1)
525	return false;
526
527	if (!Base->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: MemOpNo + X86::AddrBaseReg))
528	return false;
529	if (!Scale->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow,
530	OpNum: MemOpNo + X86::AddrScaleAmt))
531	return false;
532	if (!Index->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow,
533	OpNum: MemOpNo + X86::AddrIndexReg))
534	return false;
535	if (!Offset->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: MemOpNo + X86::AddrDisp))
536	return false;
537	return true;
538	}
539
540	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
541	MIB.addAnnotation(Inst&: *CurInst, Name: Annotation, Val: true);
542	Base->annotate(MIA&: MIB, Annotation);
543	Scale->annotate(MIA&: MIB, Annotation);
544	Index->annotate(MIA&: MIB, Annotation);
545	Offset->annotate(MIA&: MIB, Annotation);
546	}
547	};
548
549	std::unique_ptr<MCInstMatcher>
550	matchLoad(std::unique_ptr<MCInstMatcher> Base,
551	std::unique_ptr<MCInstMatcher> Scale,
552	std::unique_ptr<MCInstMatcher> Index,
553	std::unique_ptr<MCInstMatcher> Offset) const override {
554	return std::unique_ptr<MCInstMatcher>(
555	new LoadMatcherFrag1(std::move(Base), std::move(Scale),
556	std::move(Index), std::move(Offset)));
557	}
558
559	struct AddMatcher : MCInstMatcher {
560	std::unique_ptr<MCInstMatcher> A;
561	std::unique_ptr<MCInstMatcher> B;
562
563	AddMatcher(std::unique_ptr<MCInstMatcher> A,
564	std::unique_ptr<MCInstMatcher> B)
565	: A(std::move(A)), B(std::move(B)) {}
566
567	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
568	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
569	if (!MCInstMatcher::match(MRI, MIA&: MIB, InInstrWindow, OpNum))
570	return false;
571
572	if (CurInst->getOpcode() == X86::ADD64rr ||
573	CurInst->getOpcode() == X86::ADD64rr_DB ||
574	CurInst->getOpcode() == X86::ADD64rr_REV) {
575	if (!A->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 1)) {
576	if (!B->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 1))
577	return false;
578	return A->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 2);
579	}
580
581	if (B->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 2))
582	return true;
583
584	if (!B->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 1))
585	return false;
586	return A->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 2);
587	}
588
589	return false;
590	}
591
592	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
593	MIB.addAnnotation(Inst&: *CurInst, Name: Annotation, Val: true);
594	A->annotate(MIA&: MIB, Annotation);
595	B->annotate(MIA&: MIB, Annotation);
596	}
597	};
598
599	std::unique_ptr<MCInstMatcher>
600	matchAdd(std::unique_ptr<MCInstMatcher> A,
601	std::unique_ptr<MCInstMatcher> B) const override {
602	return std::unique_ptr<MCInstMatcher>(
603	new AddMatcher(std::move(A), std::move(B)));
604	}
605
606	struct LEAMatcher : MCInstMatcher {
607	std::unique_ptr<MCInstMatcher> Target;
608
609	LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
610	: Target(std::move(Target)) {}
611
612	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
613	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
614	if (!MCInstMatcher::match(MRI, MIA&: MIB, InInstrWindow, OpNum))
615	return false;
616
617	if (CurInst->getOpcode() != X86::LEA64r)
618	return false;
619
620	if (CurInst->getOperand(i: 1 + X86::AddrScaleAmt).getImm() != 1 ||
621	CurInst->getOperand(i: 1 + X86::AddrIndexReg).getReg() !=
622	X86::NoRegister ||
623	(CurInst->getOperand(i: 1 + X86::AddrBaseReg).getReg() !=
624	X86::NoRegister &&
625	CurInst->getOperand(i: 1 + X86::AddrBaseReg).getReg() != X86::RIP))
626	return false;
627
628	return Target->match(MRI, MIA&: MIB, InInstrWindow: this->InstrWindow, OpNum: 1 + X86::AddrDisp);
629	}
630
631	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
632	MIB.addAnnotation(Inst&: *CurInst, Name: Annotation, Val: true);
633	Target->annotate(MIA&: MIB, Annotation);
634	}
635	};
636
637	std::unique_ptr<MCInstMatcher>
638	matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
639	return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
640	}
641
642	bool hasPCRelOperand(const MCInst &Inst) const override {
643	for (const MCOperand &Operand : Inst)
644	if (Operand.isReg() && Operand.getReg() == X86::RIP)
645	return true;
646	return false;
647	}
648
649	int getMemoryOperandNo(const MCInst &Inst) const override {
650	unsigned Opcode = Inst.getOpcode();
651	const MCInstrDesc &Desc = Info->get(Opcode);
652	int MemOpNo = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
653	if (MemOpNo >= 0)
654	MemOpNo += X86II::getOperandBias(Desc);
655	return MemOpNo;
656	}
657
658	bool hasEVEXEncoding(const MCInst &Inst) const override {
659	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
660	return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
661	}
662
663	std::optional<X86MemOperand>
664	evaluateX86MemoryOperand(const MCInst &Inst) const override {
665	int MemOpNo = getMemoryOperandNo(Inst);
666	if (MemOpNo < 0)
667	return std::nullopt;
668	unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
669
670	if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
671	return std::nullopt;
672
673	const MCOperand &Base = Inst.getOperand(i: MemOpOffset + X86::AddrBaseReg);
674	const MCOperand &Scale = Inst.getOperand(i: MemOpOffset + X86::AddrScaleAmt);
675	const MCOperand &Index = Inst.getOperand(i: MemOpOffset + X86::AddrIndexReg);
676	const MCOperand &Disp = Inst.getOperand(i: MemOpOffset + X86::AddrDisp);
677	const MCOperand &Segment =
678	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg);
679
680	// Make sure it is a well-formed memory operand.
681	if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
682	(!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
683	return std::nullopt;
684
685	X86MemOperand MO;
686	MO.BaseRegNum = Base.getReg();
687	MO.ScaleImm = Scale.getImm();
688	MO.IndexRegNum = Index.getReg();
689	MO.DispImm = Disp.isImm() ? Disp.getImm() : 0;
690	MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr;
691	MO.SegRegNum = Segment.getReg();
692	return MO;
693	}
694
695	bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
696	uint64_t Address,
697	uint64_t Size) const override {
698	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
699	if (!MO)
700	return false;
701
702	// Make sure it's a well-formed addressing we can statically evaluate.
703	if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) ||
704	MO->IndexRegNum != X86::NoRegister ||
705	MO->SegRegNum != X86::NoRegister || MO->DispExpr)
706	return false;
707
708	Target = MO->DispImm;
709	if (MO->BaseRegNum == X86::RIP) {
710	assert(Size != 0 && "instruction size required in order to statically "
711	"evaluate RIP-relative address");
712	Target += Address + Size;
713	}
714	return true;
715	}
716
717	MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
718	int MemOpNo = getMemoryOperandNo(Inst);
719	if (MemOpNo < 0)
720	return Inst.end();
721	return Inst.begin() + (MemOpNo + X86::AddrDisp);
722	}
723
724	bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
725	MCOperand *OI = getMemOperandDisp(Inst);
726	if (OI == Inst.end())
727	return false;
728	*OI = Operand;
729	return true;
730	}
731
732	/// Get the registers used as function parameters.
733	/// This function is specific to the x86_64 abi on Linux.
734	BitVector getRegsUsedAsParams() const override {
735	BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
736	Regs |= getAliases(Reg: X86::RSI);
737	Regs |= getAliases(Reg: X86::RDI);
738	Regs |= getAliases(Reg: X86::RDX);
739	Regs |= getAliases(Reg: X86::RCX);
740	Regs |= getAliases(Reg: X86::R8);
741	Regs |= getAliases(Reg: X86::R9);
742	return Regs;
743	}
744
745	void getCalleeSavedRegs(BitVector &Regs) const override {
746	Regs |= getAliases(Reg: X86::RBX);
747	Regs |= getAliases(Reg: X86::RBP);
748	Regs |= getAliases(Reg: X86::R12);
749	Regs |= getAliases(Reg: X86::R13);
750	Regs |= getAliases(Reg: X86::R14);
751	Regs |= getAliases(Reg: X86::R15);
752	}
753
754	void getDefaultDefIn(BitVector &Regs) const override {
755	assert(Regs.size() >= RegInfo->getNumRegs() &&
756	"The size of BitVector is less than RegInfo->getNumRegs().");
757	Regs.set(X86::RAX);
758	Regs.set(X86::RCX);
759	Regs.set(X86::RDX);
760	Regs.set(X86::RSI);
761	Regs.set(X86::RDI);
762	Regs.set(X86::R8);
763	Regs.set(X86::R9);
764	Regs.set(X86::XMM0);
765	Regs.set(X86::XMM1);
766	Regs.set(X86::XMM2);
767	Regs.set(X86::XMM3);
768	Regs.set(X86::XMM4);
769	Regs.set(X86::XMM5);
770	Regs.set(X86::XMM6);
771	Regs.set(X86::XMM7);
772	}
773
774	void getDefaultLiveOut(BitVector &Regs) const override {
775	assert(Regs.size() >= RegInfo->getNumRegs() &&
776	"The size of BitVector is less than RegInfo->getNumRegs().");
777	Regs |= getAliases(Reg: X86::RAX);
778	Regs |= getAliases(Reg: X86::RDX);
779	Regs |= getAliases(Reg: X86::RCX);
780	Regs |= getAliases(Reg: X86::XMM0);
781	Regs |= getAliases(Reg: X86::XMM1);
782	}
783
784	void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
785	if (IncludeAlias) {
786	Regs |= getAliases(Reg: X86::RAX);
787	Regs |= getAliases(Reg: X86::RBX);
788	Regs |= getAliases(Reg: X86::RBP);
789	Regs |= getAliases(Reg: X86::RSI);
790	Regs |= getAliases(Reg: X86::RDI);
791	Regs |= getAliases(Reg: X86::RDX);
792	Regs |= getAliases(Reg: X86::RCX);
793	Regs |= getAliases(Reg: X86::R8);
794	Regs |= getAliases(Reg: X86::R9);
795	Regs |= getAliases(Reg: X86::R10);
796	Regs |= getAliases(Reg: X86::R11);
797	Regs |= getAliases(Reg: X86::R12);
798	Regs |= getAliases(Reg: X86::R13);
799	Regs |= getAliases(Reg: X86::R14);
800	Regs |= getAliases(Reg: X86::R15);
801	return;
802	}
803	Regs.set(X86::RAX);
804	Regs.set(X86::RBX);
805	Regs.set(X86::RBP);
806	Regs.set(X86::RSI);
807	Regs.set(X86::RDI);
808	Regs.set(X86::RDX);
809	Regs.set(X86::RCX);
810	Regs.set(X86::R8);
811	Regs.set(X86::R9);
812	Regs.set(X86::R10);
813	Regs.set(X86::R11);
814	Regs.set(X86::R12);
815	Regs.set(X86::R13);
816	Regs.set(X86::R14);
817	Regs.set(X86::R15);
818	}
819
820	void getClassicGPRegs(BitVector &Regs) const override {
821	Regs |= getAliases(Reg: X86::RAX);
822	Regs |= getAliases(Reg: X86::RBX);
823	Regs |= getAliases(Reg: X86::RBP);
824	Regs |= getAliases(Reg: X86::RSI);
825	Regs |= getAliases(Reg: X86::RDI);
826	Regs |= getAliases(Reg: X86::RDX);
827	Regs |= getAliases(Reg: X86::RCX);
828	}
829
830	void getRepRegs(BitVector &Regs) const override {
831	Regs |= getAliases(Reg: X86::RCX);
832	}
833
834	MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
835	Reg = getX86SubSuperRegister(Reg, Size: Size * 8);
836	assert((Reg != X86::NoRegister) && "Invalid register");
837	return Reg;
838	}
839
840	bool isUpper8BitReg(MCPhysReg Reg) const override {
841	switch (Reg) {
842	case X86::AH:
843	case X86::BH:
844	case X86::CH:
845	case X86::DH:
846	return true;
847	default:
848	return false;
849	}
850	}
851
852	bool cannotUseREX(const MCInst &Inst) const override {
853	switch (Inst.getOpcode()) {
854	case X86::MOV8mr_NOREX:
855	case X86::MOV8rm_NOREX:
856	case X86::MOV8rr_NOREX:
857	case X86::MOVSX32rm8_NOREX:
858	case X86::MOVSX32rr8_NOREX:
859	case X86::MOVZX32rm8_NOREX:
860	case X86::MOVZX32rr8_NOREX:
861	case X86::MOV8mr:
862	case X86::MOV8rm:
863	case X86::MOV8rr:
864	case X86::MOVSX32rm8:
865	case X86::MOVSX32rr8:
866	case X86::MOVZX32rm8:
867	case X86::MOVZX32rr8:
868	case X86::TEST8ri:
869	for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) {
870	if (!Operand.isReg())
871	continue;
872	if (isUpper8BitReg(Reg: Operand.getReg()))
873	return true;
874	}
875	[[fallthrough]];
876	default:
877	return false;
878	}
879	}
880
881	static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) {
882	using namespace llvm::X86;
883	int OpType = getOperandType(Opcode: Inst.getOpcode(), OpIdx: MemOpNo);
884	return getMemOperandSize(OpType) / 8;
885	}
886
887	/// Classifying a stack access as *not* "SIMPLE" here means we don't know how
888	/// to change this instruction memory access. It will disable any changes to
889	/// the stack layout, so we can't do the most aggressive form of shrink
890	/// wrapping. We must do so in a way that keeps the original stack layout.
891	/// Otherwise you need to adjust the offset of all instructions accessing the
892	/// stack: we can't do that anymore because there is one instruction that is
893	/// not simple. There are other implications as well. We have heuristics to
894	/// detect when a register is callee-saved and thus eligible for shrink
895	/// wrapping. If you are restoring a register using a non-simple stack access,
896	/// then it is classified as NOT callee-saved, and it disables shrink wrapping
897	/// for *that* register (but not for others).
898	///
899	/// Classifying a stack access as "size 0" or detecting an indexed memory
900	/// access (to address a vector, for example) here means we know there is a
901	/// stack access, but we can't quite understand how wide is the access in
902	/// bytes. This is very serious because we can't understand how memory
903	/// accesses alias with each other for this function. This will essentially
904	/// disable not only shrink wrapping but all frame analysis, it will fail it
905	/// as "we don't understand this function and we give up on it".
906	bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
907	bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
908	uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
909	bool &IsSimple, bool &IsIndexed) const override {
910	// Detect simple push/pop cases first
911	if (int Sz = getPushSize(Inst)) {
912	IsLoad = false;
913	IsStore = true;
914	IsStoreFromReg = true;
915	StackPtrReg = X86::RSP;
916	StackOffset = -Sz;
917	Size = Sz;
918	IsSimple = true;
919	if (Inst.getOperand(i: 0).isImm())
920	SrcImm = Inst.getOperand(i: 0).getImm();
921	else if (Inst.getOperand(i: 0).isReg())
922	Reg = Inst.getOperand(i: 0).getReg();
923	else
924	IsSimple = false;
925
926	return true;
927	}
928	if (int Sz = getPopSize(Inst)) {
929	IsLoad = true;
930	IsStore = false;
931	if (Inst.getNumOperands() == 0 || !Inst.getOperand(i: 0).isReg()) {
932	IsSimple = false;
933	} else {
934	Reg = Inst.getOperand(i: 0).getReg();
935	IsSimple = true;
936	}
937	StackPtrReg = X86::RSP;
938	StackOffset = 0;
939	Size = Sz;
940	return true;
941	}
942
943	struct InstInfo {
944	// Size in bytes that Inst loads from memory.
945	uint8_t DataSize;
946	bool IsLoad;
947	bool IsStore;
948	bool StoreFromReg;
949	bool Simple;
950	};
951
952	InstInfo I;
953	int MemOpNo = getMemoryOperandNo(Inst);
954	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
955	// If it is not dealing with a memory operand, we discard it
956	if (MemOpNo == -1 || MCII.isCall())
957	return false;
958
959	switch (Inst.getOpcode()) {
960	default: {
961	bool IsLoad = MCII.mayLoad();
962	bool IsStore = MCII.mayStore();
963	// Is it LEA? (deals with memory but is not loading nor storing)
964	if (!IsLoad && !IsStore) {
965	I = {.DataSize: 0, .IsLoad: IsLoad, .IsStore: IsStore, .StoreFromReg: false, .Simple: false};
966	break;
967	}
968	uint8_t Sz = getMemDataSize(Inst, MemOpNo);
969	I = {.DataSize: Sz, .IsLoad: IsLoad, .IsStore: IsStore, .StoreFromReg: false, .Simple: false};
970	break;
971	}
972	// Report simple stack accesses
973	case X86::MOV8rm: I = {.DataSize: 1, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
974	case X86::MOV16rm: I = {.DataSize: 2, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
975	case X86::MOV32rm: I = {.DataSize: 4, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
976	case X86::MOV64rm: I = {.DataSize: 8, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
977	case X86::MOV8mr: I = {.DataSize: 1, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
978	case X86::MOV16mr: I = {.DataSize: 2, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
979	case X86::MOV32mr: I = {.DataSize: 4, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
980	case X86::MOV64mr: I = {.DataSize: 8, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
981	case X86::MOV8mi: I = {.DataSize: 1, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
982	case X86::MOV16mi: I = {.DataSize: 2, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
983	case X86::MOV32mi: I = {.DataSize: 4, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
984	} // end switch (Inst.getOpcode())
985
986	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
987	if (!MO) {
988	LLVM_DEBUG(dbgs() << "Evaluate failed on ");
989	LLVM_DEBUG(Inst.dump());
990	return false;
991	}
992
993	// Make sure it's a stack access
994	if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP)
995	return false;
996
997	IsLoad = I.IsLoad;
998	IsStore = I.IsStore;
999	IsStoreFromReg = I.StoreFromReg;
1000	Size = I.DataSize;
1001	IsSimple = I.Simple;
1002	StackPtrReg = MO->BaseRegNum;
1003	StackOffset = MO->DispImm;
1004	IsIndexed =
1005	MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister;
1006
1007	if (!I.Simple)
1008	return true;
1009
1010	// Retrieve related register in simple MOV from/to stack operations.
1011	unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1012	if (I.IsLoad) {
1013	MCOperand RegOpnd = Inst.getOperand(i: 0);
1014	assert(RegOpnd.isReg() && "unexpected destination operand");
1015	Reg = RegOpnd.getReg();
1016	} else if (I.IsStore) {
1017	MCOperand SrcOpnd =
1018	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg + 1);
1019	if (I.StoreFromReg) {
1020	assert(SrcOpnd.isReg() && "unexpected source operand");
1021	Reg = SrcOpnd.getReg();
1022	} else {
1023	assert(SrcOpnd.isImm() && "unexpected source operand");
1024	SrcImm = SrcOpnd.getImm();
1025	}
1026	}
1027
1028	return true;
1029	}
1030
1031	void changeToPushOrPop(MCInst &Inst) const override {
1032	assert(!isPush(Inst) && !isPop(Inst));
1033
1034	struct InstInfo {
1035	// Size in bytes that Inst loads from memory.
1036	uint8_t DataSize;
1037	bool IsLoad;
1038	bool StoreFromReg;
1039	};
1040
1041	InstInfo I;
1042	switch (Inst.getOpcode()) {
1043	default: {
1044	llvm_unreachable("Unhandled opcode");
1045	return;
1046	}
1047	case X86::MOV16rm: I = {.DataSize: 2, .IsLoad: true, .StoreFromReg: false}; break;
1048	case X86::MOV32rm: I = {.DataSize: 4, .IsLoad: true, .StoreFromReg: false}; break;
1049	case X86::MOV64rm: I = {.DataSize: 8, .IsLoad: true, .StoreFromReg: false}; break;
1050	case X86::MOV16mr: I = {.DataSize: 2, .IsLoad: false, .StoreFromReg: true}; break;
1051	case X86::MOV32mr: I = {.DataSize: 4, .IsLoad: false, .StoreFromReg: true}; break;
1052	case X86::MOV64mr: I = {.DataSize: 8, .IsLoad: false, .StoreFromReg: true}; break;
1053	case X86::MOV16mi: I = {.DataSize: 2, .IsLoad: false, .StoreFromReg: false}; break;
1054	case X86::MOV32mi: I = {.DataSize: 4, .IsLoad: false, .StoreFromReg: false}; break;
1055	} // end switch (Inst.getOpcode())
1056
1057	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1058	if (!MO) {
1059	llvm_unreachable("Evaluate failed");
1060	return;
1061	}
1062	// Make sure it's a stack access
1063	if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) {
1064	llvm_unreachable("Not a stack access");
1065	return;
1066	}
1067
1068	unsigned MemOpOffset = getMemoryOperandNo(Inst);
1069	unsigned NewOpcode = 0;
1070	if (I.IsLoad) {
1071	switch (I.DataSize) {
1072	case 2: NewOpcode = X86::POP16r; break;
1073	case 4: NewOpcode = X86::POP32r; break;
1074	case 8: NewOpcode = X86::POP64r; break;
1075	default:
1076	llvm_unreachable("Unexpected size");
1077	}
1078	unsigned RegOpndNum = Inst.getOperand(i: 0).getReg();
1079	Inst.clear();
1080	Inst.setOpcode(NewOpcode);
1081	Inst.addOperand(Op: MCOperand::createReg(Reg: RegOpndNum));
1082	} else {
1083	MCOperand SrcOpnd =
1084	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg + 1);
1085	if (I.StoreFromReg) {
1086	switch (I.DataSize) {
1087	case 2: NewOpcode = X86::PUSH16r; break;
1088	case 4: NewOpcode = X86::PUSH32r; break;
1089	case 8: NewOpcode = X86::PUSH64r; break;
1090	default:
1091	llvm_unreachable("Unexpected size");
1092	}
1093	assert(SrcOpnd.isReg() && "Unexpected source operand");
1094	unsigned RegOpndNum = SrcOpnd.getReg();
1095	Inst.clear();
1096	Inst.setOpcode(NewOpcode);
1097	Inst.addOperand(Op: MCOperand::createReg(Reg: RegOpndNum));
1098	} else {
1099	switch (I.DataSize) {
1100	case 2: NewOpcode = X86::PUSH16i8; break;
1101	case 4: NewOpcode = X86::PUSH32i8; break;
1102	case 8: NewOpcode = X86::PUSH64i32; break;
1103	default:
1104	llvm_unreachable("Unexpected size");
1105	}
1106	assert(SrcOpnd.isImm() && "Unexpected source operand");
1107	int64_t SrcImm = SrcOpnd.getImm();
1108	Inst.clear();
1109	Inst.setOpcode(NewOpcode);
1110	Inst.addOperand(Op: MCOperand::createImm(Val: SrcImm));
1111	}
1112	}
1113	}
1114
1115	bool isStackAdjustment(const MCInst &Inst) const override {
1116	switch (Inst.getOpcode()) {
1117	default:
1118	return false;
1119	case X86::SUB64ri32:
1120	case X86::SUB64ri8:
1121	case X86::ADD64ri32:
1122	case X86::ADD64ri8:
1123	case X86::LEA64r:
1124	break;
1125	}
1126
1127	return any_of(Range: defOperands(Inst), P: [](const MCOperand &Op) {
1128	return Op.isReg() && Op.getReg() == X86::RSP;
1129	});
1130	}
1131
1132	bool
1133	evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output,
1134	std::pair<MCPhysReg, int64_t> Input1,
1135	std::pair<MCPhysReg, int64_t> Input2) const override {
1136
1137	auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1138	if (Reg == Input1.first)
1139	return Input1.second;
1140	if (Reg == Input2.first)
1141	return Input2.second;
1142	return make_error_code(E: errc::result_out_of_range);
1143	};
1144
1145	switch (Inst.getOpcode()) {
1146	default:
1147	return false;
1148
1149	case X86::SUB64ri32:
1150	case X86::SUB64ri8:
1151	if (!Inst.getOperand(i: 2).isImm())
1152	return false;
1153	if (ErrorOr<int64_t> InputVal =
1154	getOperandVal(Inst.getOperand(i: 1).getReg()))
1155	Output = *InputVal - Inst.getOperand(i: 2).getImm();
1156	else
1157	return false;
1158	break;
1159	case X86::ADD64ri32:
1160	case X86::ADD64ri8:
1161	if (!Inst.getOperand(i: 2).isImm())
1162	return false;
1163	if (ErrorOr<int64_t> InputVal =
1164	getOperandVal(Inst.getOperand(i: 1).getReg()))
1165	Output = *InputVal + Inst.getOperand(i: 2).getImm();
1166	else
1167	return false;
1168	break;
1169	case X86::ADD64i32:
1170	if (!Inst.getOperand(i: 0).isImm())
1171	return false;
1172	if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1173	Output = *InputVal + Inst.getOperand(i: 0).getImm();
1174	else
1175	return false;
1176	break;
1177
1178	case X86::LEA64r: {
1179	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1180	if (!MO)
1181	return false;
1182
1183	if (MO->BaseRegNum == X86::NoRegister ||
1184	MO->IndexRegNum != X86::NoRegister ||
1185	MO->SegRegNum != X86::NoRegister || MO->DispExpr)
1186	return false;
1187
1188	if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum))
1189	Output = *InputVal + MO->DispImm;
1190	else
1191	return false;
1192
1193	break;
1194	}
1195	}
1196	return true;
1197	}
1198
1199	bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1200	MCPhysReg &To) const override {
1201	switch (Inst.getOpcode()) {
1202	default:
1203	return false;
1204	case X86::LEAVE:
1205	case X86::LEAVE64:
1206	To = getStackPointer();
1207	From = getFramePointer();
1208	return true;
1209	case X86::MOV64rr:
1210	To = Inst.getOperand(i: 0).getReg();
1211	From = Inst.getOperand(i: 1).getReg();
1212	return true;
1213	}
1214	}
1215
1216	MCPhysReg getStackPointer() const override { return X86::RSP; }
1217	MCPhysReg getFramePointer() const override { return X86::RBP; }
1218	MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1219
1220	bool escapesVariable(const MCInst &Inst,
1221	bool HasFramePointer) const override {
1222	int MemOpNo = getMemoryOperandNo(Inst);
1223	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1224	const unsigned NumDefs = MCII.getNumDefs();
1225	static BitVector SPBPAliases(BitVector(getAliases(Reg: X86::RSP)) |=
1226	getAliases(Reg: X86::RBP));
1227	static BitVector SPAliases(getAliases(Reg: X86::RSP));
1228
1229	// FIXME: PUSH can be technically a leak, but let's ignore this for now
1230	// because a lot of harmless prologue code will spill SP to the stack.
1231	// Unless push is clearly pushing an object address to the stack as
1232	// demonstrated by having a MemOp.
1233	bool IsPush = isPush(Inst);
1234	if (IsPush && MemOpNo == -1)
1235	return false;
1236
1237	// We use this to detect LEA (has memop but does not access mem)
1238	bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1239	bool DoesLeak = false;
1240	for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1241	// Ignore if SP/BP is used to dereference memory -- that's fine
1242	if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1243	I <= MemOpNo + 5)
1244	continue;
1245	// Ignore if someone is writing to SP/BP
1246	if (I < static_cast<int>(NumDefs))
1247	continue;
1248
1249	const MCOperand &Operand = Inst.getOperand(i: I);
1250	if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1251	DoesLeak = true;
1252	break;
1253	}
1254	if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1255	DoesLeak = true;
1256	break;
1257	}
1258	}
1259
1260	// If potential leak, check if it is not just writing to itself/sp/bp
1261	if (DoesLeak) {
1262	DoesLeak = !any_of(Range: defOperands(Inst), P: [&](const MCOperand &Operand) {
1263	assert(Operand.isReg());
1264	MCPhysReg Reg = Operand.getReg();
1265	return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg];
1266	});
1267	}
1268	return DoesLeak;
1269	}
1270
1271	bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1272	unsigned ImmOpNo = -1U;
1273	int MemOpNo = getMemoryOperandNo(Inst);
1274	if (MemOpNo != -1)
1275	ImmOpNo = MemOpNo + X86::AddrDisp;
1276	else
1277	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1278	++Index)
1279	if (Inst.getOperand(i: Index).isImm())
1280	ImmOpNo = Index;
1281	if (ImmOpNo == -1U)
1282	return false;
1283
1284	MCOperand &Operand = Inst.getOperand(i: ImmOpNo);
1285	Amt += Operand.getImm();
1286	Operand.setImm(Amt);
1287	// Check for the need for relaxation
1288	if (int64_t(Amt) == int64_t(int8_t(Amt)))
1289	return true;
1290
1291	// Relax instruction
1292	switch (Inst.getOpcode()) {
1293	case X86::SUB64ri8:
1294	Inst.setOpcode(X86::SUB64ri32);
1295	break;
1296	case X86::ADD64ri8:
1297	Inst.setOpcode(X86::ADD64ri32);
1298	break;
1299	default:
1300	// No need for relaxation
1301	break;
1302	}
1303	return true;
1304	}
1305
1306	/// TODO: this implementation currently works for the most common opcodes that
1307	/// load from memory. It can be extended to work with memory store opcodes as
1308	/// well as more memory load opcodes.
1309	bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1310	uint64_t Offset) const override {
1311	enum CheckSignExt : uint8_t {
1312	NOCHECK = 0,
1313	CHECK8,
1314	CHECK32,
1315	};
1316
1317	using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1318	struct InstInfo {
1319	// Size in bytes that Inst loads from memory.
1320	uint8_t DataSize;
1321
1322	// True when the target operand has to be duplicated because the opcode
1323	// expects a LHS operand.
1324	bool HasLHS;
1325
1326	// List of checks and corresponding opcodes to be used. We try to use the
1327	// smallest possible immediate value when various sizes are available,
1328	// hence we may need to check whether a larger constant fits in a smaller
1329	// immediate.
1330	CheckList Checks;
1331	};
1332
1333	InstInfo I;
1334
1335	switch (Inst.getOpcode()) {
1336	default: {
1337	switch (getPopSize(Inst)) {
1338	case 2: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV16ri}}}; break;
1339	case 4: I = {.DataSize: 4, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
1340	case 8: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK32, X86::MOV64ri32},
1341	{NOCHECK, X86::MOV64rm}}}; break;
1342	default: return false;
1343	}
1344	break;
1345	}
1346
1347	// MOV
1348	case X86::MOV8rm: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV8ri}}}; break;
1349	case X86::MOV16rm: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV16ri}}}; break;
1350	case X86::MOV32rm: I = {.DataSize: 4, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
1351	case X86::MOV64rm: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK32, X86::MOV64ri32},
1352	{NOCHECK, X86::MOV64rm}}}; break;
1353
1354	// MOVZX
1355	case X86::MOVZX16rm8: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV16ri}}}; break;
1356	case X86::MOVZX32rm8: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
1357	case X86::MOVZX32rm16: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
1358
1359	// CMP
1360	case X86::CMP8rm: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::CMP8ri}}}; break;
1361	case X86::CMP16rm: I = {.DataSize: 2, .HasLHS: false, .Checks: {{CHECK8, X86::CMP16ri8},
1362	{NOCHECK, X86::CMP16ri}}}; break;
1363	case X86::CMP32rm: I = {.DataSize: 4, .HasLHS: false, .Checks: {{CHECK8, X86::CMP32ri8},
1364	{NOCHECK, X86::CMP32ri}}}; break;
1365	case X86::CMP64rm: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK8, X86::CMP64ri8},
1366	{CHECK32, X86::CMP64ri32},
1367	{NOCHECK, X86::CMP64rm}}}; break;
1368
1369	// TEST
1370	case X86::TEST8mr: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::TEST8ri}}}; break;
1371	case X86::TEST16mr: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::TEST16ri}}}; break;
1372	case X86::TEST32mr: I = {.DataSize: 4, .HasLHS: false, .Checks: {{NOCHECK, X86::TEST32ri}}}; break;
1373	case X86::TEST64mr: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK32, X86::TEST64ri32},
1374	{NOCHECK, X86::TEST64mr}}}; break;
1375
1376	// ADD
1377	case X86::ADD8rm: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::ADD8ri}}}; break;
1378	case X86::ADD16rm: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::ADD16ri8},
1379	{NOCHECK, X86::ADD16ri}}}; break;
1380	case X86::ADD32rm: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::ADD32ri8},
1381	{NOCHECK, X86::ADD32ri}}}; break;
1382	case X86::ADD64rm: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::ADD64ri8},
1383	{CHECK32, X86::ADD64ri32},
1384	{NOCHECK, X86::ADD64rm}}}; break;
1385
1386	// SUB
1387	case X86::SUB8rm: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::SUB8ri}}}; break;
1388	case X86::SUB16rm: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::SUB16ri8},
1389	{NOCHECK, X86::SUB16ri}}}; break;
1390	case X86::SUB32rm: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::SUB32ri8},
1391	{NOCHECK, X86::SUB32ri}}}; break;
1392	case X86::SUB64rm: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::SUB64ri8},
1393	{CHECK32, X86::SUB64ri32},
1394	{NOCHECK, X86::SUB64rm}}}; break;
1395
1396	// AND
1397	case X86::AND8rm: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::AND8ri}}}; break;
1398	case X86::AND16rm: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::AND16ri8},
1399	{NOCHECK, X86::AND16ri}}}; break;
1400	case X86::AND32rm: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::AND32ri8},
1401	{NOCHECK, X86::AND32ri}}}; break;
1402	case X86::AND64rm: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::AND64ri8},
1403	{CHECK32, X86::AND64ri32},
1404	{NOCHECK, X86::AND64rm}}}; break;
1405
1406	// OR
1407	case X86::OR8rm: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::OR8ri}}}; break;
1408	case X86::OR16rm: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::OR16ri8},
1409	{NOCHECK, X86::OR16ri}}}; break;
1410	case X86::OR32rm: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::OR32ri8},
1411	{NOCHECK, X86::OR32ri}}}; break;
1412	case X86::OR64rm: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::OR64ri8},
1413	{CHECK32, X86::OR64ri32},
1414	{NOCHECK, X86::OR64rm}}}; break;
1415
1416	// XOR
1417	case X86::XOR8rm: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::XOR8ri}}}; break;
1418	case X86::XOR16rm: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::XOR16ri8},
1419	{NOCHECK, X86::XOR16ri}}}; break;
1420	case X86::XOR32rm: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::XOR32ri8},
1421	{NOCHECK, X86::XOR32ri}}}; break;
1422	case X86::XOR64rm: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::XOR64ri8},
1423	{CHECK32, X86::XOR64ri32},
1424	{NOCHECK, X86::XOR64rm}}}; break;
1425	}
1426
1427	// Compute the immediate value.
1428	assert(Offset + I.DataSize <= ConstantData.size() &&
1429	"invalid offset for given constant data");
1430	int64_t ImmVal =
1431	DataExtractor(ConstantData, true, 8).getSigned(offset_ptr: &Offset, size: I.DataSize);
1432
1433	// Compute the new opcode.
1434	unsigned NewOpcode = 0;
1435	for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1436	NewOpcode = Check.second;
1437	if (Check.first == NOCHECK)
1438	break;
1439	if (Check.first == CHECK8 && isInt<8>(x: ImmVal))
1440	break;
1441	if (Check.first == CHECK32 && isInt<32>(x: ImmVal))
1442	break;
1443	}
1444	if (NewOpcode == Inst.getOpcode())
1445	return false;
1446
1447	// Modify the instruction.
1448	MCOperand ImmOp = MCOperand::createImm(Val: ImmVal);
1449	uint32_t TargetOpNum = 0;
1450	// Test instruction does not follow the regular pattern of putting the
1451	// memory reference of a load (5 MCOperands) last in the list of operands.
1452	// Since it is not modifying the register operand, it is not treated as
1453	// a destination operand and it is not the first operand as it is in the
1454	// other instructions we treat here.
1455	if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1456	NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1457	TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1458
1459	MCOperand TargetOp = Inst.getOperand(i: TargetOpNum);
1460	Inst.clear();
1461	Inst.setOpcode(NewOpcode);
1462	Inst.addOperand(Op: TargetOp);
1463	if (I.HasLHS)
1464	Inst.addOperand(Op: TargetOp);
1465	Inst.addOperand(Op: ImmOp);
1466
1467	return true;
1468	}
1469
1470	/// TODO: this implementation currently works for the most common opcodes that
1471	/// load from memory. It can be extended to work with memory store opcodes as
1472	/// well as more memory load opcodes.
1473	bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1474	unsigned NewOpcode;
1475
1476	switch (Inst.getOpcode()) {
1477	default: {
1478	switch (getPopSize(Inst)) {
1479	case 2: NewOpcode = X86::MOV16rr; break;
1480	case 4: NewOpcode = X86::MOV32rr; break;
1481	case 8: NewOpcode = X86::MOV64rr; break;
1482	default: return false;
1483	}
1484	break;
1485	}
1486
1487	// MOV
1488	case X86::MOV8rm: NewOpcode = X86::MOV8rr; break;
1489	case X86::MOV16rm: NewOpcode = X86::MOV16rr; break;
1490	case X86::MOV32rm: NewOpcode = X86::MOV32rr; break;
1491	case X86::MOV64rm: NewOpcode = X86::MOV64rr; break;
1492	}
1493
1494	// Modify the instruction.
1495	MCOperand RegOp = MCOperand::createReg(Reg: RegNum);
1496	MCOperand TargetOp = Inst.getOperand(i: 0);
1497	Inst.clear();
1498	Inst.setOpcode(NewOpcode);
1499	Inst.addOperand(Op: TargetOp);
1500	Inst.addOperand(Op: RegOp);
1501
1502	return true;
1503	}
1504
1505	bool isRedundantMove(const MCInst &Inst) const override {
1506	switch (Inst.getOpcode()) {
1507	default:
1508	return false;
1509
1510	// MOV
1511	case X86::MOV8rr:
1512	case X86::MOV16rr:
1513	case X86::MOV32rr:
1514	case X86::MOV64rr:
1515	break;
1516	}
1517
1518	assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1519	return Inst.getOperand(i: 0).getReg() == Inst.getOperand(i: 1).getReg();
1520	}
1521
1522	bool requiresAlignedAddress(const MCInst &Inst) const override {
1523	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
1524	for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1525	const MCOperandInfo &Op = Desc.operands()[I];
1526	if (Op.OperandType != MCOI::OPERAND_REGISTER)
1527	continue;
1528	if (Op.RegClass == X86::VR128RegClassID)
1529	return true;
1530	}
1531	return false;
1532	}
1533
1534	bool convertJmpToTailCall(MCInst &Inst) override {
1535	if (isTailCall(Inst))
1536	return false;
1537
1538	int NewOpcode;
1539	switch (Inst.getOpcode()) {
1540	default:
1541	return false;
1542	case X86::JMP_1:
1543	case X86::JMP_2:
1544	case X86::JMP_4:
1545	NewOpcode = X86::JMP_4;
1546	break;
1547	case X86::JMP16m:
1548	case X86::JMP32m:
1549	case X86::JMP64m:
1550	NewOpcode = X86::JMP32m;
1551	break;
1552	case X86::JMP16r:
1553	case X86::JMP32r:
1554	case X86::JMP64r:
1555	NewOpcode = X86::JMP32r;
1556	break;
1557	}
1558
1559	Inst.setOpcode(NewOpcode);
1560	setTailCall(Inst);
1561	return true;
1562	}
1563
1564	bool convertTailCallToJmp(MCInst &Inst) override {
1565	int NewOpcode;
1566	switch (Inst.getOpcode()) {
1567	default:
1568	return false;
1569	case X86::JMP_4:
1570	NewOpcode = X86::JMP_1;
1571	break;
1572	case X86::JMP32m:
1573	NewOpcode = X86::JMP64m;
1574	break;
1575	case X86::JMP32r:
1576	NewOpcode = X86::JMP64r;
1577	break;
1578	}
1579
1580	Inst.setOpcode(NewOpcode);
1581	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1582	clearOffset(Inst);
1583	return true;
1584	}
1585
1586	bool convertTailCallToCall(MCInst &Inst) override {
1587	int NewOpcode;
1588	switch (Inst.getOpcode()) {
1589	default:
1590	return false;
1591	case X86::JMP_4:
1592	NewOpcode = X86::CALL64pcrel32;
1593	break;
1594	case X86::JMP32m:
1595	NewOpcode = X86::CALL64m;
1596	break;
1597	case X86::JMP32r:
1598	NewOpcode = X86::CALL64r;
1599	break;
1600	}
1601
1602	Inst.setOpcode(NewOpcode);
1603	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1604	return true;
1605	}
1606
1607	InstructionListType createIndirectPLTCall(MCInst &&DirectCall,
1608	const MCSymbol *TargetLocation,
1609	MCContext *Ctx) override {
1610	assert((DirectCall.getOpcode() == X86::CALL64pcrel32 ||
1611	(DirectCall.getOpcode() == X86::JMP_4 && isTailCall(DirectCall))) &&
1612	"64-bit direct (tail) call instruction expected");
1613
1614	InstructionListType Code;
1615	// Create a new indirect call by converting the previous direct call.
1616	MCInst Inst = DirectCall;
1617	const auto NewOpcode =
1618	(Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1619	Inst.setOpcode(NewOpcode);
1620
1621	// Replace the first operand and preserve auxiliary operands of
1622	// the instruction.
1623	Inst.erase(I: Inst.begin());
1624	Inst.insert(I: Inst.begin(),
1625	Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
1626	Inst.insert(I: Inst.begin(),
1627	Op: MCOperand::createExpr( // Displacement
1628	Val: MCSymbolRefExpr::create(Symbol: TargetLocation, Ctx&: *Ctx)));
1629	Inst.insert(I: Inst.begin(),
1630	Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
1631	Inst.insert(I: Inst.begin(),
1632	Op: MCOperand::createImm(Val: 1)); // ScaleAmt
1633	Inst.insert(I: Inst.begin(),
1634	Op: MCOperand::createReg(Reg: X86::RIP)); // BaseReg
1635
1636	Code.emplace_back(args&: Inst);
1637	return Code;
1638	}
1639
1640	void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1641	bool IsTailCall = isTailCall(Inst);
1642	if (IsTailCall)
1643	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1644	if (Inst.getOpcode() == X86::CALL64m ||
1645	(Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1646	Inst.setOpcode(X86::MOV64rm);
1647	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1648	return;
1649	}
1650	if (Inst.getOpcode() == X86::CALL64r ||
1651	(Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1652	Inst.setOpcode(X86::MOV64rr);
1653	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1654	return;
1655	}
1656	LLVM_DEBUG(Inst.dump());
1657	llvm_unreachable("not implemented");
1658	}
1659
1660	bool shortenInstruction(MCInst &Inst,
1661	const MCSubtargetInfo &STI) const override {
1662	unsigned OldOpcode = Inst.getOpcode();
1663	unsigned NewOpcode = OldOpcode;
1664
1665	int MemOpNo = getMemoryOperandNo(Inst);
1666
1667	// Check and remove redundant Address-Size override prefix.
1668	if (opts::X86StripRedundantAddressSize) {
1669	uint64_t TSFlags = Info->get(Opcode: OldOpcode).TSFlags;
1670	unsigned Flags = Inst.getFlags();
1671
1672	if (!X86_MC::needsAddressSizeOverride(MI: Inst, STI, MemoryOperand: MemOpNo, TSFlags) &&
1673	Flags & X86::IP_HAS_AD_SIZE)
1674	Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1675	}
1676
1677	// Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1678	// SIB byte is present, but no index is used and modrm alone should have
1679	// been enough. Converting to NoRegister effectively removes the SIB byte.
1680	if (MemOpNo >= 0) {
1681	MCOperand &IndexOp =
1682	Inst.getOperand(i: static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1683	if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1684	IndexOp = MCOperand::createReg(Reg: X86::NoRegister);
1685	}
1686
1687	if (isBranch(Inst)) {
1688	NewOpcode = getShortBranchOpcode(Opcode: OldOpcode);
1689	} else if (OldOpcode == X86::MOV64ri) {
1690	if (Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1691	const int64_t Imm =
1692	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1693	if (int64_t(Imm) == int64_t(int32_t(Imm)))
1694	NewOpcode = X86::MOV64ri32;
1695	}
1696	} else {
1697	// If it's arithmetic instruction check if signed operand fits in 1 byte.
1698	const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(Opcode: OldOpcode);
1699	if (ShortOpcode != OldOpcode &&
1700	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1701	int64_t Imm =
1702	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1703	if (int64_t(Imm) == int64_t(int8_t(Imm)))
1704	NewOpcode = ShortOpcode;
1705	}
1706	}
1707
1708	if (NewOpcode == OldOpcode)
1709	return false;
1710
1711	Inst.setOpcode(NewOpcode);
1712	return true;
1713	}
1714
1715	bool
1716	convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1717	bool AllowBasePtrStackMemOp) const override {
1718	// - Register-register moves are OK
1719	// - Stores are filtered out by opcode (no store CMOV)
1720	// - Non-stack loads are prohibited (generally unsafe)
1721	// - Stack loads are OK if AllowStackMemOp is true
1722	// - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1723	if (mayLoad(Inst)) {
1724	// If stack memory operands are not allowed, no loads are allowed
1725	if (!AllowStackMemOp)
1726	return false;
1727
1728	// If stack memory operands are allowed, check if it's a load from stack
1729	bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1730	MCPhysReg Reg;
1731	int32_t SrcImm;
1732	uint16_t StackPtrReg;
1733	int64_t StackOffset;
1734	uint8_t Size;
1735	bool IsStackAccess =
1736	isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1737	StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1738	// Prohibit non-stack-based loads
1739	if (!IsStackAccess)
1740	return false;
1741	// If stack memory operands are allowed, check if it's RBP-based
1742	if (!AllowBasePtrStackMemOp &&
1743	RegInfo->isSubRegisterEq(RegA: X86::RBP, RegB: StackPtrReg))
1744	return false;
1745	}
1746
1747	unsigned NewOpcode = 0;
1748	switch (Inst.getOpcode()) {
1749	case X86::MOV16rr:
1750	NewOpcode = X86::CMOV16rr;
1751	break;
1752	case X86::MOV16rm:
1753	NewOpcode = X86::CMOV16rm;
1754	break;
1755	case X86::MOV32rr:
1756	NewOpcode = X86::CMOV32rr;
1757	break;
1758	case X86::MOV32rm:
1759	NewOpcode = X86::CMOV32rm;
1760	break;
1761	case X86::MOV64rr:
1762	NewOpcode = X86::CMOV64rr;
1763	break;
1764	case X86::MOV64rm:
1765	NewOpcode = X86::CMOV64rm;
1766	break;
1767	default:
1768	return false;
1769	}
1770	Inst.setOpcode(NewOpcode);
1771	// Insert CC at the end of prime operands, before annotations
1772	Inst.insert(I: Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1773	Op: MCOperand::createImm(Val: CC));
1774	// CMOV is a 3-operand MCInst, so duplicate the destination as src1
1775	Inst.insert(I: Inst.begin(), Op: Inst.getOperand(i: 0));
1776	return true;
1777	}
1778
1779	bool lowerTailCall(MCInst &Inst) override {
1780	if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1781	Inst.setOpcode(X86::JMP_1);
1782	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1783	return true;
1784	}
1785	return false;
1786	}
1787
1788	const MCSymbol *getTargetSymbol(const MCInst &Inst,
1789	unsigned OpNum = 0) const override {
1790	if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1791	return nullptr;
1792
1793	const MCOperand &Op = Inst.getOperand(i: OpNum);
1794	if (!Op.isExpr())
1795	return nullptr;
1796
1797	return MCPlusBuilder::getTargetSymbol(Expr: Op.getExpr());
1798	}
1799
1800	bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1801	const MCSymbol *&TBB, const MCSymbol *&FBB,
1802	MCInst *&CondBranch,
1803	MCInst *&UncondBranch) const override {
1804	auto I = End;
1805
1806	// Bottom-up analysis
1807	while (I != Begin) {
1808	--I;
1809
1810	// Ignore nops and CFIs
1811	if (isPseudo(Inst: *I))
1812	continue;
1813
1814	// Stop when we find the first non-terminator
1815	if (!isTerminator(Inst: *I))
1816	break;
1817
1818	if (!isBranch(Inst: *I))
1819	break;
1820
1821	// Handle unconditional branches.
1822	if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1823	I->getOpcode() == X86::JMP_4) &&
1824	!isTailCall(Inst: *I)) {
1825	// If any code was seen after this unconditional branch, we've seen
1826	// unreachable code. Ignore them.
1827	CondBranch = nullptr;
1828	UncondBranch = &*I;
1829	const MCSymbol *Sym = getTargetSymbol(Inst: *I);
1830	assert(Sym != nullptr &&
1831	"Couldn't extract BB symbol from jump operand");
1832	TBB = Sym;
1833	continue;
1834	}
1835
1836	// Ignore indirect branches
1837	if (getCondCode(Inst: *I) == X86::COND_INVALID)
1838	return false;
1839
1840	if (CondBranch == nullptr) {
1841	const MCSymbol *TargetBB = getTargetSymbol(Inst: *I);
1842	if (TargetBB == nullptr) {
1843	// Unrecognized branch target
1844	return false;
1845	}
1846	FBB = TBB;
1847	TBB = TargetBB;
1848	CondBranch = &*I;
1849	continue;
1850	}
1851
1852	llvm_unreachable("multiple conditional branches in one BB");
1853	}
1854	return true;
1855	}
1856
1857	/// Analyzes PIC-style jump table code template and return identified
1858	/// IndirectBranchType, MemLocInstr (all cases) and FixedEntryLoadInstr
1859	/// (POSSIBLE_PIC_FIXED_BRANCH case).
1860	template <typename Itr>
1861	std::tuple<IndirectBranchType, MCInst *, MCInst *>
1862	analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
1863	// Analyze PIC-style jump table code template:
1864	//
1865	// lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr
1866	// mov ({%r1|%r2}, %index, 4), {%r2|%r1}
1867	// add %r2, %r1
1868	// jmp *%r1
1869	//
1870	// or a fixed indirect jump template:
1871	//
1872	// movslq En(%rip), {%r2|%r1} <- FixedEntryLoadInstr
1873	// lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr
1874	// add %r2, %r1
1875	// jmp *%r1
1876	//
1877	// (with any irrelevant instructions in-between)
1878	//
1879	// When we call this helper we've already determined %r1 and %r2, and
1880	// reverse instruction iterator \p II is pointing to the ADD instruction.
1881	//
1882	// PIC jump table looks like following:
1883	//
1884	// JT: ----------
1885	// E1:| L1 - JT |
1886	// |----------|
1887	// E2:| L2 - JT |
1888	// |----------|
1889	// | |
1890	// ......
1891	// En:| Ln - JT |
1892	// ----------
1893	//
1894	// Where L1, L2, ..., Ln represent labels in the function.
1895	//
1896	// The actual relocations in the table will be of the form:
1897	//
1898	// Ln - JT
1899	// = (Ln - En) + (En - JT)
1900	// = R_X86_64_PC32(Ln) + En - JT
1901	// = R_X86_64_PC32(Ln + offsetof(En))
1902	//
1903	auto isRIPRel = [&](X86MemOperand &MO) {
1904	// NB: DispExpr should be set
1905	return MO.DispExpr != nullptr &&
1906	MO.BaseRegNum == RegInfo->getProgramCounter() &&
1907	MO.IndexRegNum == X86::NoRegister &&
1908	MO.SegRegNum == X86::NoRegister;
1909	};
1910	auto isIndexed = [](X86MemOperand &MO, MCPhysReg R) {
1911	// NB: IndexRegNum should be set.
1912	return MO.IndexRegNum != X86::NoRegister && MO.BaseRegNum == R &&
1913	MO.ScaleImm == 4 && MO.DispImm == 0 &&
1914	MO.SegRegNum == X86::NoRegister;
1915	};
1916	LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
1917	MCInst *FirstInstr = nullptr;
1918	MCInst *SecondInstr = nullptr;
1919	enum {
1920	NOMATCH = 0,
1921	MATCH_JUMP_TABLE,
1922	MATCH_FIXED_BRANCH,
1923	} MatchingState = NOMATCH;
1924	while (++II != IE) {
1925	MCInst &Instr = *II;
1926	const MCInstrDesc &InstrDesc = Info->get(Opcode: Instr.getOpcode());
1927	if (!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R1, RI: *RegInfo) &&
1928	!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R2, RI: *RegInfo)) {
1929	// Ignore instructions that don't affect R1, R2 registers.
1930	continue;
1931	}
1932	const bool IsMOVSXInstr = isMOVSX64rm32(Inst: Instr);
1933	const bool IsLEAInstr = isLEA64r(Inst: Instr);
1934	if (MatchingState == NOMATCH) {
1935	if (IsMOVSXInstr)
1936	MatchingState = MATCH_JUMP_TABLE;
1937	else if (IsLEAInstr)
1938	MatchingState = MATCH_FIXED_BRANCH;
1939	else
1940	break;
1941
1942	// Check if the first instruction is setting %r1 or %r2. In canonical
1943	// form lea sets %r1 and mov sets %r2. If it's the opposite - rename so
1944	// we have to only check a single form.
1945	unsigned DestReg = Instr.getOperand(i: 0).getReg();
1946	MCPhysReg &ExpectReg = MatchingState == MATCH_JUMP_TABLE ? R2 : R1;
1947	if (DestReg != ExpectReg)
1948	std::swap(a&: R1, b&: R2);
1949	if (DestReg != ExpectReg)
1950	break;
1951
1952	// Verify operands
1953	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst: Instr);
1954	if (!MO)
1955	break;
1956	if ((MatchingState == MATCH_JUMP_TABLE && isIndexed(*MO, R1)) ||
1957	(MatchingState == MATCH_FIXED_BRANCH && isRIPRel(*MO)))
1958	FirstInstr = &Instr;
1959	else
1960	break;
1961	} else {
1962	unsigned ExpectReg = MatchingState == MATCH_JUMP_TABLE ? R1 : R2;
1963	if (!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: ExpectReg, RI: *RegInfo))
1964	continue;
1965	if ((MatchingState == MATCH_JUMP_TABLE && !IsLEAInstr) ||
1966	(MatchingState == MATCH_FIXED_BRANCH && !IsMOVSXInstr))
1967	break;
1968	if (Instr.getOperand(i: 0).getReg() != ExpectReg)
1969	break;
1970
1971	// Verify operands.
1972	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst: Instr);
1973	if (!MO)
1974	break;
1975	if (!isRIPRel(*MO))
1976	break;
1977	SecondInstr = &Instr;
1978	break;
1979	}
1980	}
1981
1982	if (!SecondInstr)
1983	return std::make_tuple(args: IndirectBranchType::UNKNOWN, args: nullptr, args: nullptr);
1984
1985	if (MatchingState == MATCH_FIXED_BRANCH) {
1986	LLVM_DEBUG(dbgs() << "checking potential fixed indirect branch\n");
1987	return std::make_tuple(args: IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH,
1988	args&: FirstInstr, args&: SecondInstr);
1989	}
1990	LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
1991	return std::make_tuple(args: IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
1992	args&: SecondInstr, args: nullptr);
1993	}
1994
1995	IndirectBranchType
1996	analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,
1997	InstructionIterator End, const unsigned PtrSize,
1998	MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
1999	unsigned &IndexRegNumOut, int64_t &DispValueOut,
2000	const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,
2001	MCInst *&FixedEntryLoadInst) const override {
2002	// Try to find a (base) memory location from where the address for
2003	// the indirect branch is loaded. For X86-64 the memory will be specified
2004	// in the following format:
2005	//
2006	// {%rip}/{%basereg} + Imm + IndexReg * Scale
2007	//
2008	// We are interested in the cases where Scale == sizeof(uintptr_t) and
2009	// the contents of the memory are presumably an array of pointers to code.
2010	//
2011	// Normal jump table:
2012	//
2013	// jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr
2014	//
2015	// or
2016	//
2017	// mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr
2018	// ...
2019	// jmp %r1
2020	//
2021	// We handle PIC-style jump tables separately.
2022	//
2023	MemLocInstrOut = nullptr;
2024	BaseRegNumOut = X86::NoRegister;
2025	IndexRegNumOut = X86::NoRegister;
2026	DispValueOut = 0;
2027	DispExprOut = nullptr;
2028	FixedEntryLoadInst = nullptr;
2029
2030	std::reverse_iterator<InstructionIterator> II(End);
2031	std::reverse_iterator<InstructionIterator> IE(Begin);
2032
2033	IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2034
2035	// An instruction referencing memory used by jump instruction (directly or
2036	// via register). This location could be an array of function pointers
2037	// in case of indirect tail call, or a jump table.
2038	MCInst *MemLocInstr = nullptr;
2039
2040	if (MCPlus::getNumPrimeOperands(Inst: Instruction) == 1) {
2041	// If the indirect jump is on register - try to detect if the
2042	// register value is loaded from a memory location.
2043	assert(Instruction.getOperand(0).isReg() && "register operand expected");
2044	const unsigned R1 = Instruction.getOperand(i: 0).getReg();
2045	// Check if one of the previous instructions defines the jump-on register.
2046	for (auto PrevII = II; PrevII != IE; ++PrevII) {
2047	MCInst &PrevInstr = *PrevII;
2048	const MCInstrDesc &PrevInstrDesc = Info->get(Opcode: PrevInstr.getOpcode());
2049
2050	if (!PrevInstrDesc.hasDefOfPhysReg(MI: PrevInstr, Reg: R1, RI: *RegInfo))
2051	continue;
2052
2053	if (isMoveMem2Reg(Inst: PrevInstr)) {
2054	MemLocInstr = &PrevInstr;
2055	break;
2056	}
2057	if (isADD64rr(Inst: PrevInstr)) {
2058	unsigned R2 = PrevInstr.getOperand(i: 2).getReg();
2059	if (R1 == R2)
2060	return IndirectBranchType::UNKNOWN;
2061	std::tie(args&: Type, args&: MemLocInstr, args&: FixedEntryLoadInst) =
2062	analyzePICJumpTable(II: PrevII, IE, R1, R2);
2063	break;
2064	}
2065	return IndirectBranchType::UNKNOWN;
2066	}
2067	if (!MemLocInstr) {
2068	// No definition seen for the register in this function so far. Could be
2069	// an input parameter - which means it is an external code reference.
2070	// It also could be that the definition happens to be in the code that
2071	// we haven't processed yet. Since we have to be conservative, return
2072	// as UNKNOWN case.
2073	return IndirectBranchType::UNKNOWN;
2074	}
2075	} else {
2076	MemLocInstr = &Instruction;
2077	}
2078
2079	const MCRegister RIPRegister = RegInfo->getProgramCounter();
2080
2081	// Analyze the memory location.
2082	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst: *MemLocInstr);
2083	if (!MO)
2084	return IndirectBranchType::UNKNOWN;
2085
2086	BaseRegNumOut = MO->BaseRegNum;
2087	IndexRegNumOut = MO->IndexRegNum;
2088	DispValueOut = MO->DispImm;
2089	DispExprOut = MO->DispExpr;
2090
2091	if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) ||
2092	MO->SegRegNum != X86::NoRegister)
2093	return IndirectBranchType::UNKNOWN;
2094
2095	if (MemLocInstr == &Instruction &&
2096	(!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) {
2097	MemLocInstrOut = MemLocInstr;
2098	return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2099	}
2100
2101	switch (Type) {
2102	case IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE:
2103	if (MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister)
2104	return IndirectBranchType::UNKNOWN;
2105	break;
2106	case IndirectBranchType::POSSIBLE_PIC_FIXED_BRANCH:
2107	break;
2108	default:
2109	if (MO->ScaleImm != PtrSize)
2110	return IndirectBranchType::UNKNOWN;
2111	}
2112
2113	MemLocInstrOut = MemLocInstr;
2114
2115	return Type;
2116	}
2117
2118	/// Analyze a callsite to see if it could be a virtual method call. This only
2119	/// checks to see if the overall pattern is satisfied, it does not guarantee
2120	/// that the callsite is a true virtual method call.
2121	/// The format of virtual method calls that are recognized is one of the
2122	/// following:
2123	///
2124	/// Form 1: (found in debug code)
2125	/// add METHOD_OFFSET, %VtableReg
2126	/// mov (%VtableReg), %MethodReg
2127	/// ...
2128	/// call or jmp *%MethodReg
2129	///
2130	/// Form 2:
2131	/// mov METHOD_OFFSET(%VtableReg), %MethodReg
2132	/// ...
2133	/// call or jmp *%MethodReg
2134	///
2135	/// Form 3:
2136	/// ...
2137	/// call or jmp *METHOD_OFFSET(%VtableReg)
2138	///
2139	bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2140	InstructionIterator ForwardEnd,
2141	std::vector<MCInst *> &MethodFetchInsns,
2142	unsigned &VtableRegNum, unsigned &MethodRegNum,
2143	uint64_t &MethodOffset) const override {
2144	VtableRegNum = X86::NoRegister;
2145	MethodRegNum = X86::NoRegister;
2146	MethodOffset = 0;
2147
2148	std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2149	std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2150
2151	MCInst &CallInst = *Itr++;
2152	assert(isIndirectBranch(CallInst) || isCall(CallInst));
2153
2154	// The call can just be jmp offset(reg)
2155	if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst: CallInst)) {
2156	if (!MO->DispExpr && MO->BaseRegNum != X86::RIP &&
2157	MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) {
2158	MethodRegNum = MO->BaseRegNum;
2159	if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister &&
2160	MO->SegRegNum == X86::NoRegister) {
2161	VtableRegNum = MethodRegNum;
2162	MethodOffset = MO->DispImm;
2163	MethodFetchInsns.push_back(x: &CallInst);
2164	return true;
2165	}
2166	}
2167	return false;
2168	}
2169	if (CallInst.getOperand(i: 0).isReg())
2170	MethodRegNum = CallInst.getOperand(i: 0).getReg();
2171	else
2172	return false;
2173
2174	if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2175	VtableRegNum = X86::NoRegister;
2176	MethodRegNum = X86::NoRegister;
2177	return false;
2178	}
2179
2180	// find load from vtable, this may or may not include the method offset
2181	while (Itr != End) {
2182	MCInst &CurInst = *Itr++;
2183	const MCInstrDesc &Desc = Info->get(Opcode: CurInst.getOpcode());
2184	if (Desc.hasDefOfPhysReg(MI: CurInst, Reg: MethodRegNum, RI: *RegInfo)) {
2185	if (!mayLoad(Inst: CurInst))
2186	return false;
2187	if (std::optional<X86MemOperand> MO =
2188	evaluateX86MemoryOperand(Inst: CurInst)) {
2189	if (!MO->DispExpr && MO->ScaleImm == 1 &&
2190	MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP &&
2191	MO->BaseRegNum != X86::NoRegister &&
2192	MO->IndexRegNum == X86::NoRegister &&
2193	MO->SegRegNum == X86::NoRegister) {
2194	VtableRegNum = MO->BaseRegNum;
2195	MethodOffset = MO->DispImm;
2196	MethodFetchInsns.push_back(x: &CurInst);
2197	if (MethodOffset != 0)
2198	return true;
2199	break;
2200	}
2201	}
2202	return false;
2203	}
2204	}
2205
2206	if (!VtableRegNum)
2207	return false;
2208
2209	// look for any adds affecting the method register.
2210	while (Itr != End) {
2211	MCInst &CurInst = *Itr++;
2212	const MCInstrDesc &Desc = Info->get(Opcode: CurInst.getOpcode());
2213	if (Desc.hasDefOfPhysReg(MI: CurInst, Reg: VtableRegNum, RI: *RegInfo)) {
2214	if (isADDri(Inst: CurInst)) {
2215	assert(!MethodOffset);
2216	MethodOffset = CurInst.getOperand(i: 2).getImm();
2217	MethodFetchInsns.insert(position: MethodFetchInsns.begin(), x: &CurInst);
2218	break;
2219	}
2220	}
2221	}
2222
2223	return true;
2224	}
2225
2226	void createStackPointerIncrement(MCInst &Inst, int Size,
2227	bool NoFlagsClobber) const override {
2228	if (NoFlagsClobber) {
2229	Inst.setOpcode(X86::LEA64r);
2230	Inst.clear();
2231	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP));
2232	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP)); // BaseReg
2233	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2234	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
2235	Inst.addOperand(Op: MCOperand::createImm(Val: -Size)); // Displacement
2236	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
2237	return;
2238	}
2239	Inst.setOpcode(X86::SUB64ri8);
2240	Inst.clear();
2241	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP));
2242	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP));
2243	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2244	}
2245
2246	void createStackPointerDecrement(MCInst &Inst, int Size,
2247	bool NoFlagsClobber) const override {
2248	if (NoFlagsClobber) {
2249	Inst.setOpcode(X86::LEA64r);
2250	Inst.clear();
2251	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP));
2252	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP)); // BaseReg
2253	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2254	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
2255	Inst.addOperand(Op: MCOperand::createImm(Val: Size)); // Displacement
2256	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
2257	return;
2258	}
2259	Inst.setOpcode(X86::ADD64ri8);
2260	Inst.clear();
2261	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP));
2262	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RSP));
2263	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2264	}
2265
2266	void createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2267	const MCPhysReg &SrcReg, int Size) const override {
2268	unsigned NewOpcode;
2269	switch (Size) {
2270	default:
2271	llvm_unreachable("Invalid operand size");
2272	return;
2273	case 2: NewOpcode = X86::MOV16mr; break;
2274	case 4: NewOpcode = X86::MOV32mr; break;
2275	case 8: NewOpcode = X86::MOV64mr; break;
2276	}
2277	Inst.setOpcode(NewOpcode);
2278	Inst.clear();
2279	Inst.addOperand(Op: MCOperand::createReg(Reg: StackReg)); // BaseReg
2280	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2281	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
2282	Inst.addOperand(Op: MCOperand::createImm(Val: Offset)); // Displacement
2283	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
2284	Inst.addOperand(Op: MCOperand::createReg(Reg: SrcReg));
2285	}
2286
2287	void createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2288	int Offset, const MCPhysReg &DstReg,
2289	int Size) const override {
2290	return createLoad(Inst, BaseReg: StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2291	Offset, OffsetExpr: nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2292	DstReg, Size);
2293	}
2294
2295	void createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2296	const MCPhysReg &IndexReg, int64_t Offset,
2297	const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2298	const MCPhysReg &DstReg, int Size) const override {
2299	unsigned NewOpcode;
2300	switch (Size) {
2301	default:
2302	llvm_unreachable("Invalid operand size");
2303	return;
2304	case 2: NewOpcode = X86::MOV16rm; break;
2305	case 4: NewOpcode = X86::MOV32rm; break;
2306	case 8: NewOpcode = X86::MOV64rm; break;
2307	}
2308	Inst.setOpcode(NewOpcode);
2309	Inst.clear();
2310	Inst.addOperand(Op: MCOperand::createReg(Reg: DstReg));
2311	Inst.addOperand(Op: MCOperand::createReg(Reg: BaseReg));
2312	Inst.addOperand(Op: MCOperand::createImm(Val: Scale));
2313	Inst.addOperand(Op: MCOperand::createReg(Reg: IndexReg));
2314	if (OffsetExpr)
2315	Inst.addOperand(Op: MCOperand::createExpr(Val: OffsetExpr)); // Displacement
2316	else
2317	Inst.addOperand(Op: MCOperand::createImm(Val: Offset)); // Displacement
2318	Inst.addOperand(Op: MCOperand::createReg(Reg: AddrSegmentReg)); // AddrSegmentReg
2319	}
2320
2321	InstructionListType createLoadImmediate(const MCPhysReg Dest,
2322	uint64_t Imm) const override {
2323	InstructionListType Insts;
2324	Insts.emplace_back();
2325	Insts.back().setOpcode(X86::MOV64ri32);
2326	Insts.back().clear();
2327	Insts.back().addOperand(Op: MCOperand::createReg(Reg: Dest));
2328	Insts.back().addOperand(Op: MCOperand::createImm(Val: Imm));
2329	return Insts;
2330	}
2331
2332	void createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2333	const MCOperand &BaseReg, const MCOperand &Scale,
2334	const MCOperand &IndexReg, const MCOperand &Offset,
2335	const MCOperand &TmpReg) const override {
2336	// The code fragment we emit here is:
2337	//
2338	// mov32 (%base, %index, scale), %tmpreg
2339	// ijmp *(%tmpreg)
2340	//
2341	MCInst IJmp;
2342	IJmp.setOpcode(X86::JMP64r);
2343	IJmp.addOperand(Op: TmpReg);
2344
2345	MCInst Load;
2346	Load.setOpcode(X86::MOV32rm);
2347	Load.addOperand(Op: TmpReg);
2348	Load.addOperand(Op: BaseReg);
2349	Load.addOperand(Op: Scale);
2350	Load.addOperand(Op: IndexReg);
2351	Load.addOperand(Op: Offset);
2352	Load.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister));
2353
2354	Insts.push_back(Elt: Load);
2355	Insts.push_back(Elt: IJmp);
2356	}
2357
2358	void createNoop(MCInst &Inst) const override {
2359	Inst.setOpcode(X86::NOOP);
2360	Inst.clear();
2361	}
2362
2363	void createReturn(MCInst &Inst) const override {
2364	Inst.setOpcode(X86::RET64);
2365	Inst.clear();
2366	}
2367
2368	InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2369	InstructionListType Code;
2370	if (ReturnEnd)
2371	Code.emplace_back(args&: MCInstBuilder(X86::LEA64r)
2372	.addReg(Reg: X86::RAX)
2373	.addReg(Reg: X86::RDI)
2374	.addImm(Val: 1)
2375	.addReg(Reg: X86::RDX)
2376	.addImm(Val: 0)
2377	.addReg(Reg: X86::NoRegister));
2378	else
2379	Code.emplace_back(args&: MCInstBuilder(X86::MOV64rr)
2380	.addReg(Reg: X86::RAX)
2381	.addReg(Reg: X86::RDI));
2382
2383	Code.emplace_back(args&: MCInstBuilder(X86::MOV32rr)
2384	.addReg(Reg: X86::ECX)
2385	.addReg(Reg: X86::EDX));
2386	Code.emplace_back(args: MCInstBuilder(X86::REP_MOVSB_64));
2387
2388	return Code;
2389	}
2390
2391	InstructionListType createOneByteMemcpy() const override {
2392	InstructionListType Code;
2393	Code.emplace_back(args&: MCInstBuilder(X86::MOV8rm)
2394	.addReg(Reg: X86::CL)
2395	.addReg(Reg: X86::RSI)
2396	.addImm(Val: 0)
2397	.addReg(Reg: X86::NoRegister)
2398	.addImm(Val: 0)
2399	.addReg(Reg: X86::NoRegister));
2400	Code.emplace_back(args&: MCInstBuilder(X86::MOV8mr)
2401	.addReg(Reg: X86::RDI)
2402	.addImm(Val: 0)
2403	.addReg(Reg: X86::NoRegister)
2404	.addImm(Val: 0)
2405	.addReg(Reg: X86::NoRegister)
2406	.addReg(Reg: X86::CL));
2407	Code.emplace_back(args&: MCInstBuilder(X86::MOV64rr)
2408	.addReg(Reg: X86::RAX)
2409	.addReg(Reg: X86::RDI));
2410	return Code;
2411	}
2412
2413	InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2414	const MCSymbol *Target,
2415	MCContext *Ctx) const override {
2416	InstructionListType Code;
2417	Code.emplace_back(args&: MCInstBuilder(X86::CMP64ri8)
2418	.addReg(Reg: RegNo)
2419	.addImm(Val: Imm));
2420	Code.emplace_back(args&: MCInstBuilder(X86::JCC_1)
2421	.addExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx))
2422	.addImm(Val: X86::COND_E));
2423	return Code;
2424	}
2425
2426	InstructionListType createCmpJNE(MCPhysReg RegNo, int64_t Imm,
2427	const MCSymbol *Target,
2428	MCContext *Ctx) const override {
2429	InstructionListType Code;
2430	Code.emplace_back(args&: MCInstBuilder(X86::CMP64ri8).addReg(Reg: RegNo).addImm(Val: Imm));
2431	Code.emplace_back(args&: MCInstBuilder(X86::JCC_1)
2432	.addExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx))
2433	.addImm(Val: X86::COND_NE));
2434	return Code;
2435	}
2436
2437	std::optional<Relocation>
2438	createRelocation(const MCFixup &Fixup,
2439	const MCAsmBackend &MAB) const override {
2440	MCFixupKindInfo FKI = MAB.getFixupKindInfo(Kind: Fixup.getKind());
2441
2442	assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2443	const uint64_t RelOffset = Fixup.getOffset();
2444	auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
2445
2446	uint32_t RelType;
2447	if (Fixup.isPCRel()) {
2448	switch (FKI.TargetSize) {
2449	default:
2450	return std::nullopt;
2451	case 8: RelType = ELF::R_X86_64_PC8; break;
2452	case 16: RelType = ELF::R_X86_64_PC16; break;
2453	case 32: RelType = ELF::R_X86_64_PC32; break;
2454	case 64: RelType = ELF::R_X86_64_PC64; break;
2455	}
2456	// Adjust PC-relative fixup offsets, which are calculated from the start
2457	// of the next instruction.
2458	RelAddend -= FKI.TargetSize / 8;
2459	} else {
2460	switch (FKI.TargetSize) {
2461	default:
2462	return std::nullopt;
2463	case 8: RelType = ELF::R_X86_64_8; break;
2464	case 16: RelType = ELF::R_X86_64_16; break;
2465	case 32: RelType = ELF::R_X86_64_32; break;
2466	case 64: RelType = ELF::R_X86_64_64; break;
2467	}
2468	}
2469
2470	return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
2471	}
2472
2473	bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2474	int64_t Addend, MCContext *Ctx, int64_t &Value,
2475	uint32_t RelType) const override {
2476	unsigned ImmOpNo = -1U;
2477
2478	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2479	++Index) {
2480	if (Inst.getOperand(i: Index).isImm()) {
2481	ImmOpNo = Index;
2482	// TODO: this is a bit hacky. It finds the correct operand by
2483	// searching for a specific immediate value. If no value is
2484	// provided it defaults to the last immediate operand found.
2485	// This could lead to unexpected results if the instruction
2486	// has more than one immediate with the same value.
2487	if (Inst.getOperand(i: ImmOpNo).getImm() == Value)
2488	break;
2489	}
2490	}
2491
2492	if (ImmOpNo == -1U)
2493	return false;
2494
2495	Value = Inst.getOperand(i: ImmOpNo).getImm();
2496
2497	setOperandToSymbolRef(Inst, OpNum: ImmOpNo, Symbol, Addend, Ctx, RelType);
2498
2499	return true;
2500	}
2501
2502	bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2503	int64_t Imm) const override {
2504
2505	enum CheckSignExt : uint8_t {
2506	NOCHECK = 0,
2507	CHECK8,
2508	CHECK32,
2509	};
2510
2511	using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2512	struct InstInfo {
2513	// Size in bytes that Inst loads from memory.
2514	uint8_t DataSize;
2515
2516	// True when the target operand has to be duplicated because the opcode
2517	// expects a LHS operand.
2518	bool HasLHS;
2519
2520	// List of checks and corresponding opcodes to be used. We try to use the
2521	// smallest possible immediate value when various sizes are available,
2522	// hence we may need to check whether a larger constant fits in a smaller
2523	// immediate.
2524	CheckList Checks;
2525	};
2526
2527	InstInfo I;
2528
2529	switch (Inst.getOpcode()) {
2530	default: {
2531	switch (getPushSize(Inst)) {
2532
2533	case 2: I = {.DataSize: 2, .HasLHS: false, .Checks: {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break;
2534	case 4: I = {.DataSize: 4, .HasLHS: false, .Checks: {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break;
2535	case 8: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK8, X86::PUSH64i8},
2536	{CHECK32, X86::PUSH64i32},
2537	{NOCHECK, Inst.getOpcode()}}}; break;
2538	default: return false;
2539	}
2540	break;
2541	}
2542
2543	// MOV
2544	case X86::MOV8rr: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV8ri}}}; break;
2545	case X86::MOV16rr: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV16ri}}}; break;
2546	case X86::MOV32rr: I = {.DataSize: 4, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
2547	case X86::MOV64rr: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK32, X86::MOV64ri32},
2548	{NOCHECK, X86::MOV64ri}}}; break;
2549
2550	case X86::MOV8mr: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV8mi}}}; break;
2551	case X86::MOV16mr: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV16mi}}}; break;
2552	case X86::MOV32mr: I = {.DataSize: 4, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32mi}}}; break;
2553	case X86::MOV64mr: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK32, X86::MOV64mi32},
2554	{NOCHECK, X86::MOV64mr}}}; break;
2555
2556	// MOVZX
2557	case X86::MOVZX16rr8: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV16ri}}}; break;
2558	case X86::MOVZX32rr8: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
2559	case X86::MOVZX32rr16: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::MOV32ri}}}; break;
2560
2561	// CMP
2562	case X86::CMP8rr: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::CMP8ri}}}; break;
2563	case X86::CMP16rr: I = {.DataSize: 2, .HasLHS: false, .Checks: {{CHECK8, X86::CMP16ri8},
2564	{NOCHECK, X86::CMP16ri}}}; break;
2565	case X86::CMP32rr: I = {.DataSize: 4, .HasLHS: false, .Checks: {{CHECK8, X86::CMP32ri8},
2566	{NOCHECK, X86::CMP32ri}}}; break;
2567	case X86::CMP64rr: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK8, X86::CMP64ri8},
2568	{CHECK32, X86::CMP64ri32},
2569	{NOCHECK, X86::CMP64rr}}}; break;
2570
2571	// TEST
2572	case X86::TEST8rr: I = {.DataSize: 1, .HasLHS: false, .Checks: {{NOCHECK, X86::TEST8ri}}}; break;
2573	case X86::TEST16rr: I = {.DataSize: 2, .HasLHS: false, .Checks: {{NOCHECK, X86::TEST16ri}}}; break;
2574	case X86::TEST32rr: I = {.DataSize: 4, .HasLHS: false, .Checks: {{NOCHECK, X86::TEST32ri}}}; break;
2575	case X86::TEST64rr: I = {.DataSize: 8, .HasLHS: false, .Checks: {{CHECK32, X86::TEST64ri32},
2576	{NOCHECK, X86::TEST64rr}}}; break;
2577
2578	// ADD
2579	case X86::ADD8rr: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::ADD8ri}}}; break;
2580	case X86::ADD16rr: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::ADD16ri8},
2581	{NOCHECK, X86::ADD16ri}}}; break;
2582	case X86::ADD32rr: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::ADD32ri8},
2583	{NOCHECK, X86::ADD32ri}}}; break;
2584	case X86::ADD64rr: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::ADD64ri8},
2585	{CHECK32, X86::ADD64ri32},
2586	{NOCHECK, X86::ADD64rr}}}; break;
2587
2588	// SUB
2589	case X86::SUB8rr: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::SUB8ri}}}; break;
2590	case X86::SUB16rr: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::SUB16ri8},
2591	{NOCHECK, X86::SUB16ri}}}; break;
2592	case X86::SUB32rr: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::SUB32ri8},
2593	{NOCHECK, X86::SUB32ri}}}; break;
2594	case X86::SUB64rr: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::SUB64ri8},
2595	{CHECK32, X86::SUB64ri32},
2596	{NOCHECK, X86::SUB64rr}}}; break;
2597
2598	// AND
2599	case X86::AND8rr: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::AND8ri}}}; break;
2600	case X86::AND16rr: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::AND16ri8},
2601	{NOCHECK, X86::AND16ri}}}; break;
2602	case X86::AND32rr: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::AND32ri8},
2603	{NOCHECK, X86::AND32ri}}}; break;
2604	case X86::AND64rr: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::AND64ri8},
2605	{CHECK32, X86::AND64ri32},
2606	{NOCHECK, X86::AND64rr}}}; break;
2607
2608	// OR
2609	case X86::OR8rr: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::OR8ri}}}; break;
2610	case X86::OR16rr: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::OR16ri8},
2611	{NOCHECK, X86::OR16ri}}}; break;
2612	case X86::OR32rr: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::OR32ri8},
2613	{NOCHECK, X86::OR32ri}}}; break;
2614	case X86::OR64rr: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::OR64ri8},
2615	{CHECK32, X86::OR64ri32},
2616	{NOCHECK, X86::OR64rr}}}; break;
2617
2618	// XOR
2619	case X86::XOR8rr: I = {.DataSize: 1, .HasLHS: true, .Checks: {{NOCHECK, X86::XOR8ri}}}; break;
2620	case X86::XOR16rr: I = {.DataSize: 2, .HasLHS: true, .Checks: {{CHECK8, X86::XOR16ri8},
2621	{NOCHECK, X86::XOR16ri}}}; break;
2622	case X86::XOR32rr: I = {.DataSize: 4, .HasLHS: true, .Checks: {{CHECK8, X86::XOR32ri8},
2623	{NOCHECK, X86::XOR32ri}}}; break;
2624	case X86::XOR64rr: I = {.DataSize: 8, .HasLHS: true, .Checks: {{CHECK8, X86::XOR64ri8},
2625	{CHECK32, X86::XOR64ri32},
2626	{NOCHECK, X86::XOR64rr}}}; break;
2627	}
2628
2629	// Compute the new opcode.
2630	unsigned NewOpcode = 0;
2631	for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2632	NewOpcode = Check.second;
2633	if (Check.first == NOCHECK)
2634	break;
2635	if (Check.first == CHECK8 && isInt<8>(x: Imm))
2636	break;
2637	if (Check.first == CHECK32 && isInt<32>(x: Imm))
2638	break;
2639	}
2640	if (NewOpcode == Inst.getOpcode())
2641	return false;
2642
2643	const MCInstrDesc &InstDesc = Info->get(Opcode: Inst.getOpcode());
2644
2645	unsigned NumFound = 0;
2646	for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2647	E = InstDesc.getNumOperands();
2648	Index != E; ++Index)
2649	if (Inst.getOperand(i: Index).isReg() &&
2650	Inst.getOperand(i: Index).getReg() == Register)
2651	NumFound++;
2652
2653	if (NumFound != 1)
2654	return false;
2655
2656	MCOperand TargetOp = Inst.getOperand(i: 0);
2657	Inst.clear();
2658	Inst.setOpcode(NewOpcode);
2659	Inst.addOperand(Op: TargetOp);
2660	if (I.HasLHS)
2661	Inst.addOperand(Op: TargetOp);
2662	Inst.addOperand(Op: MCOperand::createImm(Val: Imm));
2663
2664	return true;
2665	}
2666
2667	bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2668	unsigned ReplaceWith) const override {
2669
2670	// Get the HasLHS value so that iteration can be done
2671	bool HasLHS;
2672	if (X86::isAND(Opcode: Inst.getOpcode()) || X86::isADD(Opcode: Inst.getOpcode()) ||
2673	X86::isSUB(Opcode: Inst.getOpcode())) {
2674	HasLHS = true;
2675	} else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Opcode: Inst.getOpcode()) ||
2676	X86::isTEST(Opcode: Inst.getOpcode())) {
2677	HasLHS = false;
2678	} else {
2679	switch (Inst.getOpcode()) {
2680	case X86::MOV8rr:
2681	case X86::MOV8rm:
2682	case X86::MOV8mr:
2683	case X86::MOV8ri:
2684	case X86::MOV16rr:
2685	case X86::MOV16rm:
2686	case X86::MOV16mr:
2687	case X86::MOV16ri:
2688	case X86::MOV32rr:
2689	case X86::MOV32rm:
2690	case X86::MOV32mr:
2691	case X86::MOV32ri:
2692	case X86::MOV64rr:
2693	case X86::MOV64rm:
2694	case X86::MOV64mr:
2695	case X86::MOV64ri:
2696	case X86::MOVZX16rr8:
2697	case X86::MOVZX32rr8:
2698	case X86::MOVZX32rr16:
2699	case X86::MOVSX32rm8:
2700	case X86::MOVSX32rr8:
2701	case X86::MOVSX64rm32:
2702	case X86::LEA64r:
2703	HasLHS = false;
2704	break;
2705	default:
2706	return false;
2707	}
2708	}
2709
2710	const MCInstrDesc &InstDesc = Info->get(Opcode: Inst.getOpcode());
2711
2712	bool FoundOne = false;
2713
2714	// Iterate only through src operands that arent also dest operands
2715	for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2716	E = InstDesc.getNumOperands();
2717	Index != E; ++Index) {
2718	BitVector RegAliases = getAliases(Reg: ToReplace, OnlySmaller: true);
2719	if (!Inst.getOperand(i: Index).isReg() ||
2720	!RegAliases.test(Idx: Inst.getOperand(i: Index).getReg()))
2721	continue;
2722	// Resize register if needed
2723	unsigned SizedReplaceWith = getAliasSized(
2724	Reg: ReplaceWith, Size: getRegSize(Reg: Inst.getOperand(i: Index).getReg()));
2725	MCOperand NewOperand = MCOperand::createReg(Reg: SizedReplaceWith);
2726	Inst.getOperand(i: Index) = NewOperand;
2727	FoundOne = true;
2728	}
2729
2730	// Return true if at least one operand was replaced
2731	return FoundOne;
2732	}
2733
2734	void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2735	MCContext *Ctx) const override {
2736	Inst.clear();
2737	Inst.setOpcode(X86::JMP_1);
2738	Inst.clear();
2739	Inst.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: TBB, Ctx&: *Ctx)));
2740	}
2741
2742	void createLongUncondBranch(MCInst &Inst, const MCSymbol *Target,
2743	MCContext *Ctx) const override {
2744	Inst.setOpcode(X86::JMP_4);
2745	Inst.clear();
2746	Inst.addOperand(
2747	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
2748	}
2749
2750	void createCall(MCInst &Inst, const MCSymbol *Target,
2751	MCContext *Ctx) override {
2752	Inst.setOpcode(X86::CALL64pcrel32);
2753	Inst.clear();
2754	Inst.addOperand(
2755	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
2756	}
2757
2758	void createTailCall(MCInst &Inst, const MCSymbol *Target,
2759	MCContext *Ctx) override {
2760	return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2761	}
2762
2763	void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2764	MCContext *Ctx) override {
2765	Seq.clear();
2766	Seq.emplace_back();
2767	createDirectCall(Inst&: Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2768	}
2769
2770	void createTrap(MCInst &Inst) const override {
2771	Inst.clear();
2772	Inst.setOpcode(X86::TRAP);
2773	}
2774
2775	void createCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC,
2776	MCContext *Ctx) const override {
2777	Inst.setOpcode(X86::JCC_1);
2778	Inst.clear();
2779	Inst.addOperand(
2780	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
2781	Inst.addOperand(Op: MCOperand::createImm(Val: CC));
2782	}
2783
2784	void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC,
2785	MCContext *Ctx) const override {
2786	Inst.setOpcode(X86::JCC_4);
2787	Inst.clear();
2788	Inst.addOperand(
2789	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
2790	Inst.addOperand(Op: MCOperand::createImm(Val: CC));
2791	}
2792
2793	void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2794	MCContext *Ctx) const override {
2795	unsigned InvCC = getInvertedCondCode(CC: getCondCode(Inst));
2796	assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2797	Inst.getOperand(i: Info->get(Opcode: Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2798	Inst.getOperand(i: 0) =
2799	MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: TBB, Ctx&: *Ctx));
2800	}
2801
2802	bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2803	unsigned CC) const override {
2804	if (CC == X86::COND_INVALID)
2805	return false;
2806	Inst.getOperand(i: Info->get(Opcode: Inst.getOpcode()).NumOperands - 1).setImm(CC);
2807	Inst.getOperand(i: 0) =
2808	MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: TBB, Ctx&: *Ctx));
2809	return true;
2810	}
2811
2812	unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2813	switch (CC) {
2814	default: return X86::COND_INVALID;
2815
2816	case X86::COND_E: return X86::COND_E;
2817	case X86::COND_NE: return X86::COND_E;
2818
2819	case X86::COND_L: return X86::COND_L;
2820	case X86::COND_GE: return X86::COND_L;
2821
2822	case X86::COND_LE: return X86::COND_G;
2823	case X86::COND_G: return X86::COND_G;
2824
2825	case X86::COND_B: return X86::COND_B;
2826	case X86::COND_AE: return X86::COND_B;
2827
2828	case X86::COND_BE: return X86::COND_A;
2829	case X86::COND_A: return X86::COND_A;
2830
2831	case X86::COND_S: return X86::COND_S;
2832	case X86::COND_NS: return X86::COND_S;
2833
2834	case X86::COND_P: return X86::COND_P;
2835	case X86::COND_NP: return X86::COND_P;
2836
2837	case X86::COND_O: return X86::COND_O;
2838	case X86::COND_NO: return X86::COND_O;
2839	}
2840	}
2841
2842	void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
2843	MCContext *Ctx) const override {
2844	assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
2845	"Invalid instruction");
2846	Inst.getOperand(i: 0) =
2847	MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: TBB, Ctx&: *Ctx));
2848	}
2849
2850	MCPhysReg getX86R11() const override { return X86::R11; }
2851
2852	unsigned getShortBranchOpcode(unsigned Opcode) const override {
2853	switch (Opcode) {
2854	default:
2855	return Opcode;
2856	case X86::JMP_2:
2857	return X86::JMP_1;
2858	case X86::JMP_4:
2859	return X86::JMP_1;
2860	case X86::JCC_2:
2861	return X86::JCC_1;
2862	case X86::JCC_4:
2863	return X86::JCC_1;
2864	}
2865	}
2866
2867	MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
2868	// FIXME: this should depend on the calling convention.
2869	switch (ArgNo) {
2870	case 0: return X86::RDI;
2871	case 1: return X86::RSI;
2872	case 2: return X86::RDX;
2873	case 3: return X86::RCX;
2874	case 4: return X86::R8;
2875	case 5: return X86::R9;
2876	default: return getNoRegister();
2877	}
2878	}
2879
2880	void createPause(MCInst &Inst) const override {
2881	Inst.clear();
2882	Inst.setOpcode(X86::PAUSE);
2883	}
2884
2885	void createLfence(MCInst &Inst) const override {
2886	Inst.clear();
2887	Inst.setOpcode(X86::LFENCE);
2888	}
2889
2890	void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
2891	bool IsTailCall) override {
2892	Inst.clear();
2893	Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
2894	Inst.addOperand(
2895	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
2896	if (IsTailCall)
2897	setTailCall(Inst);
2898	}
2899
2900	void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
2901	MCContext *Ctx, bool IsTailCall) override {
2902	Seq.clear();
2903	MCInst Inst;
2904	Inst.setOpcode(X86::JMP_1);
2905	Inst.addOperand(
2906	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Target, Ctx&: *Ctx)));
2907	if (IsTailCall)
2908	setTailCall(Inst);
2909	Seq.emplace_back(args&: Inst);
2910	}
2911
2912	bool isConditionalMove(const MCInst &Inst) const override {
2913	unsigned OpCode = Inst.getOpcode();
2914	return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
2915	OpCode == X86::CMOV64rr);
2916	}
2917
2918	bool isBranchOnMem(const MCInst &Inst) const override {
2919	unsigned OpCode = Inst.getOpcode();
2920	if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
2921	OpCode == X86::JMP64m)
2922	return true;
2923
2924	return false;
2925	}
2926
2927	bool isBranchOnReg(const MCInst &Inst) const override {
2928	unsigned OpCode = Inst.getOpcode();
2929	if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
2930	OpCode == X86::JMP64r)
2931	return true;
2932
2933	return false;
2934	}
2935
2936	void createPushRegister(MCInst &Inst, MCPhysReg Reg,
2937	unsigned Size) const override {
2938	Inst.clear();
2939	unsigned NewOpcode = 0;
2940	if (Reg == X86::EFLAGS) {
2941	switch (Size) {
2942	case 2: NewOpcode = X86::PUSHF16; break;
2943	case 4: NewOpcode = X86::PUSHF32; break;
2944	case 8: NewOpcode = X86::PUSHF64; break;
2945	default:
2946	llvm_unreachable("Unexpected size");
2947	}
2948	Inst.setOpcode(NewOpcode);
2949	return;
2950	}
2951	switch (Size) {
2952	case 2: NewOpcode = X86::PUSH16r; break;
2953	case 4: NewOpcode = X86::PUSH32r; break;
2954	case 8: NewOpcode = X86::PUSH64r; break;
2955	default:
2956	llvm_unreachable("Unexpected size");
2957	}
2958	Inst.setOpcode(NewOpcode);
2959	Inst.addOperand(Op: MCOperand::createReg(Reg));
2960	}
2961
2962	void createPopRegister(MCInst &Inst, MCPhysReg Reg,
2963	unsigned Size) const override {
2964	Inst.clear();
2965	unsigned NewOpcode = 0;
2966	if (Reg == X86::EFLAGS) {
2967	switch (Size) {
2968	case 2: NewOpcode = X86::POPF16; break;
2969	case 4: NewOpcode = X86::POPF32; break;
2970	case 8: NewOpcode = X86::POPF64; break;
2971	default:
2972	llvm_unreachable("Unexpected size");
2973	}
2974	Inst.setOpcode(NewOpcode);
2975	return;
2976	}
2977	switch (Size) {
2978	case 2: NewOpcode = X86::POP16r; break;
2979	case 4: NewOpcode = X86::POP32r; break;
2980	case 8: NewOpcode = X86::POP64r; break;
2981	default:
2982	llvm_unreachable("Unexpected size");
2983	}
2984	Inst.setOpcode(NewOpcode);
2985	Inst.addOperand(Op: MCOperand::createReg(Reg));
2986	}
2987
2988	void createPushFlags(MCInst &Inst, unsigned Size) const override {
2989	return createPushRegister(Inst, Reg: X86::EFLAGS, Size);
2990	}
2991
2992	void createPopFlags(MCInst &Inst, unsigned Size) const override {
2993	return createPopRegister(Inst, Reg: X86::EFLAGS, Size);
2994	}
2995
2996	void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
2997	unsigned Size) const {
2998	unsigned int Opcode;
2999	switch (Size) {
3000	case 1: Opcode = X86::ADD8ri; break;
3001	case 2: Opcode = X86::ADD16ri; break;
3002	case 4: Opcode = X86::ADD32ri; break;
3003	default:
3004	llvm_unreachable("Unexpected size");
3005	}
3006	Inst.setOpcode(Opcode);
3007	Inst.clear();
3008	Inst.addOperand(Op: MCOperand::createReg(Reg));
3009	Inst.addOperand(Op: MCOperand::createReg(Reg));
3010	Inst.addOperand(Op: MCOperand::createImm(Val: Value));
3011	}
3012
3013	void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3014	unsigned Size) const {
3015	unsigned int Opcode;
3016	switch (Size) {
3017	case 1: Opcode = X86::MOV8ri; break;
3018	case 2: Opcode = X86::MOV16ri; break;
3019	case 4: Opcode = X86::MOV32ri; break;
3020	// Writing to a 32-bit register always zeros the upper 32 bits of the
3021	// full-width register
3022	case 8:
3023	Opcode = X86::MOV32ri;
3024	Reg = getAliasSized(Reg, Size: 4);
3025	break;
3026	default:
3027	llvm_unreachable("Unexpected size");
3028	}
3029	Inst.setOpcode(Opcode);
3030	Inst.clear();
3031	Inst.addOperand(Op: MCOperand::createReg(Reg));
3032	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
3033	}
3034
3035	void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3036	Inst.setOpcode(X86::SETCCr);
3037	Inst.clear();
3038	Inst.addOperand(Op: MCOperand::createReg(Reg));
3039	Inst.addOperand(Op: MCOperand::createImm(Val: X86::COND_O));
3040	}
3041
3042	void createX86Lahf(MCInst &Inst) const {
3043	Inst.setOpcode(X86::LAHF);
3044	Inst.clear();
3045	}
3046
3047	void createX86Sahf(MCInst &Inst) const {
3048	Inst.setOpcode(X86::SAHF);
3049	Inst.clear();
3050	}
3051
3052	InstructionListType
3053	createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
3054	unsigned CodePointerSize) const override {
3055	InstructionListType Instrs(IsLeaf ? 13 : 11);
3056	unsigned int I = 0;
3057
3058	// Don't clobber application red zone (ABI dependent)
3059	if (IsLeaf)
3060	createStackPointerIncrement(Inst&: Instrs[I++], Size: 128,
3061	/*NoFlagsClobber=*/true);
3062
3063	// Performance improvements based on the optimization discussed at
3064	// https://reviews.llvm.org/D6629
3065	// LAHF/SAHF are used instead of PUSHF/POPF
3066	// PUSHF
3067	createPushRegister(Inst&: Instrs[I++], Reg: X86::RAX, Size: 8);
3068	createClearRegWithNoEFlagsUpdate(Inst&: Instrs[I++], Reg: X86::RAX, Size: 8);
3069	createX86Lahf(Inst&: Instrs[I++]);
3070	createPushRegister(Inst&: Instrs[I++], Reg: X86::RAX, Size: 8);
3071	createClearRegWithNoEFlagsUpdate(Inst&: Instrs[I++], Reg: X86::RAX, Size: 8);
3072	createX86SaveOVFlagToRegister(Inst&: Instrs[I++], Reg: X86::AL);
3073	// LOCK INC
3074	InstructionListType IncMem = createIncMemory(Target, Ctx);
3075	assert(IncMem.size() == 1 && "Invalid IncMem size");
3076	std::copy(first: IncMem.begin(), last: IncMem.end(), result: Instrs.begin() + I);
3077	I += IncMem.size();
3078	// POPF
3079	createAddRegImm(Inst&: Instrs[I++], Reg: X86::AL, Value: 127, Size: 1);
3080	createPopRegister(Inst&: Instrs[I++], Reg: X86::RAX, Size: 8);
3081	createX86Sahf(Inst&: Instrs[I++]);
3082	createPopRegister(Inst&: Instrs[I++], Reg: X86::RAX, Size: 8);
3083
3084	if (IsLeaf)
3085	createStackPointerDecrement(Inst&: Instrs[I], Size: 128,
3086	/*NoFlagsClobber=*/true);
3087	return Instrs;
3088	}
3089
3090	void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3091	int64_t Disp) const {
3092	Inst.setOpcode(X86::XCHG64rm);
3093	Inst.clear();
3094	Inst.addOperand(Op: MCOperand::createReg(Reg: Source));
3095	Inst.addOperand(Op: MCOperand::createReg(Reg: Source));
3096	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg)); // BaseReg
3097	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3098	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
3099	Inst.addOperand(Op: MCOperand::createImm(Val: Disp)); // Displacement
3100	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
3101	}
3102
3103	void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3104	int64_t Disp) const {
3105	Inst.setOpcode(X86::JMP64m);
3106	Inst.clear();
3107	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg)); // BaseReg
3108	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3109	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
3110	Inst.addOperand(Op: MCOperand::createImm(Val: Disp)); // Displacement
3111	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
3112	}
3113
3114	InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
3115	MCSymbol *HandlerFuncAddr,
3116	int CallSiteID,
3117	MCContext *Ctx) override {
3118	// Check if the target address expression used in the original indirect call
3119	// uses the stack pointer, which we are going to clobber.
3120	static BitVector SPAliases(getAliases(Reg: X86::RSP));
3121	bool UsesSP = any_of(Range: useOperands(Inst&: CallInst), P: [&](const MCOperand &Op) {
3122	return Op.isReg() && SPAliases[Op.getReg()];
3123	});
3124
3125	InstructionListType Insts;
3126	MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3127	// Code sequence used to enter indirect call instrumentation helper:
3128	// push %rdi
3129	// add $8, %rsp ;; $rsp may be used in target, so fix it to prev val
3130	// movq target, %rdi ;; via convertIndirectCallTargetToLoad
3131	// sub $8, %rsp ;; restore correct stack value
3132	// push %rdi
3133	// movq $CallSiteID, %rdi
3134	// push %rdi
3135	// callq/jmp HandlerFuncAddr
3136	Insts.emplace_back();
3137	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3138	if (UsesSP) { // Only adjust SP if we really need to
3139	Insts.emplace_back();
3140	createStackPointerDecrement(Inst&: Insts.back(), Size: 8, /*NoFlagsClobber=*/false);
3141	}
3142	Insts.emplace_back(args&: CallInst);
3143	// Insts.back() and CallInst now share the same annotation instruction.
3144	// Strip it from Insts.back(), only preserving tail call annotation.
3145	stripAnnotations(Inst&: Insts.back(), /*KeepTC=*/true);
3146	convertIndirectCallToLoad(Inst&: Insts.back(), Reg: TempReg);
3147	if (UsesSP) {
3148	Insts.emplace_back();
3149	createStackPointerIncrement(Inst&: Insts.back(), Size: 8, /*NoFlagsClobber=*/false);
3150	}
3151	Insts.emplace_back();
3152	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3153	InstructionListType LoadImm = createLoadImmediate(Dest: TempReg, Imm: CallSiteID);
3154	Insts.insert(position: Insts.end(), first: LoadImm.begin(), last: LoadImm.end());
3155	Insts.emplace_back();
3156	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3157
3158	MCInst &NewCallInst = Insts.emplace_back();
3159	createDirectCall(Inst&: NewCallInst, Target: HandlerFuncAddr, Ctx, IsTailCall: isTailCall(Inst: CallInst));
3160
3161	// Carry over metadata including tail call marker if present.
3162	stripAnnotations(Inst&: NewCallInst);
3163	moveAnnotations(SrcInst: std::move(CallInst), DstInst&: NewCallInst);
3164
3165	return Insts;
3166	}
3167
3168	InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3169	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3170	// We just need to undo the sequence created for every ind call in
3171	// instrumentIndirectTarget(), which can be accomplished minimally with:
3172	// popfq
3173	// pop %rdi
3174	// add $16, %rsp
3175	// xchg (%rsp), %rdi
3176	// jmp *-8(%rsp)
3177	InstructionListType Insts(5);
3178	createPopFlags(Inst&: Insts[0], Size: 8);
3179	createPopRegister(Inst&: Insts[1], Reg: TempReg, Size: 8);
3180	createStackPointerDecrement(Inst&: Insts[2], Size: 16, /*NoFlagsClobber=*/false);
3181	createSwap(Inst&: Insts[3], Source: TempReg, MemBaseReg: X86::RSP, Disp: 0);
3182	createIndirectBranch(Inst&: Insts[4], MemBaseReg: X86::RSP, Disp: -8);
3183	return Insts;
3184	}
3185
3186	InstructionListType
3187	createInstrumentedIndTailCallHandlerExitBB() const override {
3188	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3189	// Same thing as above, but for tail calls
3190	// popfq
3191	// add $16, %rsp
3192	// pop %rdi
3193	// jmp *-16(%rsp)
3194	InstructionListType Insts(4);
3195	createPopFlags(Inst&: Insts[0], Size: 8);
3196	createStackPointerDecrement(Inst&: Insts[1], Size: 16, /*NoFlagsClobber=*/false);
3197	createPopRegister(Inst&: Insts[2], Reg: TempReg, Size: 8);
3198	createIndirectBranch(Inst&: Insts[3], MemBaseReg: X86::RSP, Disp: -16);
3199	return Insts;
3200	}
3201
3202	InstructionListType
3203	createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3204	const MCSymbol *IndCallHandler,
3205	MCContext *Ctx) override {
3206	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3207	// Code sequence used to check whether InstrTampoline was initialized
3208	// and call it if so, returns via IndCallHandler.
3209	// pushfq
3210	// mov InstrTrampoline,%rdi
3211	// cmp $0x0,%rdi
3212	// je IndCallHandler
3213	// callq *%rdi
3214	// jmpq IndCallHandler
3215	InstructionListType Insts;
3216	Insts.emplace_back();
3217	createPushFlags(Inst&: Insts.back(), Size: 8);
3218	Insts.emplace_back();
3219	createMove(Inst&: Insts.back(), Src: InstrTrampoline, Reg: TempReg, Ctx);
3220	InstructionListType cmpJmp = createCmpJE(RegNo: TempReg, Imm: 0, Target: IndCallHandler, Ctx);
3221	Insts.insert(position: Insts.end(), first: cmpJmp.begin(), last: cmpJmp.end());
3222	Insts.emplace_back();
3223	Insts.back().setOpcode(X86::CALL64r);
3224	Insts.back().addOperand(Op: MCOperand::createReg(Reg: TempReg));
3225	Insts.emplace_back();
3226	createDirectCall(Inst&: Insts.back(), Target: IndCallHandler, Ctx, /*IsTailCall*/ true);
3227	return Insts;
3228	}
3229
3230	InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3231	InstructionListType Insts(2);
3232	MCSymbol *NumLocs = Ctx->getOrCreateSymbol(Name: "__bolt_num_counters");
3233	createMove(Inst&: Insts[0], Src: NumLocs, Reg: X86::EAX, Ctx);
3234	createReturn(Inst&: Insts[1]);
3235	return Insts;
3236	}
3237
3238	InstructionListType
3239	createInstrLocationsGetter(MCContext *Ctx) const override {
3240	InstructionListType Insts(2);
3241	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_locations");
3242	createLea(Inst&: Insts[0], Src: Locs, Reg: X86::EAX, Ctx);
3243	createReturn(Inst&: Insts[1]);
3244	return Insts;
3245	}
3246
3247	InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3248	InstructionListType Insts(2);
3249	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_tables");
3250	createLea(Inst&: Insts[0], Src: Locs, Reg: X86::EAX, Ctx);
3251	createReturn(Inst&: Insts[1]);
3252	return Insts;
3253	}
3254
3255	InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3256	InstructionListType Insts(2);
3257	MCSymbol *NumFuncs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_num_funcs");
3258	createMove(Inst&: Insts[0], Src: NumFuncs, Reg: X86::EAX, Ctx);
3259	createReturn(Inst&: Insts[1]);
3260	return Insts;
3261	}
3262
3263	InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3264	MCContext *Ctx) override {
3265	InstructionListType Insts(1);
3266	createUncondBranch(Inst&: Insts[0], TBB: TgtSym, Ctx);
3267	return Insts;
3268	}
3269
3270	BlocksVectorTy indirectCallPromotion(
3271	const MCInst &CallInst,
3272	const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3273	const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3274	const std::vector<MCInst *> &MethodFetchInsns,
3275	const bool MinimizeCodeSize, MCContext *Ctx) override {
3276	const bool IsTailCall = isTailCall(Inst: CallInst);
3277	const bool IsJumpTable = getJumpTable(Inst: CallInst) != 0;
3278	BlocksVectorTy Results;
3279
3280	// Label for the current code block.
3281	MCSymbol *NextTarget = nullptr;
3282
3283	// The join block which contains all the instructions following CallInst.
3284	// MergeBlock remains null if CallInst is a tail call.
3285	MCSymbol *MergeBlock = nullptr;
3286
3287	unsigned FuncAddrReg = X86::R10;
3288
3289	const bool LoadElim = !VtableSyms.empty();
3290	assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3291	"There must be a vtable entry for every method "
3292	"in the targets vector.");
3293
3294	if (MinimizeCodeSize && !LoadElim) {
3295	std::set<unsigned> UsedRegs;
3296
3297	for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(Inst: CallInst); ++I) {
3298	const MCOperand &Op = CallInst.getOperand(i: I);
3299	if (Op.isReg())
3300	UsedRegs.insert(x: Op.getReg());
3301	}
3302
3303	if (UsedRegs.count(x: X86::R10) == 0)
3304	FuncAddrReg = X86::R10;
3305	else if (UsedRegs.count(x: X86::R11) == 0)
3306	FuncAddrReg = X86::R11;
3307	else
3308	return Results;
3309	}
3310
3311	const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3312	assert(MergeBlock);
3313	NewCall.push_back(x: CallInst);
3314	MCInst &Merge = NewCall.back();
3315	Merge.clear();
3316	createUncondBranch(Inst&: Merge, TBB: MergeBlock, Ctx);
3317	};
3318
3319	for (unsigned int i = 0; i < Targets.size(); ++i) {
3320	Results.emplace_back(args&: NextTarget, args: InstructionListType());
3321	InstructionListType *NewCall = &Results.back().second;
3322
3323	if (MinimizeCodeSize && !LoadElim) {
3324	// Load the call target into FuncAddrReg.
3325	NewCall->push_back(x: CallInst); // Copy CallInst in order to get SMLoc
3326	MCInst &Target = NewCall->back();
3327	Target.clear();
3328	Target.setOpcode(X86::MOV64ri32);
3329	Target.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3330	if (Targets[i].first) {
3331	// Is this OK?
3332	Target.addOperand(Op: MCOperand::createExpr(
3333	Val: MCSymbolRefExpr::create(Symbol: Targets[i].first, Ctx&: *Ctx)));
3334	} else {
3335	const uint64_t Addr = Targets[i].second;
3336	// Immediate address is out of sign extended 32 bit range.
3337	if (int64_t(Addr) != int64_t(int32_t(Addr)))
3338	return BlocksVectorTy();
3339
3340	Target.addOperand(Op: MCOperand::createImm(Val: Addr));
3341	}
3342
3343	// Compare current call target to a specific address.
3344	NewCall->push_back(x: CallInst);
3345	MCInst &Compare = NewCall->back();
3346	Compare.clear();
3347	if (isBranchOnReg(Inst: CallInst))
3348	Compare.setOpcode(X86::CMP64rr);
3349	else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3350	Compare.setOpcode(X86::CMP64ri32);
3351	else
3352	Compare.setOpcode(X86::CMP64rm);
3353
3354	Compare.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3355
3356	// TODO: Would be preferable to only load this value once.
3357	for (unsigned i = 0;
3358	i < Info->get(Opcode: CallInst.getOpcode()).getNumOperands(); ++i)
3359	if (!CallInst.getOperand(i).isInst())
3360	Compare.addOperand(Op: CallInst.getOperand(i));
3361	} else {
3362	// Compare current call target to a specific address.
3363	NewCall->push_back(x: CallInst);
3364	MCInst &Compare = NewCall->back();
3365	Compare.clear();
3366	if (isBranchOnReg(Inst: CallInst))
3367	Compare.setOpcode(X86::CMP64ri32);
3368	else
3369	Compare.setOpcode(X86::CMP64mi32);
3370
3371	// Original call address.
3372	for (unsigned i = 0;
3373	i < Info->get(Opcode: CallInst.getOpcode()).getNumOperands(); ++i)
3374	if (!CallInst.getOperand(i).isInst())
3375	Compare.addOperand(Op: CallInst.getOperand(i));
3376
3377	// Target address.
3378	if (Targets[i].first || LoadElim) {
3379	const MCSymbol *Sym =
3380	LoadElim ? VtableSyms[i].first : Targets[i].first;
3381	const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3382	const MCExpr *Expr = MCSymbolRefExpr::create(Symbol: Sym, Ctx&: *Ctx);
3383	if (Addend)
3384	Expr = MCBinaryExpr::createAdd(
3385	LHS: Expr, RHS: MCConstantExpr::create(Value: Addend, Ctx&: *Ctx), Ctx&: *Ctx);
3386	Compare.addOperand(Op: MCOperand::createExpr(Val: Expr));
3387	} else {
3388	const uint64_t Addr = Targets[i].second;
3389	// Immediate address is out of sign extended 32 bit range.
3390	if (int64_t(Addr) != int64_t(int32_t(Addr)))
3391	return BlocksVectorTy();
3392
3393	Compare.addOperand(Op: MCOperand::createImm(Val: Addr));
3394	}
3395	}
3396
3397	// jump to next target compare.
3398	NextTarget =
3399	Ctx->createNamedTempSymbol(); // generate label for the next block
3400	NewCall->push_back(x: CallInst);
3401
3402	if (IsJumpTable) {
3403	MCInst &Je = NewCall->back();
3404
3405	// Jump to next compare if target addresses don't match.
3406	Je.clear();
3407	Je.setOpcode(X86::JCC_1);
3408	if (Targets[i].first)
3409	Je.addOperand(Op: MCOperand::createExpr(
3410	Val: MCSymbolRefExpr::create(Symbol: Targets[i].first, Ctx&: *Ctx)));
3411	else
3412	Je.addOperand(Op: MCOperand::createImm(Val: Targets[i].second));
3413
3414	Je.addOperand(Op: MCOperand::createImm(Val: X86::COND_E));
3415	assert(!isInvoke(CallInst));
3416	} else {
3417	MCInst &Jne = NewCall->back();
3418
3419	// Jump to next compare if target addresses don't match.
3420	Jne.clear();
3421	Jne.setOpcode(X86::JCC_1);
3422	Jne.addOperand(
3423	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: NextTarget, Ctx&: *Ctx)));
3424	Jne.addOperand(Op: MCOperand::createImm(Val: X86::COND_NE));
3425
3426	// Call specific target directly.
3427	Results.emplace_back(args: Ctx->createNamedTempSymbol(),
3428	args: InstructionListType());
3429	NewCall = &Results.back().second;
3430	NewCall->push_back(x: CallInst);
3431	MCInst &CallOrJmp = NewCall->back();
3432
3433	CallOrJmp.clear();
3434
3435	if (MinimizeCodeSize && !LoadElim) {
3436	CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3437	CallOrJmp.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3438	} else {
3439	CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3440
3441	if (Targets[i].first)
3442	CallOrJmp.addOperand(Op: MCOperand::createExpr(
3443	Val: MCSymbolRefExpr::create(Symbol: Targets[i].first, Ctx&: *Ctx)));
3444	else
3445	CallOrJmp.addOperand(Op: MCOperand::createImm(Val: Targets[i].second));
3446	}
3447	if (IsTailCall)
3448	setTailCall(CallOrJmp);
3449
3450	if (CallOrJmp.getOpcode() == X86::CALL64r ||
3451	CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3452	if (std::optional<uint32_t> Offset = getOffset(Inst: CallInst))
3453	// Annotated as duplicated call
3454	setOffset(Inst&: CallOrJmp, Offset: *Offset);
3455	}
3456
3457	if (isInvoke(Inst: CallInst) && !isInvoke(Inst: CallOrJmp)) {
3458	// Copy over any EH or GNU args size information from the original
3459	// call.
3460	std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(Inst: CallInst);
3461	if (EHInfo)
3462	addEHInfo(Inst&: CallOrJmp, LP: *EHInfo);
3463	int64_t GnuArgsSize = getGnuArgsSize(Inst: CallInst);
3464	if (GnuArgsSize >= 0)
3465	addGnuArgsSize(Inst&: CallOrJmp, GnuArgsSize);
3466	}
3467
3468	if (!IsTailCall) {
3469	// The fallthrough block for the most common target should be
3470	// the merge block.
3471	if (i == 0) {
3472	// Fallthrough to merge block.
3473	MergeBlock = Ctx->createNamedTempSymbol();
3474	} else {
3475	// Insert jump to the merge block if we are not doing a fallthrough.
3476	jumpToMergeBlock(*NewCall);
3477	}
3478	}
3479	}
3480	}
3481
3482	// Cold call block.
3483	Results.emplace_back(args&: NextTarget, args: InstructionListType());
3484	InstructionListType &NewCall = Results.back().second;
3485	for (const MCInst *Inst : MethodFetchInsns)
3486	if (Inst != &CallInst)
3487	NewCall.push_back(x: *Inst);
3488	NewCall.push_back(x: CallInst);
3489
3490	// Jump to merge block from cold call block
3491	if (!IsTailCall && !IsJumpTable) {
3492	jumpToMergeBlock(NewCall);
3493
3494	// Record merge block
3495	Results.emplace_back(args&: MergeBlock, args: InstructionListType());
3496	}
3497
3498	return Results;
3499	}
3500
3501	BlocksVectorTy jumpTablePromotion(
3502	const MCInst &IJmpInst,
3503	const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3504	const std::vector<MCInst *> &TargetFetchInsns,
3505	MCContext *Ctx) const override {
3506	assert(getJumpTable(IJmpInst) != 0);
3507	uint16_t IndexReg = getAnnotationAs<uint16_t>(Inst: IJmpInst, Name: "JTIndexReg");
3508	if (IndexReg == 0)
3509	return BlocksVectorTy();
3510
3511	BlocksVectorTy Results;
3512
3513	// Label for the current code block.
3514	MCSymbol *NextTarget = nullptr;
3515
3516	for (unsigned int i = 0; i < Targets.size(); ++i) {
3517	Results.emplace_back(args&: NextTarget, args: InstructionListType());
3518	InstructionListType *CurBB = &Results.back().second;
3519
3520	// Compare current index to a specific index.
3521	CurBB->emplace_back(args: MCInst());
3522	MCInst &CompareInst = CurBB->back();
3523	CompareInst.setLoc(IJmpInst.getLoc());
3524	CompareInst.setOpcode(X86::CMP64ri32);
3525	CompareInst.addOperand(Op: MCOperand::createReg(Reg: IndexReg));
3526
3527	const uint64_t CaseIdx = Targets[i].second;
3528	// Immediate address is out of sign extended 32 bit range.
3529	if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3530	return BlocksVectorTy();
3531
3532	CompareInst.addOperand(Op: MCOperand::createImm(Val: CaseIdx));
3533	shortenInstruction(Inst&: CompareInst, STI: *Ctx->getSubtargetInfo());
3534
3535	// jump to next target compare.
3536	NextTarget =
3537	Ctx->createNamedTempSymbol(); // generate label for the next block
3538	CurBB->push_back(x: MCInst());
3539
3540	MCInst &JEInst = CurBB->back();
3541	JEInst.setLoc(IJmpInst.getLoc());
3542
3543	// Jump to target if indices match
3544	JEInst.setOpcode(X86::JCC_1);
3545	JEInst.addOperand(Op: MCOperand::createExpr(
3546	Val: MCSymbolRefExpr::create(Symbol: Targets[i].first, Ctx&: *Ctx)));
3547	JEInst.addOperand(Op: MCOperand::createImm(Val: X86::COND_E));
3548	}
3549
3550	// Cold call block.
3551	Results.emplace_back(args&: NextTarget, args: InstructionListType());
3552	InstructionListType &CurBB = Results.back().second;
3553	for (const MCInst *Inst : TargetFetchInsns)
3554	if (Inst != &IJmpInst)
3555	CurBB.push_back(x: *Inst);
3556
3557	CurBB.push_back(x: IJmpInst);
3558
3559	return Results;
3560	}
3561
3562	private:
3563	void createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3564	MCContext *Ctx) const {
3565	Inst.setOpcode(X86::MOV64rm);
3566	Inst.clear();
3567	Inst.addOperand(Op: MCOperand::createReg(Reg));
3568	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RIP)); // BaseReg
3569	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3570	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
3571	Inst.addOperand(
3572	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Src,
3573	Ctx&: *Ctx))); // Displacement
3574	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
3575	}
3576
3577	void createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3578	MCContext *Ctx) const {
3579	Inst.setOpcode(X86::LEA64r);
3580	Inst.clear();
3581	Inst.addOperand(Op: MCOperand::createReg(Reg));
3582	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::RIP)); // BaseReg
3583	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3584	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // IndexReg
3585	Inst.addOperand(
3586	Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(Symbol: Src,
3587	Ctx&: *Ctx))); // Displacement
3588	Inst.addOperand(Op: MCOperand::createReg(Reg: X86::NoRegister)); // AddrSegmentReg
3589	}
3590	};
3591
3592	} // namespace
3593
3594	namespace llvm {
3595	namespace bolt {
3596
3597	MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3598	const MCInstrInfo *Info,
3599	const MCRegisterInfo *RegInfo,
3600	const MCSubtargetInfo *STI) {
3601	return new X86MCPlusBuilder(Analysis, Info, RegInfo, STI);
3602	}
3603
3604	} // namespace bolt
3605	} // namespace llvm
3606