1	//===- bolt/Target/X86/X86MCPlusBuilder.cpp -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file provides X86-specific MCPlus builder.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "MCTargetDesc/X86BaseInfo.h"
14	#include "MCTargetDesc/X86EncodingOptimization.h"
15	#include "MCTargetDesc/X86MCTargetDesc.h"
16	#include "X86MCSymbolizer.h"
17	#include "bolt/Core/MCPlus.h"
18	#include "bolt/Core/MCPlusBuilder.h"
19	#include "llvm/BinaryFormat/ELF.h"
20	#include "llvm/MC/MCContext.h"
21	#include "llvm/MC/MCFixupKindInfo.h"
22	#include "llvm/MC/MCInst.h"
23	#include "llvm/MC/MCInstBuilder.h"
24	#include "llvm/MC/MCInstrInfo.h"
25	#include "llvm/MC/MCRegister.h"
26	#include "llvm/MC/MCRegisterInfo.h"
27	#include "llvm/Support/CommandLine.h"
28	#include "llvm/Support/DataExtractor.h"
29	#include "llvm/Support/Debug.h"
30	#include "llvm/Support/Errc.h"
31	#include "llvm/Support/ErrorHandling.h"
32	#include "llvm/Support/ErrorOr.h"
33	#include <set>
34
35	#define DEBUG_TYPE "mcplus"
36
37	using namespace llvm;
38	using namespace bolt;
39
40	namespace opts {
41
42	extern cl::OptionCategory BoltOptCategory;
43
44	static cl::opt<bool> X86StripRedundantAddressSize(
45	"x86-strip-redundant-address-size",
46	cl::desc("Remove redundant Address-Size override prefix"), cl::init(Val: true),
47	cl::cat(BoltOptCategory));
48
49	} // namespace opts
50
51	namespace {
52
53	bool isMOVSX64rm32(const MCInst &Inst) {
54	return Inst.getOpcode() == X86::MOVSX64rm32;
55	}
56
57	bool isADD64rr(const MCInst &Inst) { return Inst.getOpcode() == X86::ADD64rr; }
58
59	bool isADDri(const MCInst &Inst) {
60	return Inst.getOpcode() == X86::ADD64ri32 ||
61	Inst.getOpcode() == X86::ADD64ri8;
62	}
63
64	// Create instruction to increment contents of target by 1
65	static InstructionListType createIncMemory(const MCSymbol *Target,
66	MCContext *Ctx) {
67	InstructionListType Insts;
68	Insts.emplace_back();
69	Insts.back().setOpcode(X86::LOCK_INC64m);
70	Insts.back().clear();
71	Insts.back().addOperand(Op: MCOperand::createReg(X86::Reg: RIP)); // BaseReg
72	Insts.back().addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
73	Insts.back().addOperand(Op: MCOperand::createReg(X86::Reg: NoRegister)); // IndexReg
74
75	Insts.back().addOperand(Op: MCOperand::createExpr(
76	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None,
77	Ctx&: *Ctx))); // Displacement
78	Insts.back().addOperand(
79	Op: MCOperand::createReg(X86::Reg: NoRegister)); // AddrSegmentReg
80	return Insts;
81	}
82
83	#define GET_INSTRINFO_OPERAND_TYPES_ENUM
84	#define GET_INSTRINFO_OPERAND_TYPE
85	#define GET_INSTRINFO_MEM_OPERAND_SIZE
86	#include "X86GenInstrInfo.inc"
87
88	class X86MCPlusBuilder : public MCPlusBuilder {
89	public:
90	using MCPlusBuilder::MCPlusBuilder;
91
92	std::unique_ptr<MCSymbolizer>
93	createTargetSymbolizer(BinaryFunction &Function,
94	bool CreateNewSymbols) const override {
95	return std::make_unique<X86MCSymbolizer>(Function, CreateNewSymbols);
96	}
97
98	bool isBranch(const MCInst &Inst) const override {
99	return Analysis->isBranch(Inst) && !isTailCall(Inst);
100	}
101
102	bool isNoop(const MCInst &Inst) const override {
103	return X86::isNOP(Inst.getOpcode());
104	}
105
106	unsigned getCondCode(const MCInst &Inst) const override {
107	unsigned Opcode = Inst.getOpcode();
108	if (X86::isJCC(Opcode))
109	return Inst.getOperand(i: Info->get(Opcode).NumOperands - 1).getImm();
110	return X86::COND_INVALID;
111	}
112
113	unsigned getInvertedCondCode(unsigned CC) const override {
114	switch (CC) {
115	default: return X86::COND_INVALID;
116	case X86::COND_E: return X86::COND_NE;
117	case X86::COND_NE: return X86::COND_E;
118	case X86::COND_L: return X86::COND_GE;
119	case X86::COND_LE: return X86::COND_G;
120	case X86::COND_G: return X86::COND_LE;
121	case X86::COND_GE: return X86::COND_L;
122	case X86::COND_B: return X86::COND_AE;
123	case X86::COND_BE: return X86::COND_A;
124	case X86::COND_A: return X86::COND_BE;
125	case X86::COND_AE: return X86::COND_B;
126	case X86::COND_S: return X86::COND_NS;
127	case X86::COND_NS: return X86::COND_S;
128	case X86::COND_P: return X86::COND_NP;
129	case X86::COND_NP: return X86::COND_P;
130	case X86::COND_O: return X86::COND_NO;
131	case X86::COND_NO: return X86::COND_O;
132	}
133	}
134
135	unsigned getCondCodesLogicalOr(unsigned CC1, unsigned CC2) const override {
136	enum DecodedCondCode : uint8_t {
137	DCC_EQUAL = 0x1,
138	DCC_GREATER = 0x2,
139	DCC_LESSER = 0x4,
140	DCC_GREATER_OR_LESSER = 0x6,
141	DCC_UNSIGNED = 0x8,
142	DCC_SIGNED = 0x10,
143	DCC_INVALID = 0x20,
144	};
145
146	auto decodeCondCode = [&](unsigned CC) -> uint8_t {
147	switch (CC) {
148	default: return DCC_INVALID;
149	case X86::COND_E: return DCC_EQUAL;
150	case X86::COND_NE: return DCC_GREATER | DCC_LESSER;
151	case X86::COND_L: return DCC_LESSER | DCC_SIGNED;
152	case X86::COND_LE: return DCC_EQUAL | DCC_LESSER | DCC_SIGNED;
153	case X86::COND_G: return DCC_GREATER | DCC_SIGNED;
154	case X86::COND_GE: return DCC_GREATER | DCC_EQUAL | DCC_SIGNED;
155	case X86::COND_B: return DCC_LESSER | DCC_UNSIGNED;
156	case X86::COND_BE: return DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED;
157	case X86::COND_A: return DCC_GREATER | DCC_UNSIGNED;
158	case X86::COND_AE: return DCC_GREATER | DCC_EQUAL | DCC_UNSIGNED;
159	}
160	};
161
162	uint8_t DCC = decodeCondCode(CC1) | decodeCondCode(CC2);
163
164	if (DCC & DCC_INVALID)
165	return X86::COND_INVALID;
166
167	if (DCC & DCC_SIGNED && DCC & DCC_UNSIGNED)
168	return X86::COND_INVALID;
169
170	switch (DCC) {
171	default: return X86::COND_INVALID;
172	case DCC_EQUAL | DCC_LESSER | DCC_SIGNED: return X86::COND_LE;
173	case DCC_EQUAL | DCC_LESSER | DCC_UNSIGNED: return X86::COND_BE;
174	case DCC_EQUAL | DCC_GREATER | DCC_SIGNED: return X86::COND_GE;
175	case DCC_EQUAL | DCC_GREATER | DCC_UNSIGNED: return X86::COND_AE;
176	case DCC_GREATER | DCC_LESSER | DCC_SIGNED: return X86::COND_NE;
177	case DCC_GREATER | DCC_LESSER | DCC_UNSIGNED: return X86::COND_NE;
178	case DCC_GREATER | DCC_LESSER: return X86::COND_NE;
179	case DCC_EQUAL | DCC_SIGNED: return X86::COND_E;
180	case DCC_EQUAL | DCC_UNSIGNED: return X86::COND_E;
181	case DCC_EQUAL: return X86::COND_E;
182	case DCC_LESSER | DCC_SIGNED: return X86::COND_L;
183	case DCC_LESSER | DCC_UNSIGNED: return X86::COND_B;
184	case DCC_GREATER | DCC_SIGNED: return X86::COND_G;
185	case DCC_GREATER | DCC_UNSIGNED: return X86::COND_A;
186	}
187	}
188
189	bool isValidCondCode(unsigned CC) const override {
190	return (CC != X86::COND_INVALID);
191	}
192
193	bool isBreakpoint(const MCInst &Inst) const override {
194	return Inst.getOpcode() == X86::INT3;
195	}
196
197	bool isPrefix(const MCInst &Inst) const override {
198	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
199	return X86II::isPrefix(TSFlags: Desc.TSFlags);
200	}
201
202	bool isRep(const MCInst &Inst) const override {
203	return Inst.getFlags() == X86::IP_HAS_REPEAT;
204	}
205
206	bool deleteREPPrefix(MCInst &Inst) const override {
207	if (Inst.getFlags() == X86::IP_HAS_REPEAT) {
208	Inst.setFlags(0);
209	return true;
210	}
211	return false;
212	}
213
214	bool isIndirectCall(const MCInst &Inst) const override {
215	return isCall(Inst) &&
216	((getMemoryOperandNo(Inst) != -1) || Inst.getOperand(i: 0).isReg());
217	}
218
219	bool isPop(const MCInst &Inst) const override {
220	return getPopSize(Inst) == 0 ? false : true;
221	}
222
223	bool isTerminateBranch(const MCInst &Inst) const override {
224	return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
225	}
226
227	int getPopSize(const MCInst &Inst) const override {
228	switch (Inst.getOpcode()) {
229	case X86::POP16r:
230	case X86::POP16rmm:
231	case X86::POP16rmr:
232	case X86::POPF16:
233	case X86::POPA16:
234	case X86::POPDS16:
235	case X86::POPES16:
236	case X86::POPFS16:
237	case X86::POPGS16:
238	case X86::POPSS16:
239	return 2;
240	case X86::POP32r:
241	case X86::POP32rmm:
242	case X86::POP32rmr:
243	case X86::POPA32:
244	case X86::POPDS32:
245	case X86::POPES32:
246	case X86::POPF32:
247	case X86::POPFS32:
248	case X86::POPGS32:
249	case X86::POPSS32:
250	return 4;
251	case X86::POP64r:
252	case X86::POP64rmm:
253	case X86::POP64rmr:
254	case X86::POPF64:
255	case X86::POPFS64:
256	case X86::POPGS64:
257	return 8;
258	}
259	return 0;
260	}
261
262	bool isPush(const MCInst &Inst) const override {
263	return getPushSize(Inst) == 0 ? false : true;
264	}
265
266	int getPushSize(const MCInst &Inst) const override {
267	switch (Inst.getOpcode()) {
268	case X86::PUSH16i8:
269	case X86::PUSH16r:
270	case X86::PUSH16rmm:
271	case X86::PUSH16rmr:
272	case X86::PUSHA16:
273	case X86::PUSHCS16:
274	case X86::PUSHDS16:
275	case X86::PUSHES16:
276	case X86::PUSHF16:
277	case X86::PUSHFS16:
278	case X86::PUSHGS16:
279	case X86::PUSHSS16:
280	case X86::PUSH16i:
281	return 2;
282	case X86::PUSH32i8:
283	case X86::PUSH32r:
284	case X86::PUSH32rmm:
285	case X86::PUSH32rmr:
286	case X86::PUSHA32:
287	case X86::PUSHCS32:
288	case X86::PUSHDS32:
289	case X86::PUSHES32:
290	case X86::PUSHF32:
291	case X86::PUSHFS32:
292	case X86::PUSHGS32:
293	case X86::PUSHSS32:
294	case X86::PUSH32i:
295	return 4;
296	case X86::PUSH64i32:
297	case X86::PUSH64i8:
298	case X86::PUSH64r:
299	case X86::PUSH64rmm:
300	case X86::PUSH64rmr:
301	case X86::PUSHF64:
302	case X86::PUSHFS64:
303	case X86::PUSHGS64:
304	return 8;
305	}
306	return 0;
307	}
308
309	bool isSUB(const MCInst &Inst) const override {
310	return X86::isSUB(Inst.getOpcode());
311	}
312
313	bool isLEA64r(const MCInst &Inst) const override {
314	return Inst.getOpcode() == X86::LEA64r;
315	}
316
317	bool isLeave(const MCInst &Inst) const override {
318	return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64;
319	}
320
321	bool isMoveMem2Reg(const MCInst &Inst) const override {
322	switch (Inst.getOpcode()) {
323	case X86::MOV16rm:
324	case X86::MOV32rm:
325	case X86::MOV64rm:
326	return true;
327	}
328	return false;
329	}
330
331	bool isUnsupportedBranch(const MCInst &Inst) const override {
332	if (isDynamicBranch(Inst))
333	return true;
334
335	switch (Inst.getOpcode()) {
336	default:
337	return false;
338	case X86::LOOP:
339	case X86::LOOPE:
340	case X86::LOOPNE:
341	case X86::JECXZ:
342	case X86::JRCXZ:
343	return true;
344	}
345	}
346
347	bool mayLoad(const MCInst &Inst) const override {
348	if (isPop(Inst))
349	return true;
350
351	int MemOpNo = getMemoryOperandNo(Inst);
352	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
353
354	if (MemOpNo == -1)
355	return false;
356
357	return MCII.mayLoad();
358	}
359
360	bool mayStore(const MCInst &Inst) const override {
361	if (isPush(Inst))
362	return true;
363
364	int MemOpNo = getMemoryOperandNo(Inst);
365	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
366
367	if (MemOpNo == -1)
368	return false;
369
370	return MCII.mayStore();
371	}
372
373	bool isCleanRegXOR(const MCInst &Inst) const override {
374	switch (Inst.getOpcode()) {
375	case X86::XOR16rr:
376	case X86::XOR32rr:
377	case X86::XOR64rr:
378	break;
379	default:
380	return false;
381	}
382	return (Inst.getOperand(i: 0).getReg() == Inst.getOperand(i: 2).getReg());
383	}
384
385	bool isPacked(const MCInst &Inst) const override {
386	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
387	return (Desc.TSFlags & X86II::OpPrefixMask) == X86II::PD;
388	}
389
390	bool shouldRecordCodeRelocation(uint64_t RelType) const override {
391	switch (RelType) {
392	case ELF::R_X86_64_8:
393	case ELF::R_X86_64_16:
394	case ELF::R_X86_64_32:
395	case ELF::R_X86_64_32S:
396	case ELF::R_X86_64_64:
397	case ELF::R_X86_64_PC8:
398	case ELF::R_X86_64_PC32:
399	case ELF::R_X86_64_PC64:
400	case ELF::R_X86_64_GOTPC64:
401	case ELF::R_X86_64_GOTPCRELX:
402	case ELF::R_X86_64_REX_GOTPCRELX:
403	return true;
404	case ELF::R_X86_64_PLT32:
405	case ELF::R_X86_64_GOTPCREL:
406	case ELF::R_X86_64_TPOFF32:
407	case ELF::R_X86_64_GOTTPOFF:
408	return false;
409	default:
410	llvm_unreachable("Unexpected x86 relocation type in code");
411	}
412	}
413
414	StringRef getTrapFillValue() const override { return StringRef("\314", 1); }
415
416	struct IndJmpMatcherFrag1 : MCInstMatcher {
417	std::unique_ptr<MCInstMatcher> Base;
418	std::unique_ptr<MCInstMatcher> Scale;
419	std::unique_ptr<MCInstMatcher> Index;
420	std::unique_ptr<MCInstMatcher> Offset;
421
422	IndJmpMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
423	std::unique_ptr<MCInstMatcher> Scale,
424	std::unique_ptr<MCInstMatcher> Index,
425	std::unique_ptr<MCInstMatcher> Offset)
426	: Base(std::move(Base)), Scale(std::move(Scale)),
427	Index(std::move(Index)), Offset(std::move(Offset)) {}
428
429	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
430	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
431	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
432	return false;
433
434	if (CurInst->getOpcode() != X86::JMP64m)
435	return false;
436
437	int MemOpNo = MIB.getMemoryOperandNo(Inst: *CurInst);
438	if (MemOpNo == -1)
439	return false;
440
441	if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
442	return false;
443	if (!Scale->match(MRI, MIB, this->InstrWindow,
444	MemOpNo + X86::AddrScaleAmt))
445	return false;
446	if (!Index->match(MRI, MIB, this->InstrWindow,
447	MemOpNo + X86::AddrIndexReg))
448	return false;
449	if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
450	return false;
451	return true;
452	}
453
454	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
455	MIB.addAnnotation(*CurInst, Annotation, true);
456	Base->annotate(MIB, Annotation);
457	Scale->annotate(MIB, Annotation);
458	Index->annotate(MIB, Annotation);
459	Offset->annotate(MIB, Annotation);
460	}
461	};
462
463	std::unique_ptr<MCInstMatcher>
464	matchIndJmp(std::unique_ptr<MCInstMatcher> Base,
465	std::unique_ptr<MCInstMatcher> Scale,
466	std::unique_ptr<MCInstMatcher> Index,
467	std::unique_ptr<MCInstMatcher> Offset) const override {
468	return std::unique_ptr<MCInstMatcher>(
469	new IndJmpMatcherFrag1(std::move(Base), std::move(Scale),
470	std::move(Index), std::move(Offset)));
471	}
472
473	struct IndJmpMatcherFrag2 : MCInstMatcher {
474	std::unique_ptr<MCInstMatcher> Reg;
475
476	IndJmpMatcherFrag2(std::unique_ptr<MCInstMatcher> Reg)
477	: Reg(std::move(Reg)) {}
478
479	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
480	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
481	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
482	return false;
483
484	if (CurInst->getOpcode() != X86::JMP64r)
485	return false;
486
487	return Reg->match(MRI, MIB, this->InstrWindow, 0);
488	}
489
490	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
491	MIB.addAnnotation(*CurInst, Annotation, true);
492	Reg->annotate(MIB, Annotation);
493	}
494	};
495
496	std::unique_ptr<MCInstMatcher>
497	matchIndJmp(std::unique_ptr<MCInstMatcher> Target) const override {
498	return std::unique_ptr<MCInstMatcher>(
499	new IndJmpMatcherFrag2(std::move(Target)));
500	}
501
502	struct LoadMatcherFrag1 : MCInstMatcher {
503	std::unique_ptr<MCInstMatcher> Base;
504	std::unique_ptr<MCInstMatcher> Scale;
505	std::unique_ptr<MCInstMatcher> Index;
506	std::unique_ptr<MCInstMatcher> Offset;
507
508	LoadMatcherFrag1(std::unique_ptr<MCInstMatcher> Base,
509	std::unique_ptr<MCInstMatcher> Scale,
510	std::unique_ptr<MCInstMatcher> Index,
511	std::unique_ptr<MCInstMatcher> Offset)
512	: Base(std::move(Base)), Scale(std::move(Scale)),
513	Index(std::move(Index)), Offset(std::move(Offset)) {}
514
515	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
516	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
517	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
518	return false;
519
520	if (CurInst->getOpcode() != X86::MOV64rm &&
521	CurInst->getOpcode() != X86::MOVSX64rm32)
522	return false;
523
524	int MemOpNo = MIB.getMemoryOperandNo(Inst: *CurInst);
525	if (MemOpNo == -1)
526	return false;
527
528	if (!Base->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrBaseReg))
529	return false;
530	if (!Scale->match(MRI, MIB, this->InstrWindow,
531	MemOpNo + X86::AddrScaleAmt))
532	return false;
533	if (!Index->match(MRI, MIB, this->InstrWindow,
534	MemOpNo + X86::AddrIndexReg))
535	return false;
536	if (!Offset->match(MRI, MIB, this->InstrWindow, MemOpNo + X86::AddrDisp))
537	return false;
538	return true;
539	}
540
541	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
542	MIB.addAnnotation(*CurInst, Annotation, true);
543	Base->annotate(MIB, Annotation);
544	Scale->annotate(MIB, Annotation);
545	Index->annotate(MIB, Annotation);
546	Offset->annotate(MIB, Annotation);
547	}
548	};
549
550	std::unique_ptr<MCInstMatcher>
551	matchLoad(std::unique_ptr<MCInstMatcher> Base,
552	std::unique_ptr<MCInstMatcher> Scale,
553	std::unique_ptr<MCInstMatcher> Index,
554	std::unique_ptr<MCInstMatcher> Offset) const override {
555	return std::unique_ptr<MCInstMatcher>(
556	new LoadMatcherFrag1(std::move(Base), std::move(Scale),
557	std::move(Index), std::move(Offset)));
558	}
559
560	struct AddMatcher : MCInstMatcher {
561	std::unique_ptr<MCInstMatcher> A;
562	std::unique_ptr<MCInstMatcher> B;
563
564	AddMatcher(std::unique_ptr<MCInstMatcher> A,
565	std::unique_ptr<MCInstMatcher> B)
566	: A(std::move(A)), B(std::move(B)) {}
567
568	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
569	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
570	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
571	return false;
572
573	if (CurInst->getOpcode() == X86::ADD64rr ||
574	CurInst->getOpcode() == X86::ADD64rr_DB ||
575	CurInst->getOpcode() == X86::ADD64rr_REV) {
576	if (!A->match(MRI, MIB, this->InstrWindow, 1)) {
577	if (!B->match(MRI, MIB, this->InstrWindow, 1))
578	return false;
579	return A->match(MRI, MIB, this->InstrWindow, 2);
580	}
581
582	if (B->match(MRI, MIB, this->InstrWindow, 2))
583	return true;
584
585	if (!B->match(MRI, MIB, this->InstrWindow, 1))
586	return false;
587	return A->match(MRI, MIB, this->InstrWindow, 2);
588	}
589
590	return false;
591	}
592
593	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
594	MIB.addAnnotation(*CurInst, Annotation, true);
595	A->annotate(MIB, Annotation);
596	B->annotate(MIB, Annotation);
597	}
598	};
599
600	std::unique_ptr<MCInstMatcher>
601	matchAdd(std::unique_ptr<MCInstMatcher> A,
602	std::unique_ptr<MCInstMatcher> B) const override {
603	return std::unique_ptr<MCInstMatcher>(
604	new AddMatcher(std::move(A), std::move(B)));
605	}
606
607	struct LEAMatcher : MCInstMatcher {
608	std::unique_ptr<MCInstMatcher> Target;
609
610	LEAMatcher(std::unique_ptr<MCInstMatcher> Target)
611	: Target(std::move(Target)) {}
612
613	bool match(const MCRegisterInfo &MRI, MCPlusBuilder &MIB,
614	MutableArrayRef<MCInst> InInstrWindow, int OpNum) override {
615	if (!MCInstMatcher::match(MRI, MIB, InInstrWindow, OpNum))
616	return false;
617
618	if (CurInst->getOpcode() != X86::LEA64r)
619	return false;
620
621	if (CurInst->getOperand(1 + X86::AddrScaleAmt).getImm() != 1 ||
622	CurInst->getOperand(1 + X86::AddrIndexReg).getReg() !=
623	X86::NoRegister ||
624	(CurInst->getOperand(1 + X86::AddrBaseReg).getReg() !=
625	X86::NoRegister &&
626	CurInst->getOperand(1 + X86::AddrBaseReg).getReg() != X86::RIP))
627	return false;
628
629	return Target->match(MRI, MIB, this->InstrWindow, 1 + X86::AddrDisp);
630	}
631
632	void annotate(MCPlusBuilder &MIB, StringRef Annotation) override {
633	MIB.addAnnotation(*CurInst, Annotation, true);
634	Target->annotate(MIB, Annotation);
635	}
636	};
637
638	std::unique_ptr<MCInstMatcher>
639	matchLoadAddr(std::unique_ptr<MCInstMatcher> Target) const override {
640	return std::unique_ptr<MCInstMatcher>(new LEAMatcher(std::move(Target)));
641	}
642
643	bool hasPCRelOperand(const MCInst &Inst) const override {
644	for (const MCOperand &Operand : Inst)
645	if (Operand.isReg() && Operand.getReg() == X86::RIP)
646	return true;
647	return false;
648	}
649
650	int getMemoryOperandNo(const MCInst &Inst) const override {
651	unsigned Opcode = Inst.getOpcode();
652	const MCInstrDesc &Desc = Info->get(Opcode);
653	int MemOpNo = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
654	if (MemOpNo >= 0)
655	MemOpNo += X86II::getOperandBias(Desc);
656	return MemOpNo;
657	}
658
659	bool hasEVEXEncoding(const MCInst &Inst) const override {
660	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
661	return (Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX;
662	}
663
664	bool isMacroOpFusionPair(ArrayRef<MCInst> Insts) const override {
665	const auto *I = Insts.begin();
666	while (I != Insts.end() && isPrefix(Inst: *I))
667	++I;
668	if (I == Insts.end())
669	return false;
670
671	const MCInst &FirstInst = *I;
672	++I;
673	while (I != Insts.end() && isPrefix(Inst: *I))
674	++I;
675	if (I == Insts.end())
676	return false;
677	const MCInst &SecondInst = *I;
678
679	if (!isConditionalBranch(Inst: SecondInst))
680	return false;
681	// Cannot fuse if the first instruction uses RIP-relative memory.
682	if (hasPCRelOperand(Inst: FirstInst))
683	return false;
684
685	const X86::FirstMacroFusionInstKind CmpKind =
686	X86::classifyFirstOpcodeInMacroFusion(Opcode: FirstInst.getOpcode());
687	if (CmpKind == X86::FirstMacroFusionInstKind::Invalid)
688	return false;
689
690	X86::CondCode CC = static_cast<X86::CondCode>(getCondCode(Inst: SecondInst));
691	X86::SecondMacroFusionInstKind BranchKind =
692	X86::classifySecondCondCodeInMacroFusion(CC);
693	if (BranchKind == X86::SecondMacroFusionInstKind::Invalid)
694	return false;
695	return X86::isMacroFused(FirstKind: CmpKind, SecondKind: BranchKind);
696	}
697
698	std::optional<X86MemOperand>
699	evaluateX86MemoryOperand(const MCInst &Inst) const override {
700	int MemOpNo = getMemoryOperandNo(Inst);
701	if (MemOpNo < 0)
702	return std::nullopt;
703	unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
704
705	if (MemOpOffset + X86::AddrSegmentReg >= MCPlus::getNumPrimeOperands(Inst))
706	return std::nullopt;
707
708	const MCOperand &Base = Inst.getOperand(i: MemOpOffset + X86::AddrBaseReg);
709	const MCOperand &Scale = Inst.getOperand(i: MemOpOffset + X86::AddrScaleAmt);
710	const MCOperand &Index = Inst.getOperand(i: MemOpOffset + X86::AddrIndexReg);
711	const MCOperand &Disp = Inst.getOperand(i: MemOpOffset + X86::AddrDisp);
712	const MCOperand &Segment =
713	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg);
714
715	// Make sure it is a well-formed memory operand.
716	if (!Base.isReg() || !Scale.isImm() || !Index.isReg() ||
717	(!Disp.isImm() && !Disp.isExpr()) || !Segment.isReg())
718	return std::nullopt;
719
720	X86MemOperand MO;
721	MO.BaseRegNum = Base.getReg();
722	MO.ScaleImm = Scale.getImm();
723	MO.IndexRegNum = Index.getReg();
724	MO.DispImm = Disp.isImm() ? Disp.getImm() : 0;
725	MO.DispExpr = Disp.isExpr() ? Disp.getExpr() : nullptr;
726	MO.SegRegNum = Segment.getReg();
727	return MO;
728	}
729
730	bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,
731	uint64_t Address,
732	uint64_t Size) const override {
733	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
734	if (!MO)
735	return false;
736
737	// Make sure it's a well-formed addressing we can statically evaluate.
738	if ((MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::NoRegister) ||
739	MO->IndexRegNum != X86::NoRegister ||
740	MO->SegRegNum != X86::NoRegister || MO->DispExpr)
741	return false;
742
743	Target = MO->DispImm;
744	if (MO->BaseRegNum == X86::RIP) {
745	assert(Size != 0 && "instruction size required in order to statically "
746	"evaluate RIP-relative address");
747	Target += Address + Size;
748	}
749	return true;
750	}
751
752	MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {
753	int MemOpNo = getMemoryOperandNo(Inst);
754	if (MemOpNo < 0)
755	return Inst.end();
756	return Inst.begin() + (MemOpNo + X86::AddrDisp);
757	}
758
759	bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {
760	MCOperand *OI = getMemOperandDisp(Inst);
761	if (OI == Inst.end())
762	return false;
763	*OI = Operand;
764	return true;
765	}
766
767	/// Get the registers used as function parameters.
768	/// This function is specific to the x86_64 abi on Linux.
769	BitVector getRegsUsedAsParams() const override {
770	BitVector Regs = BitVector(RegInfo->getNumRegs(), false);
771	Regs |= getAliases(X86::RSI);
772	Regs |= getAliases(X86::RDI);
773	Regs |= getAliases(X86::RDX);
774	Regs |= getAliases(X86::RCX);
775	Regs |= getAliases(X86::R8);
776	Regs |= getAliases(X86::R9);
777	return Regs;
778	}
779
780	void getCalleeSavedRegs(BitVector &Regs) const override {
781	Regs |= getAliases(X86::RBX);
782	Regs |= getAliases(X86::RBP);
783	Regs |= getAliases(X86::R12);
784	Regs |= getAliases(X86::R13);
785	Regs |= getAliases(X86::R14);
786	Regs |= getAliases(X86::R15);
787	}
788
789	void getDefaultDefIn(BitVector &Regs) const override {
790	assert(Regs.size() >= RegInfo->getNumRegs() &&
791	"The size of BitVector is less than RegInfo->getNumRegs().");
792	Regs.set(X86::RAX);
793	Regs.set(X86::RCX);
794	Regs.set(X86::RDX);
795	Regs.set(X86::RSI);
796	Regs.set(X86::RDI);
797	Regs.set(X86::R8);
798	Regs.set(X86::R9);
799	Regs.set(X86::XMM0);
800	Regs.set(X86::XMM1);
801	Regs.set(X86::XMM2);
802	Regs.set(X86::XMM3);
803	Regs.set(X86::XMM4);
804	Regs.set(X86::XMM5);
805	Regs.set(X86::XMM6);
806	Regs.set(X86::XMM7);
807	}
808
809	void getDefaultLiveOut(BitVector &Regs) const override {
810	assert(Regs.size() >= RegInfo->getNumRegs() &&
811	"The size of BitVector is less than RegInfo->getNumRegs().");
812	Regs |= getAliases(X86::RAX);
813	Regs |= getAliases(X86::RDX);
814	Regs |= getAliases(X86::RCX);
815	Regs |= getAliases(X86::XMM0);
816	Regs |= getAliases(X86::XMM1);
817	}
818
819	void getGPRegs(BitVector &Regs, bool IncludeAlias) const override {
820	if (IncludeAlias) {
821	Regs |= getAliases(X86::RAX);
822	Regs |= getAliases(X86::RBX);
823	Regs |= getAliases(X86::RBP);
824	Regs |= getAliases(X86::RSI);
825	Regs |= getAliases(X86::RDI);
826	Regs |= getAliases(X86::RDX);
827	Regs |= getAliases(X86::RCX);
828	Regs |= getAliases(X86::R8);
829	Regs |= getAliases(X86::R9);
830	Regs |= getAliases(X86::R10);
831	Regs |= getAliases(X86::R11);
832	Regs |= getAliases(X86::R12);
833	Regs |= getAliases(X86::R13);
834	Regs |= getAliases(X86::R14);
835	Regs |= getAliases(X86::R15);
836	return;
837	}
838	Regs.set(X86::RAX);
839	Regs.set(X86::RBX);
840	Regs.set(X86::RBP);
841	Regs.set(X86::RSI);
842	Regs.set(X86::RDI);
843	Regs.set(X86::RDX);
844	Regs.set(X86::RCX);
845	Regs.set(X86::R8);
846	Regs.set(X86::R9);
847	Regs.set(X86::R10);
848	Regs.set(X86::R11);
849	Regs.set(X86::R12);
850	Regs.set(X86::R13);
851	Regs.set(X86::R14);
852	Regs.set(X86::R15);
853	}
854
855	void getClassicGPRegs(BitVector &Regs) const override {
856	Regs |= getAliases(X86::RAX);
857	Regs |= getAliases(X86::RBX);
858	Regs |= getAliases(X86::RBP);
859	Regs |= getAliases(X86::RSI);
860	Regs |= getAliases(X86::RDI);
861	Regs |= getAliases(X86::RDX);
862	Regs |= getAliases(X86::RCX);
863	}
864
865	void getRepRegs(BitVector &Regs) const override {
866	Regs |= getAliases(X86::RCX);
867	}
868
869	MCPhysReg getAliasSized(MCPhysReg Reg, uint8_t Size) const override {
870	Reg = getX86SubSuperRegister(Reg, Size: Size * 8);
871	assert((Reg != X86::NoRegister) && "Invalid register");
872	return Reg;
873	}
874
875	bool isUpper8BitReg(MCPhysReg Reg) const override {
876	switch (Reg) {
877	case X86::AH:
878	case X86::BH:
879	case X86::CH:
880	case X86::DH:
881	return true;
882	default:
883	return false;
884	}
885	}
886
887	bool cannotUseREX(const MCInst &Inst) const override {
888	switch (Inst.getOpcode()) {
889	case X86::MOV8mr_NOREX:
890	case X86::MOV8rm_NOREX:
891	case X86::MOV8rr_NOREX:
892	case X86::MOVSX32rm8_NOREX:
893	case X86::MOVSX32rr8_NOREX:
894	case X86::MOVZX32rm8_NOREX:
895	case X86::MOVZX32rr8_NOREX:
896	case X86::MOV8mr:
897	case X86::MOV8rm:
898	case X86::MOV8rr:
899	case X86::MOVSX32rm8:
900	case X86::MOVSX32rr8:
901	case X86::MOVZX32rm8:
902	case X86::MOVZX32rr8:
903	case X86::TEST8ri:
904	for (const MCOperand &Operand : MCPlus::primeOperands(Inst)) {
905	if (!Operand.isReg())
906	continue;
907	if (isUpper8BitReg(Reg: Operand.getReg()))
908	return true;
909	}
910	[[fallthrough]];
911	default:
912	return false;
913	}
914	}
915
916	static uint8_t getMemDataSize(const MCInst &Inst, int MemOpNo) {
917	using namespace llvm::X86;
918	int OpType = getOperandType(Inst.getOpcode(), MemOpNo);
919	return getMemOperandSize(OpType) / 8;
920	}
921
922	/// Classifying a stack access as *not* "SIMPLE" here means we don't know how
923	/// to change this instruction memory access. It will disable any changes to
924	/// the stack layout, so we can't do the most aggressive form of shrink
925	/// wrapping. We must do so in a way that keeps the original stack layout.
926	/// Otherwise you need to adjust the offset of all instructions accessing the
927	/// stack: we can't do that anymore because there is one instruction that is
928	/// not simple. There are other implications as well. We have heuristics to
929	/// detect when a register is callee-saved and thus eligible for shrink
930	/// wrapping. If you are restoring a register using a non-simple stack access,
931	/// then it is classified as NOT callee-saved, and it disables shrink wrapping
932	/// for *that* register (but not for others).
933	///
934	/// Classifying a stack access as "size 0" or detecting an indexed memory
935	/// access (to address a vector, for example) here means we know there is a
936	/// stack access, but we can't quite understand how wide is the access in
937	/// bytes. This is very serious because we can't understand how memory
938	/// accesses alias with each other for this function. This will essentially
939	/// disable not only shrink wrapping but all frame analysis, it will fail it
940	/// as "we don't understand this function and we give up on it".
941	bool isStackAccess(const MCInst &Inst, bool &IsLoad, bool &IsStore,
942	bool &IsStoreFromReg, MCPhysReg &Reg, int32_t &SrcImm,
943	uint16_t &StackPtrReg, int64_t &StackOffset, uint8_t &Size,
944	bool &IsSimple, bool &IsIndexed) const override {
945	// Detect simple push/pop cases first
946	if (int Sz = getPushSize(Inst)) {
947	IsLoad = false;
948	IsStore = true;
949	IsStoreFromReg = true;
950	StackPtrReg = X86::RSP;
951	StackOffset = -Sz;
952	Size = Sz;
953	IsSimple = true;
954	if (Inst.getOperand(i: 0).isImm())
955	SrcImm = Inst.getOperand(i: 0).getImm();
956	else if (Inst.getOperand(i: 0).isReg())
957	Reg = Inst.getOperand(i: 0).getReg();
958	else
959	IsSimple = false;
960
961	return true;
962	}
963	if (int Sz = getPopSize(Inst)) {
964	IsLoad = true;
965	IsStore = false;
966	if (Inst.getNumOperands() == 0 || !Inst.getOperand(i: 0).isReg()) {
967	IsSimple = false;
968	} else {
969	Reg = Inst.getOperand(i: 0).getReg();
970	IsSimple = true;
971	}
972	StackPtrReg = X86::RSP;
973	StackOffset = 0;
974	Size = Sz;
975	return true;
976	}
977
978	struct InstInfo {
979	// Size in bytes that Inst loads from memory.
980	uint8_t DataSize;
981	bool IsLoad;
982	bool IsStore;
983	bool StoreFromReg;
984	bool Simple;
985	};
986
987	InstInfo I;
988	int MemOpNo = getMemoryOperandNo(Inst);
989	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
990	// If it is not dealing with a memory operand, we discard it
991	if (MemOpNo == -1 || MCII.isCall())
992	return false;
993
994	switch (Inst.getOpcode()) {
995	default: {
996	bool IsLoad = MCII.mayLoad();
997	bool IsStore = MCII.mayStore();
998	// Is it LEA? (deals with memory but is not loading nor storing)
999	if (!IsLoad && !IsStore) {
1000	I = {.DataSize: 0, .IsLoad: IsLoad, .IsStore: IsStore, .StoreFromReg: false, .Simple: false};
1001	break;
1002	}
1003	uint8_t Sz = getMemDataSize(Inst, MemOpNo);
1004	I = {.DataSize: Sz, .IsLoad: IsLoad, .IsStore: IsStore, .StoreFromReg: false, .Simple: false};
1005	break;
1006	}
1007	// Report simple stack accesses
1008	case X86::MOV8rm: I = {.DataSize: 1, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
1009	case X86::MOV16rm: I = {.DataSize: 2, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
1010	case X86::MOV32rm: I = {.DataSize: 4, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
1011	case X86::MOV64rm: I = {.DataSize: 8, .IsLoad: true, .IsStore: false, .StoreFromReg: false, .Simple: true}; break;
1012	case X86::MOV8mr: I = {.DataSize: 1, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
1013	case X86::MOV16mr: I = {.DataSize: 2, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
1014	case X86::MOV32mr: I = {.DataSize: 4, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
1015	case X86::MOV64mr: I = {.DataSize: 8, .IsLoad: false, .IsStore: true, .StoreFromReg: true, .Simple: true}; break;
1016	case X86::MOV8mi: I = {.DataSize: 1, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
1017	case X86::MOV16mi: I = {.DataSize: 2, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
1018	case X86::MOV32mi: I = {.DataSize: 4, .IsLoad: false, .IsStore: true, .StoreFromReg: false, .Simple: true}; break;
1019	} // end switch (Inst.getOpcode())
1020
1021	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1022	if (!MO) {
1023	LLVM_DEBUG(dbgs() << "Evaluate failed on ");
1024	LLVM_DEBUG(Inst.dump());
1025	return false;
1026	}
1027
1028	// Make sure it's a stack access
1029	if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP)
1030	return false;
1031
1032	IsLoad = I.IsLoad;
1033	IsStore = I.IsStore;
1034	IsStoreFromReg = I.StoreFromReg;
1035	Size = I.DataSize;
1036	IsSimple = I.Simple;
1037	StackPtrReg = MO->BaseRegNum;
1038	StackOffset = MO->DispImm;
1039	IsIndexed =
1040	MO->IndexRegNum != X86::NoRegister || MO->SegRegNum != X86::NoRegister;
1041
1042	if (!I.Simple)
1043	return true;
1044
1045	// Retrieve related register in simple MOV from/to stack operations.
1046	unsigned MemOpOffset = static_cast<unsigned>(MemOpNo);
1047	if (I.IsLoad) {
1048	MCOperand RegOpnd = Inst.getOperand(i: 0);
1049	assert(RegOpnd.isReg() && "unexpected destination operand");
1050	Reg = RegOpnd.getReg();
1051	} else if (I.IsStore) {
1052	MCOperand SrcOpnd =
1053	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg + 1);
1054	if (I.StoreFromReg) {
1055	assert(SrcOpnd.isReg() && "unexpected source operand");
1056	Reg = SrcOpnd.getReg();
1057	} else {
1058	assert(SrcOpnd.isImm() && "unexpected source operand");
1059	SrcImm = SrcOpnd.getImm();
1060	}
1061	}
1062
1063	return true;
1064	}
1065
1066	void changeToPushOrPop(MCInst &Inst) const override {
1067	assert(!isPush(Inst) && !isPop(Inst));
1068
1069	struct InstInfo {
1070	// Size in bytes that Inst loads from memory.
1071	uint8_t DataSize;
1072	bool IsLoad;
1073	bool StoreFromReg;
1074	};
1075
1076	InstInfo I;
1077	switch (Inst.getOpcode()) {
1078	default: {
1079	llvm_unreachable("Unhandled opcode");
1080	return;
1081	}
1082	case X86::MOV16rm: I = {.DataSize: 2, .IsLoad: true, .StoreFromReg: false}; break;
1083	case X86::MOV32rm: I = {.DataSize: 4, .IsLoad: true, .StoreFromReg: false}; break;
1084	case X86::MOV64rm: I = {.DataSize: 8, .IsLoad: true, .StoreFromReg: false}; break;
1085	case X86::MOV16mr: I = {.DataSize: 2, .IsLoad: false, .StoreFromReg: true}; break;
1086	case X86::MOV32mr: I = {.DataSize: 4, .IsLoad: false, .StoreFromReg: true}; break;
1087	case X86::MOV64mr: I = {.DataSize: 8, .IsLoad: false, .StoreFromReg: true}; break;
1088	case X86::MOV16mi: I = {.DataSize: 2, .IsLoad: false, .StoreFromReg: false}; break;
1089	case X86::MOV32mi: I = {.DataSize: 4, .IsLoad: false, .StoreFromReg: false}; break;
1090	} // end switch (Inst.getOpcode())
1091
1092	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1093	if (!MO) {
1094	llvm_unreachable("Evaluate failed");
1095	return;
1096	}
1097	// Make sure it's a stack access
1098	if (MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::RSP) {
1099	llvm_unreachable("Not a stack access");
1100	return;
1101	}
1102
1103	unsigned MemOpOffset = getMemoryOperandNo(Inst);
1104	unsigned NewOpcode = 0;
1105	if (I.IsLoad) {
1106	switch (I.DataSize) {
1107	case 2: NewOpcode = X86::POP16r; break;
1108	case 4: NewOpcode = X86::POP32r; break;
1109	case 8: NewOpcode = X86::POP64r; break;
1110	default:
1111	llvm_unreachable("Unexpected size");
1112	}
1113	unsigned RegOpndNum = Inst.getOperand(i: 0).getReg();
1114	Inst.clear();
1115	Inst.setOpcode(NewOpcode);
1116	Inst.addOperand(Op: MCOperand::createReg(Reg: RegOpndNum));
1117	} else {
1118	MCOperand SrcOpnd =
1119	Inst.getOperand(i: MemOpOffset + X86::AddrSegmentReg + 1);
1120	if (I.StoreFromReg) {
1121	switch (I.DataSize) {
1122	case 2: NewOpcode = X86::PUSH16r; break;
1123	case 4: NewOpcode = X86::PUSH32r; break;
1124	case 8: NewOpcode = X86::PUSH64r; break;
1125	default:
1126	llvm_unreachable("Unexpected size");
1127	}
1128	assert(SrcOpnd.isReg() && "Unexpected source operand");
1129	unsigned RegOpndNum = SrcOpnd.getReg();
1130	Inst.clear();
1131	Inst.setOpcode(NewOpcode);
1132	Inst.addOperand(Op: MCOperand::createReg(Reg: RegOpndNum));
1133	} else {
1134	switch (I.DataSize) {
1135	case 2: NewOpcode = X86::PUSH16i8; break;
1136	case 4: NewOpcode = X86::PUSH32i8; break;
1137	case 8: NewOpcode = X86::PUSH64i32; break;
1138	default:
1139	llvm_unreachable("Unexpected size");
1140	}
1141	assert(SrcOpnd.isImm() && "Unexpected source operand");
1142	int64_t SrcImm = SrcOpnd.getImm();
1143	Inst.clear();
1144	Inst.setOpcode(NewOpcode);
1145	Inst.addOperand(Op: MCOperand::createImm(Val: SrcImm));
1146	}
1147	}
1148	}
1149
1150	bool isStackAdjustment(const MCInst &Inst) const override {
1151	switch (Inst.getOpcode()) {
1152	default:
1153	return false;
1154	case X86::SUB64ri32:
1155	case X86::SUB64ri8:
1156	case X86::ADD64ri32:
1157	case X86::ADD64ri8:
1158	case X86::LEA64r:
1159	break;
1160	}
1161
1162	return any_of(defOperands(Inst), [](const MCOperand &Op) {
1163	return Op.isReg() && Op.getReg() == X86::RSP;
1164	});
1165	}
1166
1167	bool
1168	evaluateStackOffsetExpr(const MCInst &Inst, int64_t &Output,
1169	std::pair<MCPhysReg, int64_t> Input1,
1170	std::pair<MCPhysReg, int64_t> Input2) const override {
1171
1172	auto getOperandVal = [&](MCPhysReg Reg) -> ErrorOr<int64_t> {
1173	if (Reg == Input1.first)
1174	return Input1.second;
1175	if (Reg == Input2.first)
1176	return Input2.second;
1177	return make_error_code(errc::result_out_of_range);
1178	};
1179
1180	switch (Inst.getOpcode()) {
1181	default:
1182	return false;
1183
1184	case X86::SUB64ri32:
1185	case X86::SUB64ri8:
1186	if (!Inst.getOperand(i: 2).isImm())
1187	return false;
1188	if (ErrorOr<int64_t> InputVal =
1189	getOperandVal(Inst.getOperand(1).getReg()))
1190	Output = *InputVal - Inst.getOperand(i: 2).getImm();
1191	else
1192	return false;
1193	break;
1194	case X86::ADD64ri32:
1195	case X86::ADD64ri8:
1196	if (!Inst.getOperand(i: 2).isImm())
1197	return false;
1198	if (ErrorOr<int64_t> InputVal =
1199	getOperandVal(Inst.getOperand(1).getReg()))
1200	Output = *InputVal + Inst.getOperand(i: 2).getImm();
1201	else
1202	return false;
1203	break;
1204	case X86::ADD64i32:
1205	if (!Inst.getOperand(i: 0).isImm())
1206	return false;
1207	if (ErrorOr<int64_t> InputVal = getOperandVal(X86::RAX))
1208	Output = *InputVal + Inst.getOperand(i: 0).getImm();
1209	else
1210	return false;
1211	break;
1212
1213	case X86::LEA64r: {
1214	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Inst);
1215	if (!MO)
1216	return false;
1217
1218	if (MO->BaseRegNum == X86::NoRegister ||
1219	MO->IndexRegNum != X86::NoRegister ||
1220	MO->SegRegNum != X86::NoRegister || MO->DispExpr)
1221	return false;
1222
1223	if (ErrorOr<int64_t> InputVal = getOperandVal(MO->BaseRegNum))
1224	Output = *InputVal + MO->DispImm;
1225	else
1226	return false;
1227
1228	break;
1229	}
1230	}
1231	return true;
1232	}
1233
1234	bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,
1235	MCPhysReg &To) const override {
1236	switch (Inst.getOpcode()) {
1237	default:
1238	return false;
1239	case X86::LEAVE:
1240	case X86::LEAVE64:
1241	To = getStackPointer();
1242	From = getFramePointer();
1243	return true;
1244	case X86::MOV64rr:
1245	To = Inst.getOperand(i: 0).getReg();
1246	From = Inst.getOperand(i: 1).getReg();
1247	return true;
1248	}
1249	}
1250
1251	MCPhysReg getStackPointer() const override { return X86::RSP; }
1252	MCPhysReg getFramePointer() const override { return X86::RBP; }
1253	MCPhysReg getFlagsReg() const override { return X86::EFLAGS; }
1254
1255	bool escapesVariable(const MCInst &Inst,
1256	bool HasFramePointer) const override {
1257	int MemOpNo = getMemoryOperandNo(Inst);
1258	const MCInstrDesc &MCII = Info->get(Opcode: Inst.getOpcode());
1259	const unsigned NumDefs = MCII.getNumDefs();
1260	static BitVector SPBPAliases(BitVector(getAliases(X86::RSP)) |=
1261	getAliases(X86::RBP));
1262	static BitVector SPAliases(getAliases(X86::RSP));
1263
1264	// FIXME: PUSH can be technically a leak, but let's ignore this for now
1265	// because a lot of harmless prologue code will spill SP to the stack.
1266	// Unless push is clearly pushing an object address to the stack as
1267	// demonstrated by having a MemOp.
1268	bool IsPush = isPush(Inst);
1269	if (IsPush && MemOpNo == -1)
1270	return false;
1271
1272	// We use this to detect LEA (has memop but does not access mem)
1273	bool AccessMem = MCII.mayLoad() || MCII.mayStore();
1274	bool DoesLeak = false;
1275	for (int I = 0, E = MCPlus::getNumPrimeOperands(Inst); I != E; ++I) {
1276	// Ignore if SP/BP is used to dereference memory -- that's fine
1277	if (MemOpNo != -1 && !IsPush && AccessMem && I >= MemOpNo &&
1278	I <= MemOpNo + 5)
1279	continue;
1280	// Ignore if someone is writing to SP/BP
1281	if (I < static_cast<int>(NumDefs))
1282	continue;
1283
1284	const MCOperand &Operand = Inst.getOperand(i: I);
1285	if (HasFramePointer && Operand.isReg() && SPBPAliases[Operand.getReg()]) {
1286	DoesLeak = true;
1287	break;
1288	}
1289	if (!HasFramePointer && Operand.isReg() && SPAliases[Operand.getReg()]) {
1290	DoesLeak = true;
1291	break;
1292	}
1293	}
1294
1295	// If potential leak, check if it is not just writing to itself/sp/bp
1296	if (DoesLeak) {
1297	DoesLeak = !any_of(defOperands(Inst), [&](const MCOperand &Operand) {
1298	assert(Operand.isReg());
1299	MCPhysReg Reg = Operand.getReg();
1300	return HasFramePointer ? SPBPAliases[Reg] : SPAliases[Reg];
1301	});
1302	}
1303	return DoesLeak;
1304	}
1305
1306	bool addToImm(MCInst &Inst, int64_t &Amt, MCContext *Ctx) const override {
1307	unsigned ImmOpNo = -1U;
1308	int MemOpNo = getMemoryOperandNo(Inst);
1309	if (MemOpNo != -1)
1310	ImmOpNo = MemOpNo + X86::AddrDisp;
1311	else
1312	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
1313	++Index)
1314	if (Inst.getOperand(i: Index).isImm())
1315	ImmOpNo = Index;
1316	if (ImmOpNo == -1U)
1317	return false;
1318
1319	MCOperand &Operand = Inst.getOperand(i: ImmOpNo);
1320	Amt += Operand.getImm();
1321	Operand.setImm(Amt);
1322	// Check for the need for relaxation
1323	if (int64_t(Amt) == int64_t(int8_t(Amt)))
1324	return true;
1325
1326	// Relax instruction
1327	switch (Inst.getOpcode()) {
1328	case X86::SUB64ri8:
1329	Inst.setOpcode(X86::SUB64ri32);
1330	break;
1331	case X86::ADD64ri8:
1332	Inst.setOpcode(X86::ADD64ri32);
1333	break;
1334	default:
1335	// No need for relaxation
1336	break;
1337	}
1338	return true;
1339	}
1340
1341	/// TODO: this implementation currently works for the most common opcodes that
1342	/// load from memory. It can be extended to work with memory store opcodes as
1343	/// well as more memory load opcodes.
1344	bool replaceMemOperandWithImm(MCInst &Inst, StringRef ConstantData,
1345	uint64_t Offset) const override {
1346	enum CheckSignExt : uint8_t {
1347	NOCHECK = 0,
1348	CHECK8,
1349	CHECK32,
1350	};
1351
1352	using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
1353	struct InstInfo {
1354	// Size in bytes that Inst loads from memory.
1355	uint8_t DataSize;
1356
1357	// True when the target operand has to be duplicated because the opcode
1358	// expects a LHS operand.
1359	bool HasLHS;
1360
1361	// List of checks and corresponding opcodes to be used. We try to use the
1362	// smallest possible immediate value when various sizes are available,
1363	// hence we may need to check whether a larger constant fits in a smaller
1364	// immediate.
1365	CheckList Checks;
1366	};
1367
1368	InstInfo I;
1369
1370	switch (Inst.getOpcode()) {
1371	default: {
1372	switch (getPopSize(Inst)) {
1373	case 2: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
1374	case 4: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
1375	case 8: I = {8, false, {{CHECK32, X86::MOV64ri32},
1376	{NOCHECK, X86::MOV64rm}}}; break;
1377	default: return false;
1378	}
1379	break;
1380	}
1381
1382	// MOV
1383	case X86::MOV8rm: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
1384	case X86::MOV16rm: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
1385	case X86::MOV32rm: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
1386	case X86::MOV64rm: I = {8, false, {{CHECK32, X86::MOV64ri32},
1387	{NOCHECK, X86::MOV64rm}}}; break;
1388
1389	// MOVZX
1390	case X86::MOVZX16rm8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
1391	case X86::MOVZX32rm8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
1392	case X86::MOVZX32rm16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
1393
1394	// CMP
1395	case X86::CMP8rm: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
1396	case X86::CMP16rm: I = {2, false, {{CHECK8, X86::CMP16ri8},
1397	{NOCHECK, X86::CMP16ri}}}; break;
1398	case X86::CMP32rm: I = {4, false, {{CHECK8, X86::CMP32ri8},
1399	{NOCHECK, X86::CMP32ri}}}; break;
1400	case X86::CMP64rm: I = {8, false, {{CHECK8, X86::CMP64ri8},
1401	{CHECK32, X86::CMP64ri32},
1402	{NOCHECK, X86::CMP64rm}}}; break;
1403
1404	// TEST
1405	case X86::TEST8mr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
1406	case X86::TEST16mr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
1407	case X86::TEST32mr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
1408	case X86::TEST64mr: I = {8, false, {{CHECK32, X86::TEST64ri32},
1409	{NOCHECK, X86::TEST64mr}}}; break;
1410
1411	// ADD
1412	case X86::ADD8rm: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
1413	case X86::ADD16rm: I = {2, true, {{CHECK8, X86::ADD16ri8},
1414	{NOCHECK, X86::ADD16ri}}}; break;
1415	case X86::ADD32rm: I = {4, true, {{CHECK8, X86::ADD32ri8},
1416	{NOCHECK, X86::ADD32ri}}}; break;
1417	case X86::ADD64rm: I = {8, true, {{CHECK8, X86::ADD64ri8},
1418	{CHECK32, X86::ADD64ri32},
1419	{NOCHECK, X86::ADD64rm}}}; break;
1420
1421	// SUB
1422	case X86::SUB8rm: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
1423	case X86::SUB16rm: I = {2, true, {{CHECK8, X86::SUB16ri8},
1424	{NOCHECK, X86::SUB16ri}}}; break;
1425	case X86::SUB32rm: I = {4, true, {{CHECK8, X86::SUB32ri8},
1426	{NOCHECK, X86::SUB32ri}}}; break;
1427	case X86::SUB64rm: I = {8, true, {{CHECK8, X86::SUB64ri8},
1428	{CHECK32, X86::SUB64ri32},
1429	{NOCHECK, X86::SUB64rm}}}; break;
1430
1431	// AND
1432	case X86::AND8rm: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
1433	case X86::AND16rm: I = {2, true, {{CHECK8, X86::AND16ri8},
1434	{NOCHECK, X86::AND16ri}}}; break;
1435	case X86::AND32rm: I = {4, true, {{CHECK8, X86::AND32ri8},
1436	{NOCHECK, X86::AND32ri}}}; break;
1437	case X86::AND64rm: I = {8, true, {{CHECK8, X86::AND64ri8},
1438	{CHECK32, X86::AND64ri32},
1439	{NOCHECK, X86::AND64rm}}}; break;
1440
1441	// OR
1442	case X86::OR8rm: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
1443	case X86::OR16rm: I = {2, true, {{CHECK8, X86::OR16ri8},
1444	{NOCHECK, X86::OR16ri}}}; break;
1445	case X86::OR32rm: I = {4, true, {{CHECK8, X86::OR32ri8},
1446	{NOCHECK, X86::OR32ri}}}; break;
1447	case X86::OR64rm: I = {8, true, {{CHECK8, X86::OR64ri8},
1448	{CHECK32, X86::OR64ri32},
1449	{NOCHECK, X86::OR64rm}}}; break;
1450
1451	// XOR
1452	case X86::XOR8rm: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
1453	case X86::XOR16rm: I = {2, true, {{CHECK8, X86::XOR16ri8},
1454	{NOCHECK, X86::XOR16ri}}}; break;
1455	case X86::XOR32rm: I = {4, true, {{CHECK8, X86::XOR32ri8},
1456	{NOCHECK, X86::XOR32ri}}}; break;
1457	case X86::XOR64rm: I = {8, true, {{CHECK8, X86::XOR64ri8},
1458	{CHECK32, X86::XOR64ri32},
1459	{NOCHECK, X86::XOR64rm}}}; break;
1460	}
1461
1462	// Compute the immediate value.
1463	assert(Offset + I.DataSize <= ConstantData.size() &&
1464	"invalid offset for given constant data");
1465	int64_t ImmVal =
1466	DataExtractor(ConstantData, true, 8).getSigned(offset_ptr: &Offset, size: I.DataSize);
1467
1468	// Compute the new opcode.
1469	unsigned NewOpcode = 0;
1470	for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
1471	NewOpcode = Check.second;
1472	if (Check.first == NOCHECK)
1473	break;
1474	if (Check.first == CHECK8 && isInt<8>(ImmVal))
1475	break;
1476	if (Check.first == CHECK32 && isInt<32>(ImmVal))
1477	break;
1478	}
1479	if (NewOpcode == Inst.getOpcode())
1480	return false;
1481
1482	// Modify the instruction.
1483	MCOperand ImmOp = MCOperand::createImm(Val: ImmVal);
1484	uint32_t TargetOpNum = 0;
1485	// Test instruction does not follow the regular pattern of putting the
1486	// memory reference of a load (5 MCOperands) last in the list of operands.
1487	// Since it is not modifying the register operand, it is not treated as
1488	// a destination operand and it is not the first operand as it is in the
1489	// other instructions we treat here.
1490	if (NewOpcode == X86::TEST8ri || NewOpcode == X86::TEST16ri ||
1491	NewOpcode == X86::TEST32ri || NewOpcode == X86::TEST64ri32)
1492	TargetOpNum = getMemoryOperandNo(Inst) + X86::AddrNumOperands;
1493
1494	MCOperand TargetOp = Inst.getOperand(i: TargetOpNum);
1495	Inst.clear();
1496	Inst.setOpcode(NewOpcode);
1497	Inst.addOperand(Op: TargetOp);
1498	if (I.HasLHS)
1499	Inst.addOperand(Op: TargetOp);
1500	Inst.addOperand(Op: ImmOp);
1501
1502	return true;
1503	}
1504
1505	/// TODO: this implementation currently works for the most common opcodes that
1506	/// load from memory. It can be extended to work with memory store opcodes as
1507	/// well as more memory load opcodes.
1508	bool replaceMemOperandWithReg(MCInst &Inst, MCPhysReg RegNum) const override {
1509	unsigned NewOpcode;
1510
1511	switch (Inst.getOpcode()) {
1512	default: {
1513	switch (getPopSize(Inst)) {
1514	case 2: NewOpcode = X86::MOV16rr; break;
1515	case 4: NewOpcode = X86::MOV32rr; break;
1516	case 8: NewOpcode = X86::MOV64rr; break;
1517	default: return false;
1518	}
1519	break;
1520	}
1521
1522	// MOV
1523	case X86::MOV8rm: NewOpcode = X86::MOV8rr; break;
1524	case X86::MOV16rm: NewOpcode = X86::MOV16rr; break;
1525	case X86::MOV32rm: NewOpcode = X86::MOV32rr; break;
1526	case X86::MOV64rm: NewOpcode = X86::MOV64rr; break;
1527	}
1528
1529	// Modify the instruction.
1530	MCOperand RegOp = MCOperand::createReg(Reg: RegNum);
1531	MCOperand TargetOp = Inst.getOperand(i: 0);
1532	Inst.clear();
1533	Inst.setOpcode(NewOpcode);
1534	Inst.addOperand(Op: TargetOp);
1535	Inst.addOperand(Op: RegOp);
1536
1537	return true;
1538	}
1539
1540	bool isRedundantMove(const MCInst &Inst) const override {
1541	switch (Inst.getOpcode()) {
1542	default:
1543	return false;
1544
1545	// MOV
1546	case X86::MOV8rr:
1547	case X86::MOV16rr:
1548	case X86::MOV32rr:
1549	case X86::MOV64rr:
1550	break;
1551	}
1552
1553	assert(Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg());
1554	return Inst.getOperand(i: 0).getReg() == Inst.getOperand(i: 1).getReg();
1555	}
1556
1557	bool requiresAlignedAddress(const MCInst &Inst) const override {
1558	const MCInstrDesc &Desc = Info->get(Opcode: Inst.getOpcode());
1559	for (unsigned int I = 0; I < Desc.getNumOperands(); ++I) {
1560	const MCOperandInfo &Op = Desc.operands()[I];
1561	if (Op.OperandType != MCOI::OPERAND_REGISTER)
1562	continue;
1563	if (Op.RegClass == X86::VR128RegClassID)
1564	return true;
1565	}
1566	return false;
1567	}
1568
1569	bool convertJmpToTailCall(MCInst &Inst) override {
1570	if (isTailCall(Inst))
1571	return false;
1572
1573	int NewOpcode;
1574	switch (Inst.getOpcode()) {
1575	default:
1576	return false;
1577	case X86::JMP_1:
1578	case X86::JMP_2:
1579	case X86::JMP_4:
1580	NewOpcode = X86::JMP_4;
1581	break;
1582	case X86::JMP16m:
1583	case X86::JMP32m:
1584	case X86::JMP64m:
1585	NewOpcode = X86::JMP32m;
1586	break;
1587	case X86::JMP16r:
1588	case X86::JMP32r:
1589	case X86::JMP64r:
1590	NewOpcode = X86::JMP32r;
1591	break;
1592	}
1593
1594	Inst.setOpcode(NewOpcode);
1595	setTailCall(Inst);
1596	return true;
1597	}
1598
1599	bool convertTailCallToJmp(MCInst &Inst) override {
1600	int NewOpcode;
1601	switch (Inst.getOpcode()) {
1602	default:
1603	return false;
1604	case X86::JMP_4:
1605	NewOpcode = X86::JMP_1;
1606	break;
1607	case X86::JMP32m:
1608	NewOpcode = X86::JMP64m;
1609	break;
1610	case X86::JMP32r:
1611	NewOpcode = X86::JMP64r;
1612	break;
1613	}
1614
1615	Inst.setOpcode(NewOpcode);
1616	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1617	clearOffset(Inst);
1618	return true;
1619	}
1620
1621	bool convertTailCallToCall(MCInst &Inst) override {
1622	int NewOpcode;
1623	switch (Inst.getOpcode()) {
1624	default:
1625	return false;
1626	case X86::JMP_4:
1627	NewOpcode = X86::CALL64pcrel32;
1628	break;
1629	case X86::JMP32m:
1630	NewOpcode = X86::CALL64m;
1631	break;
1632	case X86::JMP32r:
1633	NewOpcode = X86::CALL64r;
1634	break;
1635	}
1636
1637	Inst.setOpcode(NewOpcode);
1638	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1639	return true;
1640	}
1641
1642	bool convertCallToIndirectCall(MCInst &Inst, const MCSymbol *TargetLocation,
1643	MCContext *Ctx) override {
1644	assert((Inst.getOpcode() == X86::CALL64pcrel32 ||
1645	(Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst))) &&
1646	"64-bit direct (tail) call instruction expected");
1647	const auto NewOpcode =
1648	(Inst.getOpcode() == X86::CALL64pcrel32) ? X86::CALL64m : X86::JMP32m;
1649	Inst.setOpcode(NewOpcode);
1650
1651	// Replace the first operand and preserve auxiliary operands of
1652	// the instruction.
1653	Inst.erase(I: Inst.begin());
1654	Inst.insert(Inst.begin(),
1655	MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
1656	Inst.insert(I: Inst.begin(),
1657	Op: MCOperand::createExpr( // Displacement
1658	Val: MCSymbolRefExpr::create(Symbol: TargetLocation,
1659	Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
1660	Inst.insert(Inst.begin(),
1661	MCOperand::createReg(X86::NoRegister)); // IndexReg
1662	Inst.insert(I: Inst.begin(),
1663	Op: MCOperand::createImm(Val: 1)); // ScaleAmt
1664	Inst.insert(Inst.begin(),
1665	MCOperand::createReg(X86::RIP)); // BaseReg
1666
1667	return true;
1668	}
1669
1670	void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {
1671	bool IsTailCall = isTailCall(Inst);
1672	if (IsTailCall)
1673	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1674	if (Inst.getOpcode() == X86::CALL64m ||
1675	(Inst.getOpcode() == X86::JMP32m && IsTailCall)) {
1676	Inst.setOpcode(X86::MOV64rm);
1677	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1678	return;
1679	}
1680	if (Inst.getOpcode() == X86::CALL64r ||
1681	(Inst.getOpcode() == X86::JMP32r && IsTailCall)) {
1682	Inst.setOpcode(X86::MOV64rr);
1683	Inst.insert(I: Inst.begin(), Op: MCOperand::createReg(Reg));
1684	return;
1685	}
1686	LLVM_DEBUG(Inst.dump());
1687	llvm_unreachable("not implemented");
1688	}
1689
1690	bool shortenInstruction(MCInst &Inst,
1691	const MCSubtargetInfo &STI) const override {
1692	unsigned OldOpcode = Inst.getOpcode();
1693	unsigned NewOpcode = OldOpcode;
1694
1695	int MemOpNo = getMemoryOperandNo(Inst);
1696
1697	// Check and remove redundant Address-Size override prefix.
1698	if (opts::X86StripRedundantAddressSize) {
1699	uint64_t TSFlags = Info->get(Opcode: OldOpcode).TSFlags;
1700	unsigned Flags = Inst.getFlags();
1701
1702	if (!X86_MC::needsAddressSizeOverride(MI: Inst, STI, MemoryOperand: MemOpNo, TSFlags) &&
1703	Flags & X86::IP_HAS_AD_SIZE)
1704	Inst.setFlags(Flags ^ X86::IP_HAS_AD_SIZE);
1705	}
1706
1707	// Check and remove EIZ/RIZ. These cases represent ambiguous cases where
1708	// SIB byte is present, but no index is used and modrm alone should have
1709	// been enough. Converting to NoRegister effectively removes the SIB byte.
1710	if (MemOpNo >= 0) {
1711	MCOperand &IndexOp =
1712	Inst.getOperand(i: static_cast<unsigned>(MemOpNo) + X86::AddrIndexReg);
1713	if (IndexOp.getReg() == X86::EIZ || IndexOp.getReg() == X86::RIZ)
1714	IndexOp = MCOperand::createReg(X86::NoRegister);
1715	}
1716
1717	if (isBranch(Inst)) {
1718	NewOpcode = getShortBranchOpcode(Opcode: OldOpcode);
1719	} else if (OldOpcode == X86::MOV64ri) {
1720	if (Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1721	const int64_t Imm =
1722	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1723	if (int64_t(Imm) == int64_t(int32_t(Imm)))
1724	NewOpcode = X86::MOV64ri32;
1725	}
1726	} else {
1727	// If it's arithmetic instruction check if signed operand fits in 1 byte.
1728	const unsigned ShortOpcode = X86::getOpcodeForShortImmediateForm(Opcode: OldOpcode);
1729	if (ShortOpcode != OldOpcode &&
1730	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).isImm()) {
1731	int64_t Imm =
1732	Inst.getOperand(i: MCPlus::getNumPrimeOperands(Inst) - 1).getImm();
1733	if (int64_t(Imm) == int64_t(int8_t(Imm)))
1734	NewOpcode = ShortOpcode;
1735	}
1736	}
1737
1738	if (NewOpcode == OldOpcode)
1739	return false;
1740
1741	Inst.setOpcode(NewOpcode);
1742	return true;
1743	}
1744
1745	bool
1746	convertMoveToConditionalMove(MCInst &Inst, unsigned CC, bool AllowStackMemOp,
1747	bool AllowBasePtrStackMemOp) const override {
1748	// - Register-register moves are OK
1749	// - Stores are filtered out by opcode (no store CMOV)
1750	// - Non-stack loads are prohibited (generally unsafe)
1751	// - Stack loads are OK if AllowStackMemOp is true
1752	// - Stack loads with RBP are OK if AllowBasePtrStackMemOp is true
1753	if (mayLoad(Inst)) {
1754	// If stack memory operands are not allowed, no loads are allowed
1755	if (!AllowStackMemOp)
1756	return false;
1757
1758	// If stack memory operands are allowed, check if it's a load from stack
1759	bool IsLoad, IsStore, IsStoreFromReg, IsSimple, IsIndexed;
1760	MCPhysReg Reg;
1761	int32_t SrcImm;
1762	uint16_t StackPtrReg;
1763	int64_t StackOffset;
1764	uint8_t Size;
1765	bool IsStackAccess =
1766	isStackAccess(Inst, IsLoad, IsStore, IsStoreFromReg, Reg, SrcImm,
1767	StackPtrReg, StackOffset, Size, IsSimple, IsIndexed);
1768	// Prohibit non-stack-based loads
1769	if (!IsStackAccess)
1770	return false;
1771	// If stack memory operands are allowed, check if it's RBP-based
1772	if (!AllowBasePtrStackMemOp &&
1773	RegInfo->isSubRegisterEq(X86::RBP, StackPtrReg))
1774	return false;
1775	}
1776
1777	unsigned NewOpcode = 0;
1778	switch (Inst.getOpcode()) {
1779	case X86::MOV16rr:
1780	NewOpcode = X86::CMOV16rr;
1781	break;
1782	case X86::MOV16rm:
1783	NewOpcode = X86::CMOV16rm;
1784	break;
1785	case X86::MOV32rr:
1786	NewOpcode = X86::CMOV32rr;
1787	break;
1788	case X86::MOV32rm:
1789	NewOpcode = X86::CMOV32rm;
1790	break;
1791	case X86::MOV64rr:
1792	NewOpcode = X86::CMOV64rr;
1793	break;
1794	case X86::MOV64rm:
1795	NewOpcode = X86::CMOV64rm;
1796	break;
1797	default:
1798	return false;
1799	}
1800	Inst.setOpcode(NewOpcode);
1801	// Insert CC at the end of prime operands, before annotations
1802	Inst.insert(I: Inst.begin() + MCPlus::getNumPrimeOperands(Inst),
1803	Op: MCOperand::createImm(Val: CC));
1804	// CMOV is a 3-operand MCInst, so duplicate the destination as src1
1805	Inst.insert(I: Inst.begin(), Op: Inst.getOperand(i: 0));
1806	return true;
1807	}
1808
1809	bool lowerTailCall(MCInst &Inst) override {
1810	if (Inst.getOpcode() == X86::JMP_4 && isTailCall(Inst)) {
1811	Inst.setOpcode(X86::JMP_1);
1812	removeAnnotation(Inst, Index: MCPlus::MCAnnotation::kTailCall);
1813	return true;
1814	}
1815	return false;
1816	}
1817
1818	const MCSymbol *getTargetSymbol(const MCInst &Inst,
1819	unsigned OpNum = 0) const override {
1820	if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
1821	return nullptr;
1822
1823	const MCOperand &Op = Inst.getOperand(i: OpNum);
1824	if (!Op.isExpr())
1825	return nullptr;
1826
1827	auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
1828	if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None)
1829	return nullptr;
1830
1831	return &SymExpr->getSymbol();
1832	}
1833
1834	// This is the same as the base class, but since we are overriding one of
1835	// getTargetSymbol's signatures above, we need to override all of them.
1836	const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
1837	return &cast<const MCSymbolRefExpr>(Expr)->getSymbol();
1838	}
1839
1840	bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,
1841	const MCSymbol *&TBB, const MCSymbol *&FBB,
1842	MCInst *&CondBranch,
1843	MCInst *&UncondBranch) const override {
1844	auto I = End;
1845
1846	// Bottom-up analysis
1847	while (I != Begin) {
1848	--I;
1849
1850	// Ignore nops and CFIs
1851	if (isPseudo(Inst: *I))
1852	continue;
1853
1854	// Stop when we find the first non-terminator
1855	if (!isTerminator(Inst: *I))
1856	break;
1857
1858	if (!isBranch(Inst: *I))
1859	break;
1860
1861	// Handle unconditional branches.
1862	if ((I->getOpcode() == X86::JMP_1 || I->getOpcode() == X86::JMP_2 ||
1863	I->getOpcode() == X86::JMP_4) &&
1864	!isTailCall(*I)) {
1865	// If any code was seen after this unconditional branch, we've seen
1866	// unreachable code. Ignore them.
1867	CondBranch = nullptr;
1868	UncondBranch = &*I;
1869	const MCSymbol *Sym = getTargetSymbol(Inst: *I);
1870	assert(Sym != nullptr &&
1871	"Couldn't extract BB symbol from jump operand");
1872	TBB = Sym;
1873	continue;
1874	}
1875
1876	// Handle conditional branches and ignore indirect branches
1877	if (!isUnsupportedBranch(Inst: *I) && getCondCode(Inst: *I) == X86::COND_INVALID) {
1878	// Indirect branch
1879	return false;
1880	}
1881
1882	if (CondBranch == nullptr) {
1883	const MCSymbol *TargetBB = getTargetSymbol(Inst: *I);
1884	if (TargetBB == nullptr) {
1885	// Unrecognized branch target
1886	return false;
1887	}
1888	FBB = TBB;
1889	TBB = TargetBB;
1890	CondBranch = &*I;
1891	continue;
1892	}
1893
1894	llvm_unreachable("multiple conditional branches in one BB");
1895	}
1896	return true;
1897	}
1898
1899	template <typename Itr>
1900	std::pair<IndirectBranchType, MCInst *>
1901	analyzePICJumpTable(Itr II, Itr IE, MCPhysReg R1, MCPhysReg R2) const {
1902	// Analyze PIC-style jump table code template:
1903	//
1904	// lea PIC_JUMP_TABLE(%rip), {%r1|%r2} <- MemLocInstr
1905	// mov ({%r1|%r2}, %index, 4), {%r2|%r1}
1906	// add %r2, %r1
1907	// jmp *%r1
1908	//
1909	// (with any irrelevant instructions in-between)
1910	//
1911	// When we call this helper we've already determined %r1 and %r2, and
1912	// reverse instruction iterator \p II is pointing to the ADD instruction.
1913	//
1914	// PIC jump table looks like following:
1915	//
1916	// JT: ----------
1917	// E1:| L1 - JT |
1918	// |----------|
1919	// E2:| L2 - JT |
1920	// |----------|
1921	// | |
1922	// ......
1923	// En:| Ln - JT |
1924	// ----------
1925	//
1926	// Where L1, L2, ..., Ln represent labels in the function.
1927	//
1928	// The actual relocations in the table will be of the form:
1929	//
1930	// Ln - JT
1931	// = (Ln - En) + (En - JT)
1932	// = R_X86_64_PC32(Ln) + En - JT
1933	// = R_X86_64_PC32(Ln + offsetof(En))
1934	//
1935	LLVM_DEBUG(dbgs() << "Checking for PIC jump table\n");
1936	MCInst *MemLocInstr = nullptr;
1937	const MCInst *MovInstr = nullptr;
1938	while (++II != IE) {
1939	MCInst &Instr = *II;
1940	const MCInstrDesc &InstrDesc = Info->get(Opcode: Instr.getOpcode());
1941	if (!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R1, RI: *RegInfo) &&
1942	!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R2, RI: *RegInfo)) {
1943	// Ignore instructions that don't affect R1, R2 registers.
1944	continue;
1945	}
1946	if (!MovInstr) {
1947	// Expect to see MOV instruction.
1948	if (!isMOVSX64rm32(Inst: Instr)) {
1949	LLVM_DEBUG(dbgs() << "MOV instruction expected.\n");
1950	break;
1951	}
1952
1953	// Check if it's setting %r1 or %r2. In canonical form it sets %r2.
1954	// If it sets %r1 - rename the registers so we have to only check
1955	// a single form.
1956	unsigned MovDestReg = Instr.getOperand(i: 0).getReg();
1957	if (MovDestReg != R2)
1958	std::swap(R1, R2);
1959	if (MovDestReg != R2) {
1960	LLVM_DEBUG(dbgs() << "MOV instruction expected to set %r2\n");
1961	break;
1962	}
1963
1964	// Verify operands for MOV.
1965	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
1966	if (!MO)
1967	break;
1968	if (MO->BaseRegNum != R1 || MO->ScaleImm != 4 ||
1969	MO->IndexRegNum == X86::NoRegister || MO->DispImm != 0 ||
1970	MO->SegRegNum != X86::NoRegister)
1971	break;
1972	MovInstr = &Instr;
1973	} else {
1974	if (!InstrDesc.hasDefOfPhysReg(MI: Instr, Reg: R1, RI: *RegInfo))
1975	continue;
1976	if (!isLEA64r(Inst: Instr)) {
1977	LLVM_DEBUG(dbgs() << "LEA instruction expected\n");
1978	break;
1979	}
1980	if (Instr.getOperand(i: 0).getReg() != R1) {
1981	LLVM_DEBUG(dbgs() << "LEA instruction expected to set %r1\n");
1982	break;
1983	}
1984
1985	// Verify operands for LEA.
1986	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(Instr);
1987	if (!MO)
1988	break;
1989	if (MO->BaseRegNum != RegInfo->getProgramCounter() ||
1990	MO->IndexRegNum != X86::NoRegister ||
1991	MO->SegRegNum != X86::NoRegister || MO->DispExpr == nullptr)
1992	break;
1993	MemLocInstr = &Instr;
1994	break;
1995	}
1996	}
1997
1998	if (!MemLocInstr)
1999	return std::make_pair(IndirectBranchType::UNKNOWN, nullptr);
2000
2001	LLVM_DEBUG(dbgs() << "checking potential PIC jump table\n");
2002	return std::make_pair(IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE,
2003	MemLocInstr);
2004	}
2005
2006	IndirectBranchType analyzeIndirectBranch(
2007	MCInst &Instruction, InstructionIterator Begin, InstructionIterator End,
2008	const unsigned PtrSize, MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,
2009	unsigned &IndexRegNumOut, int64_t &DispValueOut,
2010	const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut) const override {
2011	// Try to find a (base) memory location from where the address for
2012	// the indirect branch is loaded. For X86-64 the memory will be specified
2013	// in the following format:
2014	//
2015	// {%rip}/{%basereg} + Imm + IndexReg * Scale
2016	//
2017	// We are interested in the cases where Scale == sizeof(uintptr_t) and
2018	// the contents of the memory are presumably an array of pointers to code.
2019	//
2020	// Normal jump table:
2021	//
2022	// jmp *(JUMP_TABLE, %index, Scale) <- MemLocInstr
2023	//
2024	// or
2025	//
2026	// mov (JUMP_TABLE, %index, Scale), %r1 <- MemLocInstr
2027	// ...
2028	// jmp %r1
2029	//
2030	// We handle PIC-style jump tables separately.
2031	//
2032	MemLocInstrOut = nullptr;
2033	BaseRegNumOut = X86::NoRegister;
2034	IndexRegNumOut = X86::NoRegister;
2035	DispValueOut = 0;
2036	DispExprOut = nullptr;
2037
2038	std::reverse_iterator<InstructionIterator> II(End);
2039	std::reverse_iterator<InstructionIterator> IE(Begin);
2040
2041	IndirectBranchType Type = IndirectBranchType::UNKNOWN;
2042
2043	// An instruction referencing memory used by jump instruction (directly or
2044	// via register). This location could be an array of function pointers
2045	// in case of indirect tail call, or a jump table.
2046	MCInst *MemLocInstr = nullptr;
2047
2048	if (MCPlus::getNumPrimeOperands(Inst: Instruction) == 1) {
2049	// If the indirect jump is on register - try to detect if the
2050	// register value is loaded from a memory location.
2051	assert(Instruction.getOperand(0).isReg() && "register operand expected");
2052	const unsigned R1 = Instruction.getOperand(i: 0).getReg();
2053	// Check if one of the previous instructions defines the jump-on register.
2054	for (auto PrevII = II; PrevII != IE; ++PrevII) {
2055	MCInst &PrevInstr = *PrevII;
2056	const MCInstrDesc &PrevInstrDesc = Info->get(Opcode: PrevInstr.getOpcode());
2057
2058	if (!PrevInstrDesc.hasDefOfPhysReg(MI: PrevInstr, Reg: R1, RI: *RegInfo))
2059	continue;
2060
2061	if (isMoveMem2Reg(Inst: PrevInstr)) {
2062	MemLocInstr = &PrevInstr;
2063	break;
2064	}
2065	if (isADD64rr(Inst: PrevInstr)) {
2066	unsigned R2 = PrevInstr.getOperand(i: 2).getReg();
2067	if (R1 == R2)
2068	return IndirectBranchType::UNKNOWN;
2069	std::tie(Type, MemLocInstr) = analyzePICJumpTable(PrevII, IE, R1, R2);
2070	break;
2071	}
2072	return IndirectBranchType::UNKNOWN;
2073	}
2074	if (!MemLocInstr) {
2075	// No definition seen for the register in this function so far. Could be
2076	// an input parameter - which means it is an external code reference.
2077	// It also could be that the definition happens to be in the code that
2078	// we haven't processed yet. Since we have to be conservative, return
2079	// as UNKNOWN case.
2080	return IndirectBranchType::UNKNOWN;
2081	}
2082	} else {
2083	MemLocInstr = &Instruction;
2084	}
2085
2086	const MCRegister RIPRegister = RegInfo->getProgramCounter();
2087
2088	// Analyze the memory location.
2089	std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(*MemLocInstr);
2090	if (!MO)
2091	return IndirectBranchType::UNKNOWN;
2092
2093	BaseRegNumOut = MO->BaseRegNum;
2094	IndexRegNumOut = MO->IndexRegNum;
2095	DispValueOut = MO->DispImm;
2096	DispExprOut = MO->DispExpr;
2097
2098	if ((MO->BaseRegNum != X86::NoRegister && MO->BaseRegNum != RIPRegister) ||
2099	MO->SegRegNum != X86::NoRegister)
2100	return IndirectBranchType::UNKNOWN;
2101
2102	if (MemLocInstr == &Instruction &&
2103	(!MO->ScaleImm || MO->IndexRegNum == X86::NoRegister)) {
2104	MemLocInstrOut = MemLocInstr;
2105	return IndirectBranchType::POSSIBLE_FIXED_BRANCH;
2106	}
2107
2108	if (Type == IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2109	(MO->ScaleImm != 1 || MO->BaseRegNum != RIPRegister))
2110	return IndirectBranchType::UNKNOWN;
2111
2112	if (Type != IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE &&
2113	MO->ScaleImm != PtrSize)
2114	return IndirectBranchType::UNKNOWN;
2115
2116	MemLocInstrOut = MemLocInstr;
2117
2118	return Type;
2119	}
2120
2121	/// Analyze a callsite to see if it could be a virtual method call. This only
2122	/// checks to see if the overall pattern is satisfied, it does not guarantee
2123	/// that the callsite is a true virtual method call.
2124	/// The format of virtual method calls that are recognized is one of the
2125	/// following:
2126	///
2127	/// Form 1: (found in debug code)
2128	/// add METHOD_OFFSET, %VtableReg
2129	/// mov (%VtableReg), %MethodReg
2130	/// ...
2131	/// call or jmp *%MethodReg
2132	///
2133	/// Form 2:
2134	/// mov METHOD_OFFSET(%VtableReg), %MethodReg
2135	/// ...
2136	/// call or jmp *%MethodReg
2137	///
2138	/// Form 3:
2139	/// ...
2140	/// call or jmp *METHOD_OFFSET(%VtableReg)
2141	///
2142	bool analyzeVirtualMethodCall(InstructionIterator ForwardBegin,
2143	InstructionIterator ForwardEnd,
2144	std::vector<MCInst *> &MethodFetchInsns,
2145	unsigned &VtableRegNum, unsigned &MethodRegNum,
2146	uint64_t &MethodOffset) const override {
2147	VtableRegNum = X86::NoRegister;
2148	MethodRegNum = X86::NoRegister;
2149	MethodOffset = 0;
2150
2151	std::reverse_iterator<InstructionIterator> Itr(ForwardEnd);
2152	std::reverse_iterator<InstructionIterator> End(ForwardBegin);
2153
2154	MCInst &CallInst = *Itr++;
2155	assert(isIndirectBranch(CallInst) || isCall(CallInst));
2156
2157	// The call can just be jmp offset(reg)
2158	if (std::optional<X86MemOperand> MO = evaluateX86MemoryOperand(CallInst)) {
2159	if (!MO->DispExpr && MO->BaseRegNum != X86::RIP &&
2160	MO->BaseRegNum != X86::RBP && MO->BaseRegNum != X86::NoRegister) {
2161	MethodRegNum = MO->BaseRegNum;
2162	if (MO->ScaleImm == 1 && MO->IndexRegNum == X86::NoRegister &&
2163	MO->SegRegNum == X86::NoRegister) {
2164	VtableRegNum = MethodRegNum;
2165	MethodOffset = MO->DispImm;
2166	MethodFetchInsns.push_back(&CallInst);
2167	return true;
2168	}
2169	}
2170	return false;
2171	}
2172	if (CallInst.getOperand(i: 0).isReg())
2173	MethodRegNum = CallInst.getOperand(i: 0).getReg();
2174	else
2175	return false;
2176
2177	if (MethodRegNum == X86::RIP || MethodRegNum == X86::RBP) {
2178	VtableRegNum = X86::NoRegister;
2179	MethodRegNum = X86::NoRegister;
2180	return false;
2181	}
2182
2183	// find load from vtable, this may or may not include the method offset
2184	while (Itr != End) {
2185	MCInst &CurInst = *Itr++;
2186	const MCInstrDesc &Desc = Info->get(Opcode: CurInst.getOpcode());
2187	if (Desc.hasDefOfPhysReg(MI: CurInst, Reg: MethodRegNum, RI: *RegInfo)) {
2188	if (!mayLoad(Inst: CurInst))
2189	return false;
2190	if (std::optional<X86MemOperand> MO =
2191	evaluateX86MemoryOperand(CurInst)) {
2192	if (!MO->DispExpr && MO->ScaleImm == 1 &&
2193	MO->BaseRegNum != X86::RIP && MO->BaseRegNum != X86::RBP &&
2194	MO->BaseRegNum != X86::NoRegister &&
2195	MO->IndexRegNum == X86::NoRegister &&
2196	MO->SegRegNum == X86::NoRegister) {
2197	VtableRegNum = MO->BaseRegNum;
2198	MethodOffset = MO->DispImm;
2199	MethodFetchInsns.push_back(&CurInst);
2200	if (MethodOffset != 0)
2201	return true;
2202	break;
2203	}
2204	}
2205	return false;
2206	}
2207	}
2208
2209	if (!VtableRegNum)
2210	return false;
2211
2212	// look for any adds affecting the method register.
2213	while (Itr != End) {
2214	MCInst &CurInst = *Itr++;
2215	const MCInstrDesc &Desc = Info->get(Opcode: CurInst.getOpcode());
2216	if (Desc.hasDefOfPhysReg(MI: CurInst, Reg: VtableRegNum, RI: *RegInfo)) {
2217	if (isADDri(Inst: CurInst)) {
2218	assert(!MethodOffset);
2219	MethodOffset = CurInst.getOperand(i: 2).getImm();
2220	MethodFetchInsns.insert(MethodFetchInsns.begin(), &CurInst);
2221	break;
2222	}
2223	}
2224	}
2225
2226	return true;
2227	}
2228
2229	void createStackPointerIncrement(MCInst &Inst, int Size,
2230	bool NoFlagsClobber) const override {
2231	if (NoFlagsClobber) {
2232	Inst.setOpcode(X86::LEA64r);
2233	Inst.clear();
2234	Inst.addOperand(MCOperand::createReg(X86::RSP));
2235	Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg
2236	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2237	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2238	Inst.addOperand(Op: MCOperand::createImm(Val: -Size)); // Displacement
2239	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2240	return;
2241	}
2242	Inst.setOpcode(X86::SUB64ri8);
2243	Inst.clear();
2244	Inst.addOperand(MCOperand::createReg(X86::RSP));
2245	Inst.addOperand(MCOperand::createReg(X86::RSP));
2246	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2247	}
2248
2249	void createStackPointerDecrement(MCInst &Inst, int Size,
2250	bool NoFlagsClobber) const override {
2251	if (NoFlagsClobber) {
2252	Inst.setOpcode(X86::LEA64r);
2253	Inst.clear();
2254	Inst.addOperand(MCOperand::createReg(X86::RSP));
2255	Inst.addOperand(MCOperand::createReg(X86::RSP)); // BaseReg
2256	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2257	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2258	Inst.addOperand(Op: MCOperand::createImm(Val: Size)); // Displacement
2259	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2260	return;
2261	}
2262	Inst.setOpcode(X86::ADD64ri8);
2263	Inst.clear();
2264	Inst.addOperand(MCOperand::createReg(X86::RSP));
2265	Inst.addOperand(MCOperand::createReg(X86::RSP));
2266	Inst.addOperand(Op: MCOperand::createImm(Val: Size));
2267	}
2268
2269	void createSaveToStack(MCInst &Inst, const MCPhysReg &StackReg, int Offset,
2270	const MCPhysReg &SrcReg, int Size) const override {
2271	unsigned NewOpcode;
2272	switch (Size) {
2273	default:
2274	llvm_unreachable("Invalid operand size");
2275	return;
2276	case 2: NewOpcode = X86::MOV16mr; break;
2277	case 4: NewOpcode = X86::MOV32mr; break;
2278	case 8: NewOpcode = X86::MOV64mr; break;
2279	}
2280	Inst.setOpcode(NewOpcode);
2281	Inst.clear();
2282	Inst.addOperand(Op: MCOperand::createReg(Reg: StackReg)); // BaseReg
2283	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
2284	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
2285	Inst.addOperand(Op: MCOperand::createImm(Val: Offset)); // Displacement
2286	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
2287	Inst.addOperand(Op: MCOperand::createReg(Reg: SrcReg));
2288	}
2289
2290	void createRestoreFromStack(MCInst &Inst, const MCPhysReg &StackReg,
2291	int Offset, const MCPhysReg &DstReg,
2292	int Size) const override {
2293	return createLoad(Inst, StackReg, /*Scale=*/1, /*IndexReg=*/X86::NoRegister,
2294	Offset, nullptr, /*AddrSegmentReg=*/X86::NoRegister,
2295	DstReg, Size);
2296	}
2297
2298	void createLoad(MCInst &Inst, const MCPhysReg &BaseReg, int64_t Scale,
2299	const MCPhysReg &IndexReg, int64_t Offset,
2300	const MCExpr *OffsetExpr, const MCPhysReg &AddrSegmentReg,
2301	const MCPhysReg &DstReg, int Size) const override {
2302	unsigned NewOpcode;
2303	switch (Size) {
2304	default:
2305	llvm_unreachable("Invalid operand size");
2306	return;
2307	case 2: NewOpcode = X86::MOV16rm; break;
2308	case 4: NewOpcode = X86::MOV32rm; break;
2309	case 8: NewOpcode = X86::MOV64rm; break;
2310	}
2311	Inst.setOpcode(NewOpcode);
2312	Inst.clear();
2313	Inst.addOperand(Op: MCOperand::createReg(Reg: DstReg));
2314	Inst.addOperand(Op: MCOperand::createReg(Reg: BaseReg));
2315	Inst.addOperand(Op: MCOperand::createImm(Val: Scale));
2316	Inst.addOperand(Op: MCOperand::createReg(Reg: IndexReg));
2317	if (OffsetExpr)
2318	Inst.addOperand(Op: MCOperand::createExpr(Val: OffsetExpr)); // Displacement
2319	else
2320	Inst.addOperand(Op: MCOperand::createImm(Val: Offset)); // Displacement
2321	Inst.addOperand(Op: MCOperand::createReg(Reg: AddrSegmentReg)); // AddrSegmentReg
2322	}
2323
2324	InstructionListType createLoadImmediate(const MCPhysReg Dest,
2325	uint64_t Imm) const override {
2326	InstructionListType Insts;
2327	Insts.emplace_back();
2328	Insts.back().setOpcode(X86::MOV64ri32);
2329	Insts.back().clear();
2330	Insts.back().addOperand(Op: MCOperand::createReg(Reg: Dest));
2331	Insts.back().addOperand(Op: MCOperand::createImm(Val: Imm));
2332	return Insts;
2333	}
2334
2335	void createIJmp32Frag(SmallVectorImpl<MCInst> &Insts,
2336	const MCOperand &BaseReg, const MCOperand &Scale,
2337	const MCOperand &IndexReg, const MCOperand &Offset,
2338	const MCOperand &TmpReg) const override {
2339	// The code fragment we emit here is:
2340	//
2341	// mov32 (%base, %index, scale), %tmpreg
2342	// ijmp *(%tmpreg)
2343	//
2344	MCInst IJmp;
2345	IJmp.setOpcode(X86::JMP64r);
2346	IJmp.addOperand(Op: TmpReg);
2347
2348	MCInst Load;
2349	Load.setOpcode(X86::MOV32rm);
2350	Load.addOperand(Op: TmpReg);
2351	Load.addOperand(Op: BaseReg);
2352	Load.addOperand(Op: Scale);
2353	Load.addOperand(Op: IndexReg);
2354	Load.addOperand(Op: Offset);
2355	Load.addOperand(MCOperand::createReg(X86::NoRegister));
2356
2357	Insts.push_back(Load);
2358	Insts.push_back(IJmp);
2359	}
2360
2361	void createNoop(MCInst &Inst) const override {
2362	Inst.setOpcode(X86::NOOP);
2363	Inst.clear();
2364	}
2365
2366	void createReturn(MCInst &Inst) const override {
2367	Inst.setOpcode(X86::RET64);
2368	Inst.clear();
2369	}
2370
2371	InstructionListType createInlineMemcpy(bool ReturnEnd) const override {
2372	InstructionListType Code;
2373	if (ReturnEnd)
2374	Code.emplace_back(MCInstBuilder(X86::LEA64r)
2375	.addReg(X86::RAX)
2376	.addReg(X86::RDI)
2377	.addImm(1)
2378	.addReg(X86::RDX)
2379	.addImm(0)
2380	.addReg(X86::NoRegister));
2381	else
2382	Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2383	.addReg(X86::RAX)
2384	.addReg(X86::RDI));
2385
2386	Code.emplace_back(MCInstBuilder(X86::MOV32rr)
2387	.addReg(X86::ECX)
2388	.addReg(X86::EDX));
2389	Code.emplace_back(MCInstBuilder(X86::REP_MOVSB_64));
2390
2391	return Code;
2392	}
2393
2394	InstructionListType createOneByteMemcpy() const override {
2395	InstructionListType Code;
2396	Code.emplace_back(MCInstBuilder(X86::MOV8rm)
2397	.addReg(X86::CL)
2398	.addReg(X86::RSI)
2399	.addImm(0)
2400	.addReg(X86::NoRegister)
2401	.addImm(0)
2402	.addReg(X86::NoRegister));
2403	Code.emplace_back(MCInstBuilder(X86::MOV8mr)
2404	.addReg(X86::RDI)
2405	.addImm(0)
2406	.addReg(X86::NoRegister)
2407	.addImm(0)
2408	.addReg(X86::NoRegister)
2409	.addReg(X86::CL));
2410	Code.emplace_back(MCInstBuilder(X86::MOV64rr)
2411	.addReg(X86::RAX)
2412	.addReg(X86::RDI));
2413	return Code;
2414	}
2415
2416	InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,
2417	const MCSymbol *Target,
2418	MCContext *Ctx) const override {
2419	InstructionListType Code;
2420	Code.emplace_back(MCInstBuilder(X86::CMP64ri8)
2421	.addReg(RegNo)
2422	.addImm(Imm));
2423	Code.emplace_back(MCInstBuilder(X86::JCC_1)
2424	.addExpr(MCSymbolRefExpr::create(
2425	Target, MCSymbolRefExpr::VK_None, *Ctx))
2426	.addImm(X86::COND_E));
2427	return Code;
2428	}
2429
2430	std::optional<Relocation>
2431	createRelocation(const MCFixup &Fixup,
2432	const MCAsmBackend &MAB) const override {
2433	const MCFixupKindInfo &FKI = MAB.getFixupKindInfo(Kind: Fixup.getKind());
2434
2435	assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");
2436	const uint64_t RelOffset = Fixup.getOffset();
2437
2438	uint64_t RelType;
2439	if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
2440	switch (FKI.TargetSize) {
2441	default:
2442	return std::nullopt;
2443	case 8: RelType = ELF::R_X86_64_PC8; break;
2444	case 16: RelType = ELF::R_X86_64_PC16; break;
2445	case 32: RelType = ELF::R_X86_64_PC32; break;
2446	case 64: RelType = ELF::R_X86_64_PC64; break;
2447	}
2448	} else {
2449	switch (FKI.TargetSize) {
2450	default:
2451	return std::nullopt;
2452	case 8: RelType = ELF::R_X86_64_8; break;
2453	case 16: RelType = ELF::R_X86_64_16; break;
2454	case 32: RelType = ELF::R_X86_64_32; break;
2455	case 64: RelType = ELF::R_X86_64_64; break;
2456	}
2457	}
2458
2459	auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);
2460
2461	return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});
2462	}
2463
2464	bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,
2465	int64_t Addend, MCContext *Ctx, int64_t &Value,
2466	uint64_t RelType) const override {
2467	unsigned ImmOpNo = -1U;
2468
2469	for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);
2470	++Index) {
2471	if (Inst.getOperand(i: Index).isImm()) {
2472	ImmOpNo = Index;
2473	// TODO: this is a bit hacky. It finds the correct operand by
2474	// searching for a specific immediate value. If no value is
2475	// provided it defaults to the last immediate operand found.
2476	// This could lead to unexpected results if the instruction
2477	// has more than one immediate with the same value.
2478	if (Inst.getOperand(i: ImmOpNo).getImm() == Value)
2479	break;
2480	}
2481	}
2482
2483	if (ImmOpNo == -1U)
2484	return false;
2485
2486	Value = Inst.getOperand(i: ImmOpNo).getImm();
2487
2488	setOperandToSymbolRef(Inst, OpNum: ImmOpNo, Symbol, Addend, Ctx, RelType);
2489
2490	return true;
2491	}
2492
2493	bool replaceRegWithImm(MCInst &Inst, unsigned Register,
2494	int64_t Imm) const override {
2495
2496	enum CheckSignExt : uint8_t {
2497	NOCHECK = 0,
2498	CHECK8,
2499	CHECK32,
2500	};
2501
2502	using CheckList = std::vector<std::pair<CheckSignExt, unsigned>>;
2503	struct InstInfo {
2504	// Size in bytes that Inst loads from memory.
2505	uint8_t DataSize;
2506
2507	// True when the target operand has to be duplicated because the opcode
2508	// expects a LHS operand.
2509	bool HasLHS;
2510
2511	// List of checks and corresponding opcodes to be used. We try to use the
2512	// smallest possible immediate value when various sizes are available,
2513	// hence we may need to check whether a larger constant fits in a smaller
2514	// immediate.
2515	CheckList Checks;
2516	};
2517
2518	InstInfo I;
2519
2520	switch (Inst.getOpcode()) {
2521	default: {
2522	switch (getPushSize(Inst)) {
2523
2524	case 2: I = {2, false, {{CHECK8, X86::PUSH16i8}, {NOCHECK, X86::PUSH16i}}}; break;
2525	case 4: I = {4, false, {{CHECK8, X86::PUSH32i8}, {NOCHECK, X86::PUSH32i}}}; break;
2526	case 8: I = {8, false, {{CHECK8, X86::PUSH64i8},
2527	{CHECK32, X86::PUSH64i32},
2528	{NOCHECK, Inst.getOpcode()}}}; break;
2529	default: return false;
2530	}
2531	break;
2532	}
2533
2534	// MOV
2535	case X86::MOV8rr: I = {1, false, {{NOCHECK, X86::MOV8ri}}}; break;
2536	case X86::MOV16rr: I = {2, false, {{NOCHECK, X86::MOV16ri}}}; break;
2537	case X86::MOV32rr: I = {4, false, {{NOCHECK, X86::MOV32ri}}}; break;
2538	case X86::MOV64rr: I = {8, false, {{CHECK32, X86::MOV64ri32},
2539	{NOCHECK, X86::MOV64ri}}}; break;
2540
2541	case X86::MOV8mr: I = {1, false, {{NOCHECK, X86::MOV8mi}}}; break;
2542	case X86::MOV16mr: I = {2, false, {{NOCHECK, X86::MOV16mi}}}; break;
2543	case X86::MOV32mr: I = {4, false, {{NOCHECK, X86::MOV32mi}}}; break;
2544	case X86::MOV64mr: I = {8, false, {{CHECK32, X86::MOV64mi32},
2545	{NOCHECK, X86::MOV64mr}}}; break;
2546
2547	// MOVZX
2548	case X86::MOVZX16rr8: I = {1, false, {{NOCHECK, X86::MOV16ri}}}; break;
2549	case X86::MOVZX32rr8: I = {1, false, {{NOCHECK, X86::MOV32ri}}}; break;
2550	case X86::MOVZX32rr16: I = {2, false, {{NOCHECK, X86::MOV32ri}}}; break;
2551
2552	// CMP
2553	case X86::CMP8rr: I = {1, false, {{NOCHECK, X86::CMP8ri}}}; break;
2554	case X86::CMP16rr: I = {2, false, {{CHECK8, X86::CMP16ri8},
2555	{NOCHECK, X86::CMP16ri}}}; break;
2556	case X86::CMP32rr: I = {4, false, {{CHECK8, X86::CMP32ri8},
2557	{NOCHECK, X86::CMP32ri}}}; break;
2558	case X86::CMP64rr: I = {8, false, {{CHECK8, X86::CMP64ri8},
2559	{CHECK32, X86::CMP64ri32},
2560	{NOCHECK, X86::CMP64rr}}}; break;
2561
2562	// TEST
2563	case X86::TEST8rr: I = {1, false, {{NOCHECK, X86::TEST8ri}}}; break;
2564	case X86::TEST16rr: I = {2, false, {{NOCHECK, X86::TEST16ri}}}; break;
2565	case X86::TEST32rr: I = {4, false, {{NOCHECK, X86::TEST32ri}}}; break;
2566	case X86::TEST64rr: I = {8, false, {{CHECK32, X86::TEST64ri32},
2567	{NOCHECK, X86::TEST64rr}}}; break;
2568
2569	// ADD
2570	case X86::ADD8rr: I = {1, true, {{NOCHECK, X86::ADD8ri}}}; break;
2571	case X86::ADD16rr: I = {2, true, {{CHECK8, X86::ADD16ri8},
2572	{NOCHECK, X86::ADD16ri}}}; break;
2573	case X86::ADD32rr: I = {4, true, {{CHECK8, X86::ADD32ri8},
2574	{NOCHECK, X86::ADD32ri}}}; break;
2575	case X86::ADD64rr: I = {8, true, {{CHECK8, X86::ADD64ri8},
2576	{CHECK32, X86::ADD64ri32},
2577	{NOCHECK, X86::ADD64rr}}}; break;
2578
2579	// SUB
2580	case X86::SUB8rr: I = {1, true, {{NOCHECK, X86::SUB8ri}}}; break;
2581	case X86::SUB16rr: I = {2, true, {{CHECK8, X86::SUB16ri8},
2582	{NOCHECK, X86::SUB16ri}}}; break;
2583	case X86::SUB32rr: I = {4, true, {{CHECK8, X86::SUB32ri8},
2584	{NOCHECK, X86::SUB32ri}}}; break;
2585	case X86::SUB64rr: I = {8, true, {{CHECK8, X86::SUB64ri8},
2586	{CHECK32, X86::SUB64ri32},
2587	{NOCHECK, X86::SUB64rr}}}; break;
2588
2589	// AND
2590	case X86::AND8rr: I = {1, true, {{NOCHECK, X86::AND8ri}}}; break;
2591	case X86::AND16rr: I = {2, true, {{CHECK8, X86::AND16ri8},
2592	{NOCHECK, X86::AND16ri}}}; break;
2593	case X86::AND32rr: I = {4, true, {{CHECK8, X86::AND32ri8},
2594	{NOCHECK, X86::AND32ri}}}; break;
2595	case X86::AND64rr: I = {8, true, {{CHECK8, X86::AND64ri8},
2596	{CHECK32, X86::AND64ri32},
2597	{NOCHECK, X86::AND64rr}}}; break;
2598
2599	// OR
2600	case X86::OR8rr: I = {1, true, {{NOCHECK, X86::OR8ri}}}; break;
2601	case X86::OR16rr: I = {2, true, {{CHECK8, X86::OR16ri8},
2602	{NOCHECK, X86::OR16ri}}}; break;
2603	case X86::OR32rr: I = {4, true, {{CHECK8, X86::OR32ri8},
2604	{NOCHECK, X86::OR32ri}}}; break;
2605	case X86::OR64rr: I = {8, true, {{CHECK8, X86::OR64ri8},
2606	{CHECK32, X86::OR64ri32},
2607	{NOCHECK, X86::OR64rr}}}; break;
2608
2609	// XOR
2610	case X86::XOR8rr: I = {1, true, {{NOCHECK, X86::XOR8ri}}}; break;
2611	case X86::XOR16rr: I = {2, true, {{CHECK8, X86::XOR16ri8},
2612	{NOCHECK, X86::XOR16ri}}}; break;
2613	case X86::XOR32rr: I = {4, true, {{CHECK8, X86::XOR32ri8},
2614	{NOCHECK, X86::XOR32ri}}}; break;
2615	case X86::XOR64rr: I = {8, true, {{CHECK8, X86::XOR64ri8},
2616	{CHECK32, X86::XOR64ri32},
2617	{NOCHECK, X86::XOR64rr}}}; break;
2618	}
2619
2620	// Compute the new opcode.
2621	unsigned NewOpcode = 0;
2622	for (const std::pair<CheckSignExt, unsigned> &Check : I.Checks) {
2623	NewOpcode = Check.second;
2624	if (Check.first == NOCHECK)
2625	break;
2626	if (Check.first == CHECK8 && isInt<8>(Imm))
2627	break;
2628	if (Check.first == CHECK32 && isInt<32>(Imm))
2629	break;
2630	}
2631	if (NewOpcode == Inst.getOpcode())
2632	return false;
2633
2634	const MCInstrDesc &InstDesc = Info->get(Opcode: Inst.getOpcode());
2635
2636	unsigned NumFound = 0;
2637	for (unsigned Index = InstDesc.getNumDefs() + (I.HasLHS ? 1 : 0),
2638	E = InstDesc.getNumOperands();
2639	Index != E; ++Index)
2640	if (Inst.getOperand(i: Index).isReg() &&
2641	Inst.getOperand(i: Index).getReg() == Register)
2642	NumFound++;
2643
2644	if (NumFound != 1)
2645	return false;
2646
2647	MCOperand TargetOp = Inst.getOperand(i: 0);
2648	Inst.clear();
2649	Inst.setOpcode(NewOpcode);
2650	Inst.addOperand(Op: TargetOp);
2651	if (I.HasLHS)
2652	Inst.addOperand(Op: TargetOp);
2653	Inst.addOperand(Op: MCOperand::createImm(Val: Imm));
2654
2655	return true;
2656	}
2657
2658	bool replaceRegWithReg(MCInst &Inst, unsigned ToReplace,
2659	unsigned ReplaceWith) const override {
2660
2661	// Get the HasLHS value so that iteration can be done
2662	bool HasLHS;
2663	if (X86::isAND(Inst.getOpcode()) || X86::isADD(Inst.getOpcode()) ||
2664	X86::isSUB(Inst.getOpcode())) {
2665	HasLHS = true;
2666	} else if (isPop(Inst) || isPush(Inst) || X86::isCMP(Inst.getOpcode()) ||
2667	X86::isTEST(Inst.getOpcode())) {
2668	HasLHS = false;
2669	} else {
2670	switch (Inst.getOpcode()) {
2671	case X86::MOV8rr:
2672	case X86::MOV8rm:
2673	case X86::MOV8mr:
2674	case X86::MOV8ri:
2675	case X86::MOV16rr:
2676	case X86::MOV16rm:
2677	case X86::MOV16mr:
2678	case X86::MOV16ri:
2679	case X86::MOV32rr:
2680	case X86::MOV32rm:
2681	case X86::MOV32mr:
2682	case X86::MOV32ri:
2683	case X86::MOV64rr:
2684	case X86::MOV64rm:
2685	case X86::MOV64mr:
2686	case X86::MOV64ri:
2687	case X86::MOVZX16rr8:
2688	case X86::MOVZX32rr8:
2689	case X86::MOVZX32rr16:
2690	case X86::MOVSX32rm8:
2691	case X86::MOVSX32rr8:
2692	case X86::MOVSX64rm32:
2693	case X86::LEA64r:
2694	HasLHS = false;
2695	break;
2696	default:
2697	return false;
2698	}
2699	}
2700
2701	const MCInstrDesc &InstDesc = Info->get(Opcode: Inst.getOpcode());
2702
2703	bool FoundOne = false;
2704
2705	// Iterate only through src operands that arent also dest operands
2706	for (unsigned Index = InstDesc.getNumDefs() + (HasLHS ? 1 : 0),
2707	E = InstDesc.getNumOperands();
2708	Index != E; ++Index) {
2709	BitVector RegAliases = getAliases(Reg: ToReplace, OnlySmaller: true);
2710	if (!Inst.getOperand(i: Index).isReg() ||
2711	!RegAliases.test(Idx: Inst.getOperand(i: Index).getReg()))
2712	continue;
2713	// Resize register if needed
2714	unsigned SizedReplaceWith = getAliasSized(
2715	Reg: ReplaceWith, Size: getRegSize(Reg: Inst.getOperand(i: Index).getReg()));
2716	MCOperand NewOperand = MCOperand::createReg(Reg: SizedReplaceWith);
2717	Inst.getOperand(i: Index) = NewOperand;
2718	FoundOne = true;
2719	}
2720
2721	// Return true if at least one operand was replaced
2722	return FoundOne;
2723	}
2724
2725	void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,
2726	MCContext *Ctx) const override {
2727	Inst.clear();
2728	Inst.setOpcode(X86::JMP_1);
2729	Inst.clear();
2730	Inst.addOperand(Op: MCOperand::createExpr(
2731	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2732	}
2733
2734	void createLongUncondBranch(MCInst &Inst, const MCSymbol *Target,
2735	MCContext *Ctx) const override {
2736	Inst.setOpcode(X86::JMP_4);
2737	Inst.clear();
2738	Inst.addOperand(Op: MCOperand::createExpr(
2739	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2740	}
2741
2742	void createCall(MCInst &Inst, const MCSymbol *Target,
2743	MCContext *Ctx) override {
2744	Inst.setOpcode(X86::CALL64pcrel32);
2745	Inst.clear();
2746	Inst.addOperand(Op: MCOperand::createExpr(
2747	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2748	}
2749
2750	void createTailCall(MCInst &Inst, const MCSymbol *Target,
2751	MCContext *Ctx) override {
2752	return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);
2753	}
2754
2755	void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,
2756	MCContext *Ctx) override {
2757	Seq.clear();
2758	Seq.emplace_back();
2759	createDirectCall(Inst&: Seq.back(), Target, Ctx, /*IsTailCall*/ true);
2760	}
2761
2762	void createTrap(MCInst &Inst) const override {
2763	Inst.clear();
2764	Inst.setOpcode(X86::TRAP);
2765	}
2766
2767	void createCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC,
2768	MCContext *Ctx) const override {
2769	Inst.setOpcode(X86::JCC_1);
2770	Inst.clear();
2771	Inst.addOperand(Op: MCOperand::createExpr(
2772	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2773	Inst.addOperand(Op: MCOperand::createImm(Val: CC));
2774	}
2775
2776	void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC,
2777	MCContext *Ctx) const override {
2778	Inst.setOpcode(X86::JCC_4);
2779	Inst.clear();
2780	Inst.addOperand(Op: MCOperand::createExpr(
2781	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2782	Inst.addOperand(Op: MCOperand::createImm(Val: CC));
2783	}
2784
2785	bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,
2786	MCContext *Ctx) const override {
2787	unsigned InvCC = getInvertedCondCode(CC: getCondCode(Inst));
2788	assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
2789	Inst.getOperand(i: Info->get(Opcode: Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
2790	Inst.getOperand(i: 0) = MCOperand::createExpr(
2791	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
2792	return true;
2793	}
2794
2795	bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
2796	unsigned CC) const override {
2797	if (CC == X86::COND_INVALID)
2798	return false;
2799	Inst.getOperand(i: Info->get(Opcode: Inst.getOpcode()).NumOperands - 1).setImm(CC);
2800	Inst.getOperand(i: 0) = MCOperand::createExpr(
2801	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
2802	return true;
2803	}
2804
2805	unsigned getCanonicalBranchCondCode(unsigned CC) const override {
2806	switch (CC) {
2807	default: return X86::COND_INVALID;
2808
2809	case X86::COND_E: return X86::COND_E;
2810	case X86::COND_NE: return X86::COND_E;
2811
2812	case X86::COND_L: return X86::COND_L;
2813	case X86::COND_GE: return X86::COND_L;
2814
2815	case X86::COND_LE: return X86::COND_G;
2816	case X86::COND_G: return X86::COND_G;
2817
2818	case X86::COND_B: return X86::COND_B;
2819	case X86::COND_AE: return X86::COND_B;
2820
2821	case X86::COND_BE: return X86::COND_A;
2822	case X86::COND_A: return X86::COND_A;
2823
2824	case X86::COND_S: return X86::COND_S;
2825	case X86::COND_NS: return X86::COND_S;
2826
2827	case X86::COND_P: return X86::COND_P;
2828	case X86::COND_NP: return X86::COND_P;
2829
2830	case X86::COND_O: return X86::COND_O;
2831	case X86::COND_NO: return X86::COND_O;
2832	}
2833	}
2834
2835	bool replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,
2836	MCContext *Ctx) const override {
2837	assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
2838	"Invalid instruction");
2839	Inst.getOperand(i: 0) = MCOperand::createExpr(
2840	Val: MCSymbolRefExpr::create(Symbol: TBB, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx));
2841	return true;
2842	}
2843
2844	MCPhysReg getX86R11() const override { return X86::R11; }
2845
2846	unsigned getShortBranchOpcode(unsigned Opcode) const override {
2847	switch (Opcode) {
2848	default:
2849	return Opcode;
2850	case X86::JMP_2:
2851	return X86::JMP_1;
2852	case X86::JMP_4:
2853	return X86::JMP_1;
2854	case X86::JCC_2:
2855	return X86::JCC_1;
2856	case X86::JCC_4:
2857	return X86::JCC_1;
2858	}
2859	}
2860
2861	MCPhysReg getIntArgRegister(unsigned ArgNo) const override {
2862	// FIXME: this should depend on the calling convention.
2863	switch (ArgNo) {
2864	case 0: return X86::RDI;
2865	case 1: return X86::RSI;
2866	case 2: return X86::RDX;
2867	case 3: return X86::RCX;
2868	case 4: return X86::R8;
2869	case 5: return X86::R9;
2870	default: return getNoRegister();
2871	}
2872	}
2873
2874	void createPause(MCInst &Inst) const override {
2875	Inst.clear();
2876	Inst.setOpcode(X86::PAUSE);
2877	}
2878
2879	void createLfence(MCInst &Inst) const override {
2880	Inst.clear();
2881	Inst.setOpcode(X86::LFENCE);
2882	}
2883
2884	void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,
2885	bool IsTailCall) override {
2886	Inst.clear();
2887	Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
2888	Inst.addOperand(Op: MCOperand::createExpr(
2889	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2890	if (IsTailCall)
2891	setTailCall(Inst);
2892	}
2893
2894	void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,
2895	MCContext *Ctx, bool IsTailCall) override {
2896	Seq.clear();
2897	MCInst Inst;
2898	Inst.setOpcode(X86::JMP_1);
2899	Inst.addOperand(Op: MCOperand::createExpr(
2900	Val: MCSymbolRefExpr::create(Symbol: Target, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
2901	if (IsTailCall)
2902	setTailCall(Inst);
2903	Seq.emplace_back(Inst);
2904	}
2905
2906	bool isConditionalMove(const MCInst &Inst) const override {
2907	unsigned OpCode = Inst.getOpcode();
2908	return (OpCode == X86::CMOV16rr || OpCode == X86::CMOV32rr ||
2909	OpCode == X86::CMOV64rr);
2910	}
2911
2912	bool isBranchOnMem(const MCInst &Inst) const override {
2913	unsigned OpCode = Inst.getOpcode();
2914	if (OpCode == X86::CALL64m || (OpCode == X86::JMP32m && isTailCall(Inst)) ||
2915	OpCode == X86::JMP64m)
2916	return true;
2917
2918	return false;
2919	}
2920
2921	bool isBranchOnReg(const MCInst &Inst) const override {
2922	unsigned OpCode = Inst.getOpcode();
2923	if (OpCode == X86::CALL64r || (OpCode == X86::JMP32r && isTailCall(Inst)) ||
2924	OpCode == X86::JMP64r)
2925	return true;
2926
2927	return false;
2928	}
2929
2930	void createPushRegister(MCInst &Inst, MCPhysReg Reg,
2931	unsigned Size) const override {
2932	Inst.clear();
2933	unsigned NewOpcode = 0;
2934	if (Reg == X86::EFLAGS) {
2935	switch (Size) {
2936	case 2: NewOpcode = X86::PUSHF16; break;
2937	case 4: NewOpcode = X86::PUSHF32; break;
2938	case 8: NewOpcode = X86::PUSHF64; break;
2939	default:
2940	llvm_unreachable("Unexpected size");
2941	}
2942	Inst.setOpcode(NewOpcode);
2943	return;
2944	}
2945	switch (Size) {
2946	case 2: NewOpcode = X86::PUSH16r; break;
2947	case 4: NewOpcode = X86::PUSH32r; break;
2948	case 8: NewOpcode = X86::PUSH64r; break;
2949	default:
2950	llvm_unreachable("Unexpected size");
2951	}
2952	Inst.setOpcode(NewOpcode);
2953	Inst.addOperand(Op: MCOperand::createReg(Reg));
2954	}
2955
2956	void createPopRegister(MCInst &Inst, MCPhysReg Reg,
2957	unsigned Size) const override {
2958	Inst.clear();
2959	unsigned NewOpcode = 0;
2960	if (Reg == X86::EFLAGS) {
2961	switch (Size) {
2962	case 2: NewOpcode = X86::POPF16; break;
2963	case 4: NewOpcode = X86::POPF32; break;
2964	case 8: NewOpcode = X86::POPF64; break;
2965	default:
2966	llvm_unreachable("Unexpected size");
2967	}
2968	Inst.setOpcode(NewOpcode);
2969	return;
2970	}
2971	switch (Size) {
2972	case 2: NewOpcode = X86::POP16r; break;
2973	case 4: NewOpcode = X86::POP32r; break;
2974	case 8: NewOpcode = X86::POP64r; break;
2975	default:
2976	llvm_unreachable("Unexpected size");
2977	}
2978	Inst.setOpcode(NewOpcode);
2979	Inst.addOperand(Op: MCOperand::createReg(Reg));
2980	}
2981
2982	void createPushFlags(MCInst &Inst, unsigned Size) const override {
2983	return createPushRegister(Inst, X86::EFLAGS, Size);
2984	}
2985
2986	void createPopFlags(MCInst &Inst, unsigned Size) const override {
2987	return createPopRegister(Inst, X86::EFLAGS, Size);
2988	}
2989
2990	void createAddRegImm(MCInst &Inst, MCPhysReg Reg, int64_t Value,
2991	unsigned Size) const {
2992	unsigned int Opcode;
2993	switch (Size) {
2994	case 1: Opcode = X86::ADD8ri; break;
2995	case 2: Opcode = X86::ADD16ri; break;
2996	case 4: Opcode = X86::ADD32ri; break;
2997	default:
2998	llvm_unreachable("Unexpected size");
2999	}
3000	Inst.setOpcode(Opcode);
3001	Inst.clear();
3002	Inst.addOperand(Op: MCOperand::createReg(Reg));
3003	Inst.addOperand(Op: MCOperand::createReg(Reg));
3004	Inst.addOperand(Op: MCOperand::createImm(Val: Value));
3005	}
3006
3007	void createClearRegWithNoEFlagsUpdate(MCInst &Inst, MCPhysReg Reg,
3008	unsigned Size) const {
3009	unsigned int Opcode;
3010	switch (Size) {
3011	case 1: Opcode = X86::MOV8ri; break;
3012	case 2: Opcode = X86::MOV16ri; break;
3013	case 4: Opcode = X86::MOV32ri; break;
3014	// Writing to a 32-bit register always zeros the upper 32 bits of the
3015	// full-width register
3016	case 8:
3017	Opcode = X86::MOV32ri;
3018	Reg = getAliasSized(Reg, Size: 4);
3019	break;
3020	default:
3021	llvm_unreachable("Unexpected size");
3022	}
3023	Inst.setOpcode(Opcode);
3024	Inst.clear();
3025	Inst.addOperand(Op: MCOperand::createReg(Reg));
3026	Inst.addOperand(Op: MCOperand::createImm(Val: 0));
3027	}
3028
3029	void createX86SaveOVFlagToRegister(MCInst &Inst, MCPhysReg Reg) const {
3030	Inst.setOpcode(X86::SETCCr);
3031	Inst.clear();
3032	Inst.addOperand(Op: MCOperand::createReg(Reg));
3033	Inst.addOperand(Op: MCOperand::createImm(Val: X86::COND_O));
3034	}
3035
3036	void createX86Lahf(MCInst &Inst) const {
3037	Inst.setOpcode(X86::LAHF);
3038	Inst.clear();
3039	}
3040
3041	void createX86Sahf(MCInst &Inst) const {
3042	Inst.setOpcode(X86::SAHF);
3043	Inst.clear();
3044	}
3045
3046	InstructionListType
3047	createInstrIncMemory(const MCSymbol *Target, MCContext *Ctx, bool IsLeaf,
3048	unsigned CodePointerSize) const override {
3049	InstructionListType Instrs(IsLeaf ? 13 : 11);
3050	unsigned int I = 0;
3051
3052	// Don't clobber application red zone (ABI dependent)
3053	if (IsLeaf)
3054	createStackPointerIncrement(Inst&: Instrs[I++], Size: 128,
3055	/*NoFlagsClobber=*/true);
3056
3057	// Performance improvements based on the optimization discussed at
3058	// https://reviews.llvm.org/D6629
3059	// LAHF/SAHF are used instead of PUSHF/POPF
3060	// PUSHF
3061	createPushRegister(Instrs[I++], X86::RAX, 8);
3062	createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3063	createX86Lahf(Inst&: Instrs[I++]);
3064	createPushRegister(Instrs[I++], X86::RAX, 8);
3065	createClearRegWithNoEFlagsUpdate(Instrs[I++], X86::RAX, 8);
3066	createX86SaveOVFlagToRegister(Instrs[I++], X86::AL);
3067	// LOCK INC
3068	InstructionListType IncMem = createIncMemory(Target, Ctx);
3069	assert(IncMem.size() == 1 && "Invalid IncMem size");
3070	std::copy(IncMem.begin(), IncMem.end(), Instrs.begin() + I);
3071	I += IncMem.size();
3072	// POPF
3073	createAddRegImm(Instrs[I++], X86::AL, 127, 1);
3074	createPopRegister(Instrs[I++], X86::RAX, 8);
3075	createX86Sahf(Inst&: Instrs[I++]);
3076	createPopRegister(Instrs[I++], X86::RAX, 8);
3077
3078	if (IsLeaf)
3079	createStackPointerDecrement(Inst&: Instrs[I], Size: 128,
3080	/*NoFlagsClobber=*/true);
3081	return Instrs;
3082	}
3083
3084	void createSwap(MCInst &Inst, MCPhysReg Source, MCPhysReg MemBaseReg,
3085	int64_t Disp) const {
3086	Inst.setOpcode(X86::XCHG64rm);
3087	Inst.clear();
3088	Inst.addOperand(Op: MCOperand::createReg(Reg: Source));
3089	Inst.addOperand(Op: MCOperand::createReg(Reg: Source));
3090	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg)); // BaseReg
3091	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3092	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3093	Inst.addOperand(Op: MCOperand::createImm(Val: Disp)); // Displacement
3094	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3095	}
3096
3097	void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,
3098	int64_t Disp) const {
3099	Inst.setOpcode(X86::JMP64m);
3100	Inst.clear();
3101	Inst.addOperand(Op: MCOperand::createReg(Reg: MemBaseReg)); // BaseReg
3102	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3103	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3104	Inst.addOperand(Op: MCOperand::createImm(Val: Disp)); // Displacement
3105	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3106	}
3107
3108	InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,
3109	MCSymbol *HandlerFuncAddr,
3110	int CallSiteID,
3111	MCContext *Ctx) override {
3112	// Check if the target address expression used in the original indirect call
3113	// uses the stack pointer, which we are going to clobber.
3114	static BitVector SPAliases(getAliases(X86::RSP));
3115	bool UsesSP = any_of(useOperands(Inst&: CallInst), [&](const MCOperand &Op) {
3116	return Op.isReg() && SPAliases[Op.getReg()];
3117	});
3118
3119	InstructionListType Insts;
3120	MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3121	// Code sequence used to enter indirect call instrumentation helper:
3122	// push %rdi
3123	// add $8, %rsp ;; $rsp may be used in target, so fix it to prev val
3124	// movq target, %rdi ;; via convertIndirectCallTargetToLoad
3125	// sub $8, %rsp ;; restore correct stack value
3126	// push %rdi
3127	// movq $CallSiteID, %rdi
3128	// push %rdi
3129	// callq/jmp HandlerFuncAddr
3130	Insts.emplace_back();
3131	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3132	if (UsesSP) { // Only adjust SP if we really need to
3133	Insts.emplace_back();
3134	createStackPointerDecrement(Inst&: Insts.back(), Size: 8, /*NoFlagsClobber=*/false);
3135	}
3136	Insts.emplace_back(CallInst);
3137	// Insts.back() and CallInst now share the same annotation instruction.
3138	// Strip it from Insts.back(), only preserving tail call annotation.
3139	stripAnnotations(Inst&: Insts.back(), /*KeepTC=*/true);
3140	convertIndirectCallToLoad(Inst&: Insts.back(), Reg: TempReg);
3141	if (UsesSP) {
3142	Insts.emplace_back();
3143	createStackPointerIncrement(Inst&: Insts.back(), Size: 8, /*NoFlagsClobber=*/false);
3144	}
3145	Insts.emplace_back();
3146	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3147	InstructionListType LoadImm = createLoadImmediate(Dest: TempReg, Imm: CallSiteID);
3148	Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end());
3149	Insts.emplace_back();
3150	createPushRegister(Inst&: Insts.back(), Reg: TempReg, Size: 8);
3151
3152	MCInst &NewCallInst = Insts.emplace_back();
3153	createDirectCall(Inst&: NewCallInst, Target: HandlerFuncAddr, Ctx, IsTailCall: isTailCall(Inst: CallInst));
3154
3155	// Carry over metadata including tail call marker if present.
3156	stripAnnotations(Inst&: NewCallInst);
3157	moveAnnotations(SrcInst: std::move(CallInst), DstInst&: NewCallInst);
3158
3159	return Insts;
3160	}
3161
3162	InstructionListType createInstrumentedIndCallHandlerExitBB() const override {
3163	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3164	// We just need to undo the sequence created for every ind call in
3165	// instrumentIndirectTarget(), which can be accomplished minimally with:
3166	// popfq
3167	// pop %rdi
3168	// add $16, %rsp
3169	// xchg (%rsp), %rdi
3170	// jmp *-8(%rsp)
3171	InstructionListType Insts(5);
3172	createPopFlags(Inst&: Insts[0], Size: 8);
3173	createPopRegister(Inst&: Insts[1], Reg: TempReg, Size: 8);
3174	createStackPointerDecrement(Inst&: Insts[2], Size: 16, /*NoFlagsClobber=*/false);
3175	createSwap(Insts[3], TempReg, X86::RSP, 0);
3176	createIndirectBranch(Insts[4], X86::RSP, -8);
3177	return Insts;
3178	}
3179
3180	InstructionListType
3181	createInstrumentedIndTailCallHandlerExitBB() const override {
3182	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3183	// Same thing as above, but for tail calls
3184	// popfq
3185	// add $16, %rsp
3186	// pop %rdi
3187	// jmp *-16(%rsp)
3188	InstructionListType Insts(4);
3189	createPopFlags(Inst&: Insts[0], Size: 8);
3190	createStackPointerDecrement(Inst&: Insts[1], Size: 16, /*NoFlagsClobber=*/false);
3191	createPopRegister(Inst&: Insts[2], Reg: TempReg, Size: 8);
3192	createIndirectBranch(Insts[3], X86::RSP, -16);
3193	return Insts;
3194	}
3195
3196	InstructionListType
3197	createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,
3198	const MCSymbol *IndCallHandler,
3199	MCContext *Ctx) override {
3200	const MCPhysReg TempReg = getIntArgRegister(ArgNo: 0);
3201	// Code sequence used to check whether InstrTampoline was initialized
3202	// and call it if so, returns via IndCallHandler.
3203	// pushfq
3204	// mov InstrTrampoline,%rdi
3205	// cmp $0x0,%rdi
3206	// je IndCallHandler
3207	// callq *%rdi
3208	// jmpq IndCallHandler
3209	InstructionListType Insts;
3210	Insts.emplace_back();
3211	createPushFlags(Inst&: Insts.back(), Size: 8);
3212	Insts.emplace_back();
3213	createMove(Inst&: Insts.back(), Src: InstrTrampoline, Reg: TempReg, Ctx);
3214	InstructionListType cmpJmp = createCmpJE(RegNo: TempReg, Imm: 0, Target: IndCallHandler, Ctx);
3215	Insts.insert(Insts.end(), cmpJmp.begin(), cmpJmp.end());
3216	Insts.emplace_back();
3217	Insts.back().setOpcode(X86::CALL64r);
3218	Insts.back().addOperand(Op: MCOperand::createReg(Reg: TempReg));
3219	Insts.emplace_back();
3220	createDirectCall(Inst&: Insts.back(), Target: IndCallHandler, Ctx, /*IsTailCall*/ true);
3221	return Insts;
3222	}
3223
3224	InstructionListType createNumCountersGetter(MCContext *Ctx) const override {
3225	InstructionListType Insts(2);
3226	MCSymbol *NumLocs = Ctx->getOrCreateSymbol(Name: "__bolt_num_counters");
3227	createMove(Insts[0], NumLocs, X86::EAX, Ctx);
3228	createReturn(Inst&: Insts[1]);
3229	return Insts;
3230	}
3231
3232	InstructionListType
3233	createInstrLocationsGetter(MCContext *Ctx) const override {
3234	InstructionListType Insts(2);
3235	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_locations");
3236	createLea(Insts[0], Locs, X86::EAX, Ctx);
3237	createReturn(Inst&: Insts[1]);
3238	return Insts;
3239	}
3240
3241	InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {
3242	InstructionListType Insts(2);
3243	MCSymbol *Locs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_tables");
3244	createLea(Insts[0], Locs, X86::EAX, Ctx);
3245	createReturn(Inst&: Insts[1]);
3246	return Insts;
3247	}
3248
3249	InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {
3250	InstructionListType Insts(2);
3251	MCSymbol *NumFuncs = Ctx->getOrCreateSymbol(Name: "__bolt_instr_num_funcs");
3252	createMove(Insts[0], NumFuncs, X86::EAX, Ctx);
3253	createReturn(Inst&: Insts[1]);
3254	return Insts;
3255	}
3256
3257	InstructionListType createSymbolTrampoline(const MCSymbol *TgtSym,
3258	MCContext *Ctx) override {
3259	InstructionListType Insts(1);
3260	createUncondBranch(Inst&: Insts[0], TBB: TgtSym, Ctx);
3261	return Insts;
3262	}
3263
3264	InstructionListType createDummyReturnFunction(MCContext *Ctx) const override {
3265	InstructionListType Insts(1);
3266	createReturn(Inst&: Insts[0]);
3267	return Insts;
3268	}
3269
3270	BlocksVectorTy indirectCallPromotion(
3271	const MCInst &CallInst,
3272	const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3273	const std::vector<std::pair<MCSymbol *, uint64_t>> &VtableSyms,
3274	const std::vector<MCInst *> &MethodFetchInsns,
3275	const bool MinimizeCodeSize, MCContext *Ctx) override {
3276	const bool IsTailCall = isTailCall(Inst: CallInst);
3277	const bool IsJumpTable = getJumpTable(Inst: CallInst) != 0;
3278	BlocksVectorTy Results;
3279
3280	// Label for the current code block.
3281	MCSymbol *NextTarget = nullptr;
3282
3283	// The join block which contains all the instructions following CallInst.
3284	// MergeBlock remains null if CallInst is a tail call.
3285	MCSymbol *MergeBlock = nullptr;
3286
3287	unsigned FuncAddrReg = X86::R10;
3288
3289	const bool LoadElim = !VtableSyms.empty();
3290	assert((!LoadElim || VtableSyms.size() == Targets.size()) &&
3291	"There must be a vtable entry for every method "
3292	"in the targets vector.");
3293
3294	if (MinimizeCodeSize && !LoadElim) {
3295	std::set<unsigned> UsedRegs;
3296
3297	for (unsigned int I = 0; I < MCPlus::getNumPrimeOperands(Inst: CallInst); ++I) {
3298	const MCOperand &Op = CallInst.getOperand(i: I);
3299	if (Op.isReg())
3300	UsedRegs.insert(Op.getReg());
3301	}
3302
3303	if (UsedRegs.count(X86::R10) == 0)
3304	FuncAddrReg = X86::R10;
3305	else if (UsedRegs.count(X86::R11) == 0)
3306	FuncAddrReg = X86::R11;
3307	else
3308	return Results;
3309	}
3310
3311	const auto jumpToMergeBlock = [&](InstructionListType &NewCall) {
3312	assert(MergeBlock);
3313	NewCall.push_back(x: CallInst);
3314	MCInst &Merge = NewCall.back();
3315	Merge.clear();
3316	createUncondBranch(Inst&: Merge, TBB: MergeBlock, Ctx);
3317	};
3318
3319	for (unsigned int i = 0; i < Targets.size(); ++i) {
3320	Results.emplace_back(NextTarget, InstructionListType());
3321	InstructionListType *NewCall = &Results.back().second;
3322
3323	if (MinimizeCodeSize && !LoadElim) {
3324	// Load the call target into FuncAddrReg.
3325	NewCall->push_back(x: CallInst); // Copy CallInst in order to get SMLoc
3326	MCInst &Target = NewCall->back();
3327	Target.clear();
3328	Target.setOpcode(X86::MOV64ri32);
3329	Target.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3330	if (Targets[i].first) {
3331	// Is this OK?
3332	Target.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3333	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3334	} else {
3335	const uint64_t Addr = Targets[i].second;
3336	// Immediate address is out of sign extended 32 bit range.
3337	if (int64_t(Addr) != int64_t(int32_t(Addr)))
3338	return BlocksVectorTy();
3339
3340	Target.addOperand(Op: MCOperand::createImm(Val: Addr));
3341	}
3342
3343	// Compare current call target to a specific address.
3344	NewCall->push_back(x: CallInst);
3345	MCInst &Compare = NewCall->back();
3346	Compare.clear();
3347	if (isBranchOnReg(CallInst))
3348	Compare.setOpcode(X86::CMP64rr);
3349	else if (CallInst.getOpcode() == X86::CALL64pcrel32)
3350	Compare.setOpcode(X86::CMP64ri32);
3351	else
3352	Compare.setOpcode(X86::CMP64rm);
3353
3354	Compare.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3355
3356	// TODO: Would be preferable to only load this value once.
3357	for (unsigned i = 0;
3358	i < Info->get(Opcode: CallInst.getOpcode()).getNumOperands(); ++i)
3359	if (!CallInst.getOperand(i).isInst())
3360	Compare.addOperand(Op: CallInst.getOperand(i));
3361	} else {
3362	// Compare current call target to a specific address.
3363	NewCall->push_back(x: CallInst);
3364	MCInst &Compare = NewCall->back();
3365	Compare.clear();
3366	if (isBranchOnReg(CallInst))
3367	Compare.setOpcode(X86::CMP64ri32);
3368	else
3369	Compare.setOpcode(X86::CMP64mi32);
3370
3371	// Original call address.
3372	for (unsigned i = 0;
3373	i < Info->get(Opcode: CallInst.getOpcode()).getNumOperands(); ++i)
3374	if (!CallInst.getOperand(i).isInst())
3375	Compare.addOperand(Op: CallInst.getOperand(i));
3376
3377	// Target address.
3378	if (Targets[i].first || LoadElim) {
3379	const MCSymbol *Sym =
3380	LoadElim ? VtableSyms[i].first : Targets[i].first;
3381	const uint64_t Addend = LoadElim ? VtableSyms[i].second : 0;
3382	const MCExpr *Expr = MCSymbolRefExpr::create(Symbol: Sym, Ctx&: *Ctx);
3383	if (Addend)
3384	Expr = MCBinaryExpr::createAdd(
3385	LHS: Expr, RHS: MCConstantExpr::create(Value: Addend, Ctx&: *Ctx), Ctx&: *Ctx);
3386	Compare.addOperand(Op: MCOperand::createExpr(Val: Expr));
3387	} else {
3388	const uint64_t Addr = Targets[i].second;
3389	// Immediate address is out of sign extended 32 bit range.
3390	if (int64_t(Addr) != int64_t(int32_t(Addr)))
3391	return BlocksVectorTy();
3392
3393	Compare.addOperand(Op: MCOperand::createImm(Val: Addr));
3394	}
3395	}
3396
3397	// jump to next target compare.
3398	NextTarget =
3399	Ctx->createNamedTempSymbol(); // generate label for the next block
3400	NewCall->push_back(x: CallInst);
3401
3402	if (IsJumpTable) {
3403	MCInst &Je = NewCall->back();
3404
3405	// Jump to next compare if target addresses don't match.
3406	Je.clear();
3407	Je.setOpcode(X86::JCC_1);
3408	if (Targets[i].first)
3409	Je.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3410	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3411	else
3412	Je.addOperand(Op: MCOperand::createImm(Val: Targets[i].second));
3413
3414	Je.addOperand(Op: MCOperand::createImm(Val: X86::COND_E));
3415	assert(!isInvoke(CallInst));
3416	} else {
3417	MCInst &Jne = NewCall->back();
3418
3419	// Jump to next compare if target addresses don't match.
3420	Jne.clear();
3421	Jne.setOpcode(X86::JCC_1);
3422	Jne.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3423	Symbol: NextTarget, Kind: MCSymbolRefExpr::VK_None, Ctx&: *Ctx)));
3424	Jne.addOperand(Op: MCOperand::createImm(Val: X86::COND_NE));
3425
3426	// Call specific target directly.
3427	Results.emplace_back(Ctx->createNamedTempSymbol(),
3428	InstructionListType());
3429	NewCall = &Results.back().second;
3430	NewCall->push_back(x: CallInst);
3431	MCInst &CallOrJmp = NewCall->back();
3432
3433	CallOrJmp.clear();
3434
3435	if (MinimizeCodeSize && !LoadElim) {
3436	CallOrJmp.setOpcode(IsTailCall ? X86::JMP32r : X86::CALL64r);
3437	CallOrJmp.addOperand(Op: MCOperand::createReg(Reg: FuncAddrReg));
3438	} else {
3439	CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
3440
3441	if (Targets[i].first)
3442	CallOrJmp.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3443	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3444	else
3445	CallOrJmp.addOperand(Op: MCOperand::createImm(Val: Targets[i].second));
3446	}
3447	if (IsTailCall)
3448	setTailCall(CallOrJmp);
3449
3450	if (CallOrJmp.getOpcode() == X86::CALL64r ||
3451	CallOrJmp.getOpcode() == X86::CALL64pcrel32) {
3452	if (std::optional<uint32_t> Offset = getOffset(Inst: CallInst))
3453	// Annotated as duplicated call
3454	setOffset(Inst&: CallOrJmp, Offset: *Offset);
3455	}
3456
3457	if (isInvoke(Inst: CallInst) && !isInvoke(Inst: CallOrJmp)) {
3458	// Copy over any EH or GNU args size information from the original
3459	// call.
3460	std::optional<MCPlus::MCLandingPad> EHInfo = getEHInfo(CallInst);
3461	if (EHInfo)
3462	addEHInfo(Inst&: CallOrJmp, LP: *EHInfo);
3463	int64_t GnuArgsSize = getGnuArgsSize(Inst: CallInst);
3464	if (GnuArgsSize >= 0)
3465	addGnuArgsSize(Inst&: CallOrJmp, GnuArgsSize);
3466	}
3467
3468	if (!IsTailCall) {
3469	// The fallthrough block for the most common target should be
3470	// the merge block.
3471	if (i == 0) {
3472	// Fallthrough to merge block.
3473	MergeBlock = Ctx->createNamedTempSymbol();
3474	} else {
3475	// Insert jump to the merge block if we are not doing a fallthrough.
3476	jumpToMergeBlock(*NewCall);
3477	}
3478	}
3479	}
3480	}
3481
3482	// Cold call block.
3483	Results.emplace_back(NextTarget, InstructionListType());
3484	InstructionListType &NewCall = Results.back().second;
3485	for (const MCInst *Inst : MethodFetchInsns)
3486	if (Inst != &CallInst)
3487	NewCall.push_back(*Inst);
3488	NewCall.push_back(x: CallInst);
3489
3490	// Jump to merge block from cold call block
3491	if (!IsTailCall && !IsJumpTable) {
3492	jumpToMergeBlock(NewCall);
3493
3494	// Record merge block
3495	Results.emplace_back(MergeBlock, InstructionListType());
3496	}
3497
3498	return Results;
3499	}
3500
3501	BlocksVectorTy jumpTablePromotion(
3502	const MCInst &IJmpInst,
3503	const std::vector<std::pair<MCSymbol *, uint64_t>> &Targets,
3504	const std::vector<MCInst *> &TargetFetchInsns,
3505	MCContext *Ctx) const override {
3506	assert(getJumpTable(IJmpInst) != 0);
3507	uint16_t IndexReg = getAnnotationAs<uint16_t>(IJmpInst, "JTIndexReg");
3508	if (IndexReg == 0)
3509	return BlocksVectorTy();
3510
3511	BlocksVectorTy Results;
3512
3513	// Label for the current code block.
3514	MCSymbol *NextTarget = nullptr;
3515
3516	for (unsigned int i = 0; i < Targets.size(); ++i) {
3517	Results.emplace_back(NextTarget, InstructionListType());
3518	InstructionListType *CurBB = &Results.back().second;
3519
3520	// Compare current index to a specific index.
3521	CurBB->emplace_back(MCInst());
3522	MCInst &CompareInst = CurBB->back();
3523	CompareInst.setLoc(IJmpInst.getLoc());
3524	CompareInst.setOpcode(X86::CMP64ri32);
3525	CompareInst.addOperand(Op: MCOperand::createReg(Reg: IndexReg));
3526
3527	const uint64_t CaseIdx = Targets[i].second;
3528	// Immediate address is out of sign extended 32 bit range.
3529	if (int64_t(CaseIdx) != int64_t(int32_t(CaseIdx)))
3530	return BlocksVectorTy();
3531
3532	CompareInst.addOperand(Op: MCOperand::createImm(Val: CaseIdx));
3533	shortenInstruction(Inst&: CompareInst, STI: *Ctx->getSubtargetInfo());
3534
3535	// jump to next target compare.
3536	NextTarget =
3537	Ctx->createNamedTempSymbol(); // generate label for the next block
3538	CurBB->push_back(x: MCInst());
3539
3540	MCInst &JEInst = CurBB->back();
3541	JEInst.setLoc(IJmpInst.getLoc());
3542
3543	// Jump to target if indices match
3544	JEInst.setOpcode(X86::JCC_1);
3545	JEInst.addOperand(Op: MCOperand::createExpr(Val: MCSymbolRefExpr::create(
3546	Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
3547	JEInst.addOperand(Op: MCOperand::createImm(Val: X86::COND_E));
3548	}
3549
3550	// Cold call block.
3551	Results.emplace_back(NextTarget, InstructionListType());
3552	InstructionListType &CurBB = Results.back().second;
3553	for (const MCInst *Inst : TargetFetchInsns)
3554	if (Inst != &IJmpInst)
3555	CurBB.push_back(*Inst);
3556
3557	CurBB.push_back(x: IJmpInst);
3558
3559	return Results;
3560	}
3561
3562	private:
3563	void createMove(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3564	MCContext *Ctx) const {
3565	Inst.setOpcode(X86::MOV64rm);
3566	Inst.clear();
3567	Inst.addOperand(Op: MCOperand::createReg(Reg));
3568	Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg
3569	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3570	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3571	Inst.addOperand(Op: MCOperand::createExpr(
3572	Val: MCSymbolRefExpr::create(Symbol: Src, Kind: MCSymbolRefExpr::VK_None,
3573	Ctx&: *Ctx))); // Displacement
3574	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3575	}
3576
3577	void createLea(MCInst &Inst, const MCSymbol *Src, unsigned Reg,
3578	MCContext *Ctx) const {
3579	Inst.setOpcode(X86::LEA64r);
3580	Inst.clear();
3581	Inst.addOperand(Op: MCOperand::createReg(Reg));
3582	Inst.addOperand(MCOperand::createReg(X86::RIP)); // BaseReg
3583	Inst.addOperand(Op: MCOperand::createImm(Val: 1)); // ScaleAmt
3584	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
3585	Inst.addOperand(Op: MCOperand::createExpr(
3586	Val: MCSymbolRefExpr::create(Symbol: Src, Kind: MCSymbolRefExpr::VK_None,
3587	Ctx&: *Ctx))); // Displacement
3588	Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
3589	}
3590	};
3591
3592	} // namespace
3593
3594	namespace llvm {
3595	namespace bolt {
3596
3597	MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *Analysis,
3598	const MCInstrInfo *Info,
3599	const MCRegisterInfo *RegInfo,
3600	const MCSubtargetInfo *STI) {
3601	return new X86MCPlusBuilder(Analysis, Info, RegInfo, STI);
3602	}
3603
3604	} // namespace bolt
3605	} // namespace llvm
3606