1	//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the RISC-V implementation of the TargetInstrInfo class.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "RISCVInstrInfo.h"
14	#include "MCTargetDesc/RISCVMatInt.h"
15	#include "RISCV.h"
16	#include "RISCVMachineFunctionInfo.h"
17	#include "RISCVSubtarget.h"
18	#include "RISCVTargetMachine.h"
19	#include "llvm/ADT/STLExtras.h"
20	#include "llvm/ADT/SmallVector.h"
21	#include "llvm/Analysis/MemoryLocation.h"
22	#include "llvm/Analysis/ValueTracking.h"
23	#include "llvm/CodeGen/LiveIntervals.h"
24	#include "llvm/CodeGen/LiveVariables.h"
25	#include "llvm/CodeGen/MachineCombinerPattern.h"
26	#include "llvm/CodeGen/MachineFunctionPass.h"
27	#include "llvm/CodeGen/MachineInstrBuilder.h"
28	#include "llvm/CodeGen/MachineRegisterInfo.h"
29	#include "llvm/CodeGen/MachineTraceMetrics.h"
30	#include "llvm/CodeGen/RegisterScavenging.h"
31	#include "llvm/CodeGen/StackMaps.h"
32	#include "llvm/IR/DebugInfoMetadata.h"
33	#include "llvm/MC/MCInstBuilder.h"
34	#include "llvm/MC/TargetRegistry.h"
35	#include "llvm/Support/ErrorHandling.h"
36
37	using namespace llvm;
38
39	#define GEN_CHECK_COMPRESS_INSTR
40	#include "RISCVGenCompressInstEmitter.inc"
41
42	#define GET_INSTRINFO_CTOR_DTOR
43	#define GET_INSTRINFO_NAMED_OPS
44	#include "RISCVGenInstrInfo.inc"
45
46	static cl::opt<bool> PreferWholeRegisterMove(
47	"riscv-prefer-whole-register-move", cl::init(Val: false), cl::Hidden,
48	cl::desc("Prefer whole register move for vector registers."));
49
50	static cl::opt<MachineTraceStrategy> ForceMachineCombinerStrategy(
51	"riscv-force-machine-combiner-strategy", cl::Hidden,
52	cl::desc("Force machine combiner to use a specific strategy for machine "
53	"trace metrics evaluation."),
54	cl::init(Val: MachineTraceStrategy::TS_NumStrategies),
55	cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local",
56	"Local strategy."),
57	clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
58	"MinInstrCount strategy.")));
59
60	namespace llvm::RISCVVPseudosTable {
61
62	using namespace RISCV;
63
64	#define GET_RISCVVPseudosTable_IMPL
65	#include "RISCVGenSearchableTables.inc"
66
67	} // namespace llvm::RISCVVPseudosTable
68
69	namespace llvm::RISCV {
70
71	#define GET_RISCVMaskedPseudosTable_IMPL
72	#include "RISCVGenSearchableTables.inc"
73
74	} // end namespace llvm::RISCV
75
76	RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI)
77	: RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
78	STI(STI) {}
79
80	MCInst RISCVInstrInfo::getNop() const {
81	if (STI.hasStdExtCOrZca())
82	return MCInstBuilder(RISCV::C_NOP);
83	return MCInstBuilder(RISCV::ADDI)
84	.addReg(RISCV::X0)
85	.addReg(RISCV::X0)
86	.addImm(0);
87	}
88
89	Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
90	int &FrameIndex) const {
91	unsigned Dummy;
92	return isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
93	}
94
95	Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
96	int &FrameIndex,
97	unsigned &MemBytes) const {
98	switch (MI.getOpcode()) {
99	default:
100	return 0;
101	case RISCV::LB:
102	case RISCV::LBU:
103	MemBytes = 1;
104	break;
105	case RISCV::LH:
106	case RISCV::LHU:
107	case RISCV::FLH:
108	MemBytes = 2;
109	break;
110	case RISCV::LW:
111	case RISCV::FLW:
112	case RISCV::LWU:
113	MemBytes = 4;
114	break;
115	case RISCV::LD:
116	case RISCV::FLD:
117	MemBytes = 8;
118	break;
119	}
120
121	if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
122	MI.getOperand(i: 2).getImm() == 0) {
123	FrameIndex = MI.getOperand(i: 1).getIndex();
124	return MI.getOperand(i: 0).getReg();
125	}
126
127	return 0;
128	}
129
130	Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
131	int &FrameIndex) const {
132	unsigned Dummy;
133	return isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
134	}
135
136	Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
137	int &FrameIndex,
138	unsigned &MemBytes) const {
139	switch (MI.getOpcode()) {
140	default:
141	return 0;
142	case RISCV::SB:
143	MemBytes = 1;
144	break;
145	case RISCV::SH:
146	case RISCV::FSH:
147	MemBytes = 2;
148	break;
149	case RISCV::SW:
150	case RISCV::FSW:
151	MemBytes = 4;
152	break;
153	case RISCV::SD:
154	case RISCV::FSD:
155	MemBytes = 8;
156	break;
157	}
158
159	if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
160	MI.getOperand(i: 2).getImm() == 0) {
161	FrameIndex = MI.getOperand(i: 1).getIndex();
162	return MI.getOperand(i: 0).getReg();
163	}
164
165	return 0;
166	}
167
168	static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
169	unsigned NumRegs) {
170	return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
171	}
172
173	static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
174	const MachineBasicBlock &MBB,
175	MachineBasicBlock::const_iterator MBBI,
176	MachineBasicBlock::const_iterator &DefMBBI,
177	RISCVII::VLMUL LMul) {
178	if (PreferWholeRegisterMove)
179	return false;
180
181	assert(MBBI->getOpcode() == TargetOpcode::COPY &&
182	"Unexpected COPY instruction.");
183	Register SrcReg = MBBI->getOperand(i: 1).getReg();
184	const TargetRegisterInfo *TRI = STI.getRegisterInfo();
185
186	bool FoundDef = false;
187	bool FirstVSetVLI = false;
188	unsigned FirstSEW = 0;
189	while (MBBI != MBB.begin()) {
190	--MBBI;
191	if (MBBI->isMetaInstruction())
192	continue;
193
194	if (MBBI->getOpcode() == RISCV::PseudoVSETVLI ||
195	MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||
196	MBBI->getOpcode() == RISCV::PseudoVSETIVLI) {
197	// There is a vsetvli between COPY and source define instruction.
198	// vy = def_vop ... (producing instruction)
199	// ...
200	// vsetvli
201	// ...
202	// vx = COPY vy
203	if (!FoundDef) {
204	if (!FirstVSetVLI) {
205	FirstVSetVLI = true;
206	unsigned FirstVType = MBBI->getOperand(i: 2).getImm();
207	RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(VType: FirstVType);
208	FirstSEW = RISCVVType::getSEW(VType: FirstVType);
209	// The first encountered vsetvli must have the same lmul as the
210	// register class of COPY.
211	if (FirstLMul != LMul)
212	return false;
213	}
214	// Only permit `vsetvli x0, x0, vtype` between COPY and the source
215	// define instruction.
216	if (MBBI->getOperand(i: 0).getReg() != RISCV::X0)
217	return false;
218	if (MBBI->getOperand(i: 1).isImm())
219	return false;
220	if (MBBI->getOperand(i: 1).getReg() != RISCV::X0)
221	return false;
222	continue;
223	}
224
225	// MBBI is the first vsetvli before the producing instruction.
226	unsigned VType = MBBI->getOperand(i: 2).getImm();
227	// If there is a vsetvli between COPY and the producing instruction.
228	if (FirstVSetVLI) {
229	// If SEW is different, return false.
230	if (RISCVVType::getSEW(VType) != FirstSEW)
231	return false;
232	}
233
234	// If the vsetvli is tail undisturbed, keep the whole register move.
235	if (!RISCVVType::isTailAgnostic(VType))
236	return false;
237
238	// The checking is conservative. We only have register classes for
239	// LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
240	// for fractional LMUL operations. However, we could not use the vsetvli
241	// lmul for widening operations. The result of widening operation is
242	// 2 x LMUL.
243	return LMul == RISCVVType::getVLMUL(VType);
244	} else if (MBBI->isInlineAsm() || MBBI->isCall()) {
245	return false;
246	} else if (MBBI->getNumDefs()) {
247	// Check all the instructions which will change VL.
248	// For example, vleff has implicit def VL.
249	if (MBBI->modifiesRegister(RISCV::Reg: VL, /*TRI=*/nullptr))
250	return false;
251
252	// Only converting whole register copies to vmv.v.v when the defining
253	// value appears in the explicit operands.
254	for (const MachineOperand &MO : MBBI->explicit_operands()) {
255	if (!MO.isReg() || !MO.isDef())
256	continue;
257	if (!FoundDef && TRI->regsOverlap(RegA: MO.getReg(), RegB: SrcReg)) {
258	// We only permit the source of COPY has the same LMUL as the defined
259	// operand.
260	// There are cases we need to keep the whole register copy if the LMUL
261	// is different.
262	// For example,
263	// $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
264	// $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
265	// # The COPY may be created by vlmul_trunc intrinsic.
266	// $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
267	//
268	// After widening, the valid value will be 4 x e32 elements. If we
269	// convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
270	// FIXME: The COPY of subregister of Zvlsseg register will not be able
271	// to convert to vmv.v.[v|i] under the constraint.
272	if (MO.getReg() != SrcReg)
273	return false;
274
275	// In widening reduction instructions with LMUL_1 input vector case,
276	// only checking the LMUL is insufficient due to reduction result is
277	// always LMUL_1.
278	// For example,
279	// $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
280	// $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
281	// $v26 = COPY killed renamable $v8
282	// After widening, The valid value will be 1 x e16 elements. If we
283	// convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
284	uint64_t TSFlags = MBBI->getDesc().TSFlags;
285	if (RISCVII::isRVVWideningReduction(TSFlags))
286	return false;
287
288	// If the producing instruction does not depend on vsetvli, do not
289	// convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
290	if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
291	return false;
292
293	// Found the definition.
294	FoundDef = true;
295	DefMBBI = MBBI;
296	break;
297	}
298	}
299	}
300	}
301
302	return false;
303	}
304
305	void RISCVInstrInfo::copyPhysRegVector(
306	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
307	const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
308	const TargetRegisterClass *RegClass) const {
309	const TargetRegisterInfo *TRI = STI.getRegisterInfo();
310	RISCVII::VLMUL LMul = RISCVRI::getLMul(TSFlags: RegClass->TSFlags);
311	unsigned NF = RISCVRI::getNF(TSFlags: RegClass->TSFlags);
312
313	uint16_t SrcEncoding = TRI->getEncodingValue(RegNo: SrcReg);
314	uint16_t DstEncoding = TRI->getEncodingValue(RegNo: DstReg);
315	auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(VLMUL: LMul);
316	assert(!Fractional && "It is impossible be fractional lmul here.");
317	unsigned NumRegs = NF * LMulVal;
318	bool ReversedCopy =
319	forwardCopyWillClobberTuple(DstReg: DstEncoding, SrcReg: SrcEncoding, NumRegs);
320	if (ReversedCopy) {
321	// If the src and dest overlap when copying a tuple, we need to copy the
322	// registers in reverse.
323	SrcEncoding += NumRegs - 1;
324	DstEncoding += NumRegs - 1;
325	}
326
327	unsigned I = 0;
328	auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
329	-> std::tuple<RISCVII::VLMUL, const TargetRegisterClass &, unsigned,
330	unsigned, unsigned> {
331	if (ReversedCopy) {
332	// For reversed copying, if there are enough aligned registers(8/4/2), we
333	// can do a larger copy(LMUL8/4/2).
334	// Besides, we have already known that DstEncoding is larger than
335	// SrcEncoding in forwardCopyWillClobberTuple, so the difference between
336	// DstEncoding and SrcEncoding should be >= LMUL value we try to use to
337	// avoid clobbering.
338	uint16_t Diff = DstEncoding - SrcEncoding;
339	if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
340	DstEncoding % 8 == 7)
341	return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
342	RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
343	if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
344	DstEncoding % 4 == 3)
345	return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
346	RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
347	if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
348	DstEncoding % 2 == 1)
349	return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
350	RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
351	// Or we should do LMUL1 copying.
352	return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
353	RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
354	}
355
356	// For forward copying, if source register encoding and destination register
357	// encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
358	if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
359	return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
360	RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
361	if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
362	return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
363	RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
364	if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
365	return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
366	RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
367	// Or we should do LMUL1 copying.
368	return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
369	RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
370	};
371	auto FindRegWithEncoding = [TRI](const TargetRegisterClass &RegClass,
372	uint16_t Encoding) {
373	MCRegister Reg = RISCV::V0 + Encoding;
374	if (&RegClass == &RISCV::VRRegClass)
375	return Reg;
376	return TRI->getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass);
377	};
378	while (I != NumRegs) {
379	// For non-segment copying, we only do this once as the registers are always
380	// aligned.
381	// For segment copying, we may do this several times. If the registers are
382	// aligned to larger LMUL, we can eliminate some copyings.
383	auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
384	GetCopyInfo(SrcEncoding, DstEncoding);
385	auto [NumCopied, _] = RISCVVType::decodeVLMUL(VLMUL: LMulCopied);
386
387	MachineBasicBlock::const_iterator DefMBBI;
388	if (LMul == LMulCopied &&
389	isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
390	Opc = VVOpc;
391	if (DefMBBI->getOpcode() == VIOpc)
392	Opc = VIOpc;
393	}
394
395	// Emit actual copying.
396	// For reversed copying, the encoding should be decreased.
397	MCRegister ActualSrcReg = FindRegWithEncoding(
398	RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
399	MCRegister ActualDstReg = FindRegWithEncoding(
400	RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
401
402	auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), ActualDstReg);
403	bool UseVMV_V_I = RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_I;
404	bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_V;
405	if (UseVMV)
406	MIB.addReg(ActualDstReg, RegState::Undef);
407	if (UseVMV_V_I)
408	MIB = MIB.add(DefMBBI->getOperand(i: 2));
409	else
410	MIB = MIB.addReg(ActualSrcReg, getKillRegState(B: KillSrc));
411	if (UseVMV) {
412	const MCInstrDesc &Desc = DefMBBI->getDesc();
413	MIB.add(DefMBBI->getOperand(i: RISCVII::getVLOpNum(Desc))); // AVL
414	MIB.add(DefMBBI->getOperand(i: RISCVII::getSEWOpNum(Desc))); // SEW
415	MIB.addImm(0); // tu, mu
416	MIB.addReg(RISCV::VL, RegState::Implicit);
417	MIB.addReg(RISCV::VTYPE, RegState::Implicit);
418	}
419
420	// If we are copying reversely, we should decrease the encoding.
421	SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
422	DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
423	I += NumCopied;
424	}
425	}
426
427	void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
428	MachineBasicBlock::iterator MBBI,
429	const DebugLoc &DL, MCRegister DstReg,
430	MCRegister SrcReg, bool KillSrc) const {
431	const TargetRegisterInfo *TRI = STI.getRegisterInfo();
432
433	if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
434	BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
435	.addReg(SrcReg, getKillRegState(KillSrc))
436	.addImm(0);
437	return;
438	}
439
440	if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
441	// Emit an ADDI for both parts of GPRPair.
442	BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
443	TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
444	.addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_even),
445	getKillRegState(KillSrc))
446	.addImm(0);
447	BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
448	TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
449	.addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_odd),
450	getKillRegState(KillSrc))
451	.addImm(0);
452	return;
453	}
454
455	// Handle copy from csr
456	if (RISCV::VCSRRegClass.contains(SrcReg) &&
457	RISCV::GPRRegClass.contains(DstReg)) {
458	BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)
459	.addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)
460	.addReg(RISCV::X0);
461	return;
462	}
463
464	if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
465	unsigned Opc;
466	if (STI.hasStdExtZfh()) {
467	Opc = RISCV::FSGNJ_H;
468	} else {
469	assert(STI.hasStdExtF() &&
470	(STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
471	"Unexpected extensions");
472	// Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
473	DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
474	&RISCV::FPR32RegClass);
475	SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
476	&RISCV::FPR32RegClass);
477	Opc = RISCV::FSGNJ_S;
478	}
479	BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
480	.addReg(SrcReg, getKillRegState(B: KillSrc))
481	.addReg(SrcReg, getKillRegState(B: KillSrc));
482	return;
483	}
484
485	if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
486	BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)
487	.addReg(SrcReg, getKillRegState(KillSrc))
488	.addReg(SrcReg, getKillRegState(KillSrc));
489	return;
490	}
491
492	if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
493	BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)
494	.addReg(SrcReg, getKillRegState(KillSrc))
495	.addReg(SrcReg, getKillRegState(KillSrc));
496	return;
497	}
498
499	if (RISCV::FPR32RegClass.contains(DstReg) &&
500	RISCV::GPRRegClass.contains(SrcReg)) {
501	BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
502	.addReg(SrcReg, getKillRegState(KillSrc));
503	return;
504	}
505
506	if (RISCV::GPRRegClass.contains(DstReg) &&
507	RISCV::FPR32RegClass.contains(SrcReg)) {
508	BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
509	.addReg(SrcReg, getKillRegState(KillSrc));
510	return;
511	}
512
513	if (RISCV::FPR64RegClass.contains(DstReg) &&
514	RISCV::GPRRegClass.contains(SrcReg)) {
515	assert(STI.getXLen() == 64 && "Unexpected GPR size");
516	BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
517	.addReg(SrcReg, getKillRegState(KillSrc));
518	return;
519	}
520
521	if (RISCV::GPRRegClass.contains(DstReg) &&
522	RISCV::FPR64RegClass.contains(SrcReg)) {
523	assert(STI.getXLen() == 64 && "Unexpected GPR size");
524	BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
525	.addReg(SrcReg, getKillRegState(KillSrc));
526	return;
527	}
528
529	// VR->VR copies.
530	static const TargetRegisterClass *RVVRegClasses[] = {
531	&RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
532	&RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN2M2RegClass,
533	&RISCV::VRN2M4RegClass, &RISCV::VRN3M1RegClass, &RISCV::VRN3M2RegClass,
534	&RISCV::VRN4M1RegClass, &RISCV::VRN4M2RegClass, &RISCV::VRN5M1RegClass,
535	&RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass, &RISCV::VRN8M1RegClass};
536	for (const auto &RegClass : RVVRegClasses) {
537	if (RegClass->contains(DstReg, SrcReg)) {
538	copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
539	return;
540	}
541	}
542
543	llvm_unreachable("Impossible reg-to-reg copy");
544	}
545
546	void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
547	MachineBasicBlock::iterator I,
548	Register SrcReg, bool IsKill, int FI,
549	const TargetRegisterClass *RC,
550	const TargetRegisterInfo *TRI,
551	Register VReg) const {
552	MachineFunction *MF = MBB.getParent();
553	MachineFrameInfo &MFI = MF->getFrameInfo();
554
555	unsigned Opcode;
556	bool IsScalableVector = true;
557	if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
558	Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
559	RISCV::SW : RISCV::SD;
560	IsScalableVector = false;
561	} else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
562	Opcode = RISCV::PseudoRV32ZdinxSD;
563	IsScalableVector = false;
564	} else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
565	Opcode = RISCV::FSH;
566	IsScalableVector = false;
567	} else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
568	Opcode = RISCV::FSW;
569	IsScalableVector = false;
570	} else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
571	Opcode = RISCV::FSD;
572	IsScalableVector = false;
573	} else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
574	Opcode = RISCV::VS1R_V;
575	} else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
576	Opcode = RISCV::VS2R_V;
577	} else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
578	Opcode = RISCV::VS4R_V;
579	} else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
580	Opcode = RISCV::VS8R_V;
581	} else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
582	Opcode = RISCV::PseudoVSPILL2_M1;
583	else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
584	Opcode = RISCV::PseudoVSPILL2_M2;
585	else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
586	Opcode = RISCV::PseudoVSPILL2_M4;
587	else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
588	Opcode = RISCV::PseudoVSPILL3_M1;
589	else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
590	Opcode = RISCV::PseudoVSPILL3_M2;
591	else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
592	Opcode = RISCV::PseudoVSPILL4_M1;
593	else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
594	Opcode = RISCV::PseudoVSPILL4_M2;
595	else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
596	Opcode = RISCV::PseudoVSPILL5_M1;
597	else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
598	Opcode = RISCV::PseudoVSPILL6_M1;
599	else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
600	Opcode = RISCV::PseudoVSPILL7_M1;
601	else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
602	Opcode = RISCV::PseudoVSPILL8_M1;
603	else
604	llvm_unreachable("Can't store this register to stack slot");
605
606	if (IsScalableVector) {
607	MachineMemOperand *MMO = MF->getMachineMemOperand(
608	PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
609	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
610
611	MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
612	BuildMI(MBB, I, DebugLoc(), get(Opcode))
613	.addReg(SrcReg, getKillRegState(B: IsKill))
614	.addFrameIndex(FI)
615	.addMemOperand(MMO);
616	} else {
617	MachineMemOperand *MMO = MF->getMachineMemOperand(
618	PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
619	Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
620
621	BuildMI(MBB, I, DebugLoc(), get(Opcode))
622	.addReg(SrcReg, getKillRegState(B: IsKill))
623	.addFrameIndex(FI)
624	.addImm(0)
625	.addMemOperand(MMO);
626	}
627	}
628
629	void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
630	MachineBasicBlock::iterator I,
631	Register DstReg, int FI,
632	const TargetRegisterClass *RC,
633	const TargetRegisterInfo *TRI,
634	Register VReg) const {
635	MachineFunction *MF = MBB.getParent();
636	MachineFrameInfo &MFI = MF->getFrameInfo();
637
638	unsigned Opcode;
639	bool IsScalableVector = true;
640	if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
641	Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
642	RISCV::LW : RISCV::LD;
643	IsScalableVector = false;
644	} else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
645	Opcode = RISCV::PseudoRV32ZdinxLD;
646	IsScalableVector = false;
647	} else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
648	Opcode = RISCV::FLH;
649	IsScalableVector = false;
650	} else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
651	Opcode = RISCV::FLW;
652	IsScalableVector = false;
653	} else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
654	Opcode = RISCV::FLD;
655	IsScalableVector = false;
656	} else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
657	Opcode = RISCV::VL1RE8_V;
658	} else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
659	Opcode = RISCV::VL2RE8_V;
660	} else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
661	Opcode = RISCV::VL4RE8_V;
662	} else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
663	Opcode = RISCV::VL8RE8_V;
664	} else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
665	Opcode = RISCV::PseudoVRELOAD2_M1;
666	else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
667	Opcode = RISCV::PseudoVRELOAD2_M2;
668	else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
669	Opcode = RISCV::PseudoVRELOAD2_M4;
670	else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
671	Opcode = RISCV::PseudoVRELOAD3_M1;
672	else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
673	Opcode = RISCV::PseudoVRELOAD3_M2;
674	else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
675	Opcode = RISCV::PseudoVRELOAD4_M1;
676	else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
677	Opcode = RISCV::PseudoVRELOAD4_M2;
678	else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
679	Opcode = RISCV::PseudoVRELOAD5_M1;
680	else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
681	Opcode = RISCV::PseudoVRELOAD6_M1;
682	else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
683	Opcode = RISCV::PseudoVRELOAD7_M1;
684	else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
685	Opcode = RISCV::PseudoVRELOAD8_M1;
686	else
687	llvm_unreachable("Can't load this register from stack slot");
688
689	if (IsScalableVector) {
690	MachineMemOperand *MMO = MF->getMachineMemOperand(
691	PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
692	Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
693
694	MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
695	BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
696	.addFrameIndex(FI)
697	.addMemOperand(MMO);
698	} else {
699	MachineMemOperand *MMO = MF->getMachineMemOperand(
700	PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
701	Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
702
703	BuildMI(MBB, I, DebugLoc(), get(Opcode), DstReg)
704	.addFrameIndex(FI)
705	.addImm(0)
706	.addMemOperand(MMO);
707	}
708	}
709
710	MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
711	MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
712	MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
713	VirtRegMap *VRM) const {
714	const MachineFrameInfo &MFI = MF.getFrameInfo();
715
716	// The below optimizations narrow the load so they are only valid for little
717	// endian.
718	// TODO: Support big endian by adding an offset into the frame object?
719	if (MF.getDataLayout().isBigEndian())
720	return nullptr;
721
722	// Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
723	if (Ops.size() != 1 || Ops[0] != 1)
724	return nullptr;
725
726	unsigned LoadOpc;
727	switch (MI.getOpcode()) {
728	default:
729	if (RISCV::isSEXT_W(MI)) {
730	LoadOpc = RISCV::LW;
731	break;
732	}
733	if (RISCV::isZEXT_W(MI)) {
734	LoadOpc = RISCV::LWU;
735	break;
736	}
737	if (RISCV::isZEXT_B(MI)) {
738	LoadOpc = RISCV::LBU;
739	break;
740	}
741	return nullptr;
742	case RISCV::SEXT_H:
743	LoadOpc = RISCV::LH;
744	break;
745	case RISCV::SEXT_B:
746	LoadOpc = RISCV::LB;
747	break;
748	case RISCV::ZEXT_H_RV32:
749	case RISCV::ZEXT_H_RV64:
750	LoadOpc = RISCV::LHU;
751	break;
752	}
753
754	MachineMemOperand *MMO = MF.getMachineMemOperand(
755	PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex),
756	F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIndex),
757	BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIndex));
758
759	Register DstReg = MI.getOperand(i: 0).getReg();
760	return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(LoadOpc),
761	DstReg)
762	.addFrameIndex(FrameIndex)
763	.addImm(0)
764	.addMemOperand(MMO);
765	}
766
767	void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
768	MachineBasicBlock::iterator MBBI,
769	const DebugLoc &DL, Register DstReg, uint64_t Val,
770	MachineInstr::MIFlag Flag, bool DstRenamable,
771	bool DstIsDead) const {
772	Register SrcReg = RISCV::X0;
773
774	// For RV32, allow a sign or unsigned 32 bit value.
775	if (!STI.is64Bit() && !isInt<32>(x: Val)) {
776	// If have a uimm32 it will still fit in a register so we can allow it.
777	if (!isUInt<32>(x: Val))
778	report_fatal_error(reason: "Should only materialize 32-bit constants for RV32");
779
780	// Sign extend for generateInstSeq.
781	Val = SignExtend64<32>(x: Val);
782	}
783
784	RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
785	assert(!Seq.empty());
786
787	bool SrcRenamable = false;
788	unsigned Num = 0;
789
790	for (const RISCVMatInt::Inst &Inst : Seq) {
791	bool LastItem = ++Num == Seq.size();
792	unsigned DstRegState = getDeadRegState(DstIsDead && LastItem) |
793	getRenamableRegState(DstRenamable);
794	unsigned SrcRegState = getKillRegState(SrcReg != RISCV::X0) |
795	getRenamableRegState(SrcRenamable);
796	switch (Inst.getOpndKind()) {
797	case RISCVMatInt::Imm:
798	BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
799	.addReg(DstReg, RegState::Define | DstRegState)
800	.addImm(Inst.getImm())
801	.setMIFlag(Flag);
802	break;
803	case RISCVMatInt::RegX0:
804	BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
805	.addReg(DstReg, RegState::Define | DstRegState)
806	.addReg(SrcReg, SrcRegState)
807	.addReg(RISCV::X0)
808	.setMIFlag(Flag);
809	break;
810	case RISCVMatInt::RegReg:
811	BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
812	.addReg(DstReg, RegState::Define | DstRegState)
813	.addReg(SrcReg, SrcRegState)
814	.addReg(SrcReg, SrcRegState)
815	.setMIFlag(Flag);
816	break;
817	case RISCVMatInt::RegImm:
818	BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
819	.addReg(DstReg, RegState::Define | DstRegState)
820	.addReg(SrcReg, SrcRegState)
821	.addImm(Inst.getImm())
822	.setMIFlag(Flag);
823	break;
824	}
825
826	// Only the first instruction has X0 as its source.
827	SrcReg = DstReg;
828	SrcRenamable = DstRenamable;
829	}
830	}
831
832	static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) {
833	switch (Opc) {
834	default:
835	return RISCVCC::COND_INVALID;
836	case RISCV::BEQ:
837	return RISCVCC::COND_EQ;
838	case RISCV::BNE:
839	return RISCVCC::COND_NE;
840	case RISCV::BLT:
841	return RISCVCC::COND_LT;
842	case RISCV::BGE:
843	return RISCVCC::COND_GE;
844	case RISCV::BLTU:
845	return RISCVCC::COND_LTU;
846	case RISCV::BGEU:
847	return RISCVCC::COND_GEU;
848	}
849	}
850
851	// The contents of values added to Cond are not examined outside of
852	// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
853	// push BranchOpcode, Reg1, Reg2.
854	static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
855	SmallVectorImpl<MachineOperand> &Cond) {
856	// Block ends with fall-through condbranch.
857	assert(LastInst.getDesc().isConditionalBranch() &&
858	"Unknown conditional branch");
859	Target = LastInst.getOperand(i: 2).getMBB();
860	unsigned CC = getCondFromBranchOpc(Opc: LastInst.getOpcode());
861	Cond.push_back(Elt: MachineOperand::CreateImm(Val: CC));
862	Cond.push_back(Elt: LastInst.getOperand(i: 0));
863	Cond.push_back(Elt: LastInst.getOperand(i: 1));
864	}
865
866	unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC) {
867	switch (CC) {
868	default:
869	llvm_unreachable("Unknown condition code!");
870	case RISCVCC::COND_EQ:
871	return RISCV::BEQ;
872	case RISCVCC::COND_NE:
873	return RISCV::BNE;
874	case RISCVCC::COND_LT:
875	return RISCV::BLT;
876	case RISCVCC::COND_GE:
877	return RISCV::BGE;
878	case RISCVCC::COND_LTU:
879	return RISCV::BLTU;
880	case RISCVCC::COND_GEU:
881	return RISCV::BGEU;
882	}
883	}
884
885	const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const {
886	return get(RISCVCC::getBrCond(CC));
887	}
888
889	RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
890	switch (CC) {
891	default:
892	llvm_unreachable("Unrecognized conditional branch");
893	case RISCVCC::COND_EQ:
894	return RISCVCC::COND_NE;
895	case RISCVCC::COND_NE:
896	return RISCVCC::COND_EQ;
897	case RISCVCC::COND_LT:
898	return RISCVCC::COND_GE;
899	case RISCVCC::COND_GE:
900	return RISCVCC::COND_LT;
901	case RISCVCC::COND_LTU:
902	return RISCVCC::COND_GEU;
903	case RISCVCC::COND_GEU:
904	return RISCVCC::COND_LTU;
905	}
906	}
907
908	bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
909	MachineBasicBlock *&TBB,
910	MachineBasicBlock *&FBB,
911	SmallVectorImpl<MachineOperand> &Cond,
912	bool AllowModify) const {
913	TBB = FBB = nullptr;
914	Cond.clear();
915
916	// If the block has no terminators, it just falls into the block after it.
917	MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
918	if (I == MBB.end() || !isUnpredicatedTerminator(*I))
919	return false;
920
921	// Count the number of terminators and find the first unconditional or
922	// indirect branch.
923	MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
924	int NumTerminators = 0;
925	for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
926	J++) {
927	NumTerminators++;
928	if (J->getDesc().isUnconditionalBranch() ||
929	J->getDesc().isIndirectBranch()) {
930	FirstUncondOrIndirectBr = J.getReverse();
931	}
932	}
933
934	// If AllowModify is true, we can erase any terminators after
935	// FirstUncondOrIndirectBR.
936	if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
937	while (std::next(x: FirstUncondOrIndirectBr) != MBB.end()) {
938	std::next(x: FirstUncondOrIndirectBr)->eraseFromParent();
939	NumTerminators--;
940	}
941	I = FirstUncondOrIndirectBr;
942	}
943
944	// We can't handle blocks that end in an indirect branch.
945	if (I->getDesc().isIndirectBranch())
946	return true;
947
948	// We can't handle Generic branch opcodes from Global ISel.
949	if (I->isPreISelOpcode())
950	return true;
951
952	// We can't handle blocks with more than 2 terminators.
953	if (NumTerminators > 2)
954	return true;
955
956	// Handle a single unconditional branch.
957	if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
958	TBB = getBranchDestBlock(MI: *I);
959	return false;
960	}
961
962	// Handle a single conditional branch.
963	if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
964	parseCondBranch(LastInst&: *I, Target&: TBB, Cond);
965	return false;
966	}
967
968	// Handle a conditional branch followed by an unconditional branch.
969	if (NumTerminators == 2 && std::prev(x: I)->getDesc().isConditionalBranch() &&
970	I->getDesc().isUnconditionalBranch()) {
971	parseCondBranch(LastInst&: *std::prev(x: I), Target&: TBB, Cond);
972	FBB = getBranchDestBlock(MI: *I);
973	return false;
974	}
975
976	// Otherwise, we can't handle this.
977	return true;
978	}
979
980	unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
981	int *BytesRemoved) const {
982	if (BytesRemoved)
983	*BytesRemoved = 0;
984	MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
985	if (I == MBB.end())
986	return 0;
987
988	if (!I->getDesc().isUnconditionalBranch() &&
989	!I->getDesc().isConditionalBranch())
990	return 0;
991
992	// Remove the branch.
993	if (BytesRemoved)
994	*BytesRemoved += getInstSizeInBytes(MI: *I);
995	I->eraseFromParent();
996
997	I = MBB.end();
998
999	if (I == MBB.begin())
1000	return 1;
1001	--I;
1002	if (!I->getDesc().isConditionalBranch())
1003	return 1;
1004
1005	// Remove the branch.
1006	if (BytesRemoved)
1007	*BytesRemoved += getInstSizeInBytes(MI: *I);
1008	I->eraseFromParent();
1009	return 2;
1010	}
1011
1012	// Inserts a branch into the end of the specific MachineBasicBlock, returning
1013	// the number of instructions inserted.
1014	unsigned RISCVInstrInfo::insertBranch(
1015	MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1016	ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1017	if (BytesAdded)
1018	*BytesAdded = 0;
1019
1020	// Shouldn't be a fall through.
1021	assert(TBB && "insertBranch must not be told to insert a fallthrough");
1022	assert((Cond.size() == 3 || Cond.size() == 0) &&
1023	"RISC-V branch conditions have two components!");
1024
1025	// Unconditional branch.
1026	if (Cond.empty()) {
1027	MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
1028	if (BytesAdded)
1029	*BytesAdded += getInstSizeInBytes(MI);
1030	return 1;
1031	}
1032
1033	// Either a one or two-way conditional branch.
1034	auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1035	MachineInstr &CondMI =
1036	*BuildMI(BB: &MBB, MIMD: DL, MCID: getBrCond(CC)).add(MO: Cond[1]).add(MO: Cond[2]).addMBB(MBB: TBB);
1037	if (BytesAdded)
1038	*BytesAdded += getInstSizeInBytes(MI: CondMI);
1039
1040	// One-way conditional branch.
1041	if (!FBB)
1042	return 1;
1043
1044	// Two-way conditional branch.
1045	MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
1046	if (BytesAdded)
1047	*BytesAdded += getInstSizeInBytes(MI);
1048	return 2;
1049	}
1050
1051	void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1052	MachineBasicBlock &DestBB,
1053	MachineBasicBlock &RestoreBB,
1054	const DebugLoc &DL, int64_t BrOffset,
1055	RegScavenger *RS) const {
1056	assert(RS && "RegScavenger required for long branching");
1057	assert(MBB.empty() &&
1058	"new block should be inserted for expanding unconditional branch");
1059	assert(MBB.pred_size() == 1);
1060	assert(RestoreBB.empty() &&
1061	"restore block should be inserted for restoring clobbered registers");
1062
1063	MachineFunction *MF = MBB.getParent();
1064	MachineRegisterInfo &MRI = MF->getRegInfo();
1065	RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1066	const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1067
1068	if (!isInt<32>(x: BrOffset))
1069	report_fatal_error(
1070	reason: "Branch offsets outside of the signed 32-bit range not supported");
1071
1072	// FIXME: A virtual register must be used initially, as the register
1073	// scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1074	// uses the same workaround).
1075	Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);
1076	auto II = MBB.end();
1077	// We may also update the jump target to RestoreBB later.
1078	MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
1079	.addReg(ScratchReg, RegState::Define | RegState::Dead)
1080	.addMBB(&DestBB, RISCVII::MO_CALL);
1081
1082	RS->enterBasicBlockEnd(MBB);
1083	Register TmpGPR =
1084	RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MI.getIterator(),
1085	/*RestoreAfter=*/false, /*SpAdj=*/0,
1086	/*AllowSpill=*/false);
1087	if (TmpGPR != RISCV::NoRegister)
1088	RS->setRegUsed(Reg: TmpGPR);
1089	else {
1090	// The case when there is no scavenged register needs special handling.
1091
1092	// Pick s11 because it doesn't make a difference.
1093	TmpGPR = RISCV::X27;
1094
1095	int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1096	if (FrameIndex == -1)
1097	report_fatal_error(reason: "underestimated function size");
1098
1099	storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,
1100	&RISCV::GPRRegClass, TRI, Register());
1101	TRI->eliminateFrameIndex(MI: std::prev(x: MI.getIterator()),
1102	/*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1103
1104	MI.getOperand(i: 1).setMBB(&RestoreBB);
1105
1106	loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,
1107	&RISCV::GPRRegClass, TRI, Register());
1108	TRI->eliminateFrameIndex(MI: RestoreBB.back(),
1109	/*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1110	}
1111
1112	MRI.replaceRegWith(FromReg: ScratchReg, ToReg: TmpGPR);
1113	MRI.clearVirtRegs();
1114	}
1115
1116	bool RISCVInstrInfo::reverseBranchCondition(
1117	SmallVectorImpl<MachineOperand> &Cond) const {
1118	assert((Cond.size() == 3) && "Invalid branch condition!");
1119	auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1120	Cond[0].setImm(getOppositeBranchCondition(CC));
1121	return false;
1122	}
1123
1124	bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
1125	MachineBasicBlock *MBB = MI.getParent();
1126	MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1127
1128	MachineBasicBlock *TBB, *FBB;
1129	SmallVector<MachineOperand, 3> Cond;
1130	if (analyzeBranch(MBB&: *MBB, TBB, FBB, Cond, /*AllowModify=*/false))
1131	return false;
1132
1133	RISCVCC::CondCode CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1134	assert(CC != RISCVCC::COND_INVALID);
1135
1136	if (CC == RISCVCC::COND_EQ || CC == RISCVCC::COND_NE)
1137	return false;
1138
1139	// For two constants C0 and C1 from
1140	// ```
1141	// li Y, C0
1142	// li Z, C1
1143	// ```
1144	// 1. if C1 = C0 + 1
1145	// we can turn:
1146	// (a) blt Y, X -> bge X, Z
1147	// (b) bge Y, X -> blt X, Z
1148	//
1149	// 2. if C1 = C0 - 1
1150	// we can turn:
1151	// (a) blt X, Y -> bge Z, X
1152	// (b) bge X, Y -> blt Z, X
1153	//
1154	// To make sure this optimization is really beneficial, we only
1155	// optimize for cases where Y had only one use (i.e. only used by the branch).
1156
1157	// Right now we only care about LI (i.e. ADDI x0, imm)
1158	auto isLoadImm = [](const MachineInstr *MI, int64_t &Imm) -> bool {
1159	if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1160	MI->getOperand(1).getReg() == RISCV::X0) {
1161	Imm = MI->getOperand(i: 2).getImm();
1162	return true;
1163	}
1164	return false;
1165	};
1166	// Either a load from immediate instruction or X0.
1167	auto isFromLoadImm = [&](const MachineOperand &Op, int64_t &Imm) -> bool {
1168	if (!Op.isReg())
1169	return false;
1170	Register Reg = Op.getReg();
1171	return Reg.isVirtual() && isLoadImm(MRI.getVRegDef(Reg), Imm);
1172	};
1173
1174	MachineOperand &LHS = MI.getOperand(i: 0);
1175	MachineOperand &RHS = MI.getOperand(i: 1);
1176	// Try to find the register for constant Z; return
1177	// invalid register otherwise.
1178	auto searchConst = [&](int64_t C1) -> Register {
1179	MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();
1180	auto DefC1 = std::find_if(first: ++II, last: E, pred: [&](const MachineInstr &I) -> bool {
1181	int64_t Imm;
1182	return isLoadImm(&I, Imm) && Imm == C1 &&
1183	I.getOperand(i: 0).getReg().isVirtual();
1184	});
1185	if (DefC1 != E)
1186	return DefC1->getOperand(i: 0).getReg();
1187
1188	return Register();
1189	};
1190
1191	bool Modify = false;
1192	int64_t C0;
1193	if (isFromLoadImm(LHS, C0) && MRI.hasOneUse(RegNo: LHS.getReg())) {
1194	// Might be case 1.
1195	// Signed integer overflow is UB. (UINT64_MAX is bigger so we don't need
1196	// to worry about unsigned overflow here)
1197	if (C0 < INT64_MAX)
1198	if (Register RegZ = searchConst(C0 + 1)) {
1199	reverseBranchCondition(Cond);
1200	Cond[1] = MachineOperand::CreateReg(Reg: RHS.getReg(), /*isDef=*/false);
1201	Cond[2] = MachineOperand::CreateReg(Reg: RegZ, /*isDef=*/false);
1202	// We might extend the live range of Z, clear its kill flag to
1203	// account for this.
1204	MRI.clearKillFlags(Reg: RegZ);
1205	Modify = true;
1206	}
1207	} else if (isFromLoadImm(RHS, C0) && MRI.hasOneUse(RegNo: RHS.getReg())) {
1208	// Might be case 2.
1209	// For unsigned cases, we don't want C1 to wrap back to UINT64_MAX
1210	// when C0 is zero.
1211	if ((CC == RISCVCC::COND_GE || CC == RISCVCC::COND_LT) || C0)
1212	if (Register RegZ = searchConst(C0 - 1)) {
1213	reverseBranchCondition(Cond);
1214	Cond[1] = MachineOperand::CreateReg(Reg: RegZ, /*isDef=*/false);
1215	Cond[2] = MachineOperand::CreateReg(Reg: LHS.getReg(), /*isDef=*/false);
1216	// We might extend the live range of Z, clear its kill flag to
1217	// account for this.
1218	MRI.clearKillFlags(Reg: RegZ);
1219	Modify = true;
1220	}
1221	}
1222
1223	if (!Modify)
1224	return false;
1225
1226	// Build the new branch and remove the old one.
1227	BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(),
1228	MCID: getBrCond(CC: static_cast<RISCVCC::CondCode>(Cond[0].getImm())))
1229	.add(MO: Cond[1])
1230	.add(MO: Cond[2])
1231	.addMBB(MBB: TBB);
1232	MI.eraseFromParent();
1233
1234	return true;
1235	}
1236
1237	MachineBasicBlock *
1238	RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1239	assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1240	// The branch target is always the last operand.
1241	int NumOp = MI.getNumExplicitOperands();
1242	return MI.getOperand(i: NumOp - 1).getMBB();
1243	}
1244
1245	bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1246	int64_t BrOffset) const {
1247	unsigned XLen = STI.getXLen();
1248	// Ideally we could determine the supported branch offset from the
1249	// RISCVII::FormMask, but this can't be used for Pseudo instructions like
1250	// PseudoBR.
1251	switch (BranchOp) {
1252	default:
1253	llvm_unreachable("Unexpected opcode!");
1254	case RISCV::BEQ:
1255	case RISCV::BNE:
1256	case RISCV::BLT:
1257	case RISCV::BGE:
1258	case RISCV::BLTU:
1259	case RISCV::BGEU:
1260	return isIntN(N: 13, x: BrOffset);
1261	case RISCV::JAL:
1262	case RISCV::PseudoBR:
1263	return isIntN(N: 21, x: BrOffset);
1264	case RISCV::PseudoJump:
1265	return isIntN(N: 32, x: SignExtend64(X: BrOffset + 0x800, B: XLen));
1266	}
1267	}
1268
1269	// If the operation has a predicated pseudo instruction, return the pseudo
1270	// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1271	// TODO: Support more operations.
1272	unsigned getPredicatedOpcode(unsigned Opcode) {
1273	switch (Opcode) {
1274	case RISCV::ADD: return RISCV::PseudoCCADD; break;
1275	case RISCV::SUB: return RISCV::PseudoCCSUB; break;
1276	case RISCV::SLL: return RISCV::PseudoCCSLL; break;
1277	case RISCV::SRL: return RISCV::PseudoCCSRL; break;
1278	case RISCV::SRA: return RISCV::PseudoCCSRA; break;
1279	case RISCV::AND: return RISCV::PseudoCCAND; break;
1280	case RISCV::OR: return RISCV::PseudoCCOR; break;
1281	case RISCV::XOR: return RISCV::PseudoCCXOR; break;
1282
1283	case RISCV::ADDI: return RISCV::PseudoCCADDI; break;
1284	case RISCV::SLLI: return RISCV::PseudoCCSLLI; break;
1285	case RISCV::SRLI: return RISCV::PseudoCCSRLI; break;
1286	case RISCV::SRAI: return RISCV::PseudoCCSRAI; break;
1287	case RISCV::ANDI: return RISCV::PseudoCCANDI; break;
1288	case RISCV::ORI: return RISCV::PseudoCCORI; break;
1289	case RISCV::XORI: return RISCV::PseudoCCXORI; break;
1290
1291	case RISCV::ADDW: return RISCV::PseudoCCADDW; break;
1292	case RISCV::SUBW: return RISCV::PseudoCCSUBW; break;
1293	case RISCV::SLLW: return RISCV::PseudoCCSLLW; break;
1294	case RISCV::SRLW: return RISCV::PseudoCCSRLW; break;
1295	case RISCV::SRAW: return RISCV::PseudoCCSRAW; break;
1296
1297	case RISCV::ADDIW: return RISCV::PseudoCCADDIW; break;
1298	case RISCV::SLLIW: return RISCV::PseudoCCSLLIW; break;
1299	case RISCV::SRLIW: return RISCV::PseudoCCSRLIW; break;
1300	case RISCV::SRAIW: return RISCV::PseudoCCSRAIW; break;
1301
1302	case RISCV::ANDN: return RISCV::PseudoCCANDN; break;
1303	case RISCV::ORN: return RISCV::PseudoCCORN; break;
1304	case RISCV::XNOR: return RISCV::PseudoCCXNOR; break;
1305	}
1306
1307	return RISCV::INSTRUCTION_LIST_END;
1308	}
1309
1310	/// Identify instructions that can be folded into a CCMOV instruction, and
1311	/// return the defining instruction.
1312	static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1313	const MachineRegisterInfo &MRI,
1314	const TargetInstrInfo *TII) {
1315	if (!Reg.isVirtual())
1316	return nullptr;
1317	if (!MRI.hasOneNonDBGUse(RegNo: Reg))
1318	return nullptr;
1319	MachineInstr *MI = MRI.getVRegDef(Reg);
1320	if (!MI)
1321	return nullptr;
1322	// Check if MI can be predicated and folded into the CCMOV.
1323	if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1324	return nullptr;
1325	// Don't predicate li idiom.
1326	if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1327	MI->getOperand(1).getReg() == RISCV::X0)
1328	return nullptr;
1329	// Check if MI has any other defs or physreg uses.
1330	for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
1331	// Reject frame index operands, PEI can't handle the predicated pseudos.
1332	if (MO.isFI() || MO.isCPI() || MO.isJTI())
1333	return nullptr;
1334	if (!MO.isReg())
1335	continue;
1336	// MI can't have any tied operands, that would conflict with predication.
1337	if (MO.isTied())
1338	return nullptr;
1339	if (MO.isDef())
1340	return nullptr;
1341	// Allow constant physregs.
1342	if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(PhysReg: MO.getReg()))
1343	return nullptr;
1344	}
1345	bool DontMoveAcrossStores = true;
1346	if (!MI->isSafeToMove(/* AliasAnalysis = */ AA: nullptr, SawStore&: DontMoveAcrossStores))
1347	return nullptr;
1348	return MI;
1349	}
1350
1351	bool RISCVInstrInfo::analyzeSelect(const MachineInstr &MI,
1352	SmallVectorImpl<MachineOperand> &Cond,
1353	unsigned &TrueOp, unsigned &FalseOp,
1354	bool &Optimizable) const {
1355	assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1356	"Unknown select instruction");
1357	// CCMOV operands:
1358	// 0: Def.
1359	// 1: LHS of compare.
1360	// 2: RHS of compare.
1361	// 3: Condition code.
1362	// 4: False use.
1363	// 5: True use.
1364	TrueOp = 5;
1365	FalseOp = 4;
1366	Cond.push_back(Elt: MI.getOperand(i: 1));
1367	Cond.push_back(Elt: MI.getOperand(i: 2));
1368	Cond.push_back(Elt: MI.getOperand(i: 3));
1369	// We can only fold when we support short forward branch opt.
1370	Optimizable = STI.hasShortForwardBranchOpt();
1371	return false;
1372	}
1373
1374	MachineInstr *
1375	RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
1376	SmallPtrSetImpl<MachineInstr *> &SeenMIs,
1377	bool PreferFalse) const {
1378	assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1379	"Unknown select instruction");
1380	if (!STI.hasShortForwardBranchOpt())
1381	return nullptr;
1382
1383	MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1384	MachineInstr *DefMI =
1385	canFoldAsPredicatedOp(MI.getOperand(i: 5).getReg(), MRI, this);
1386	bool Invert = !DefMI;
1387	if (!DefMI)
1388	DefMI = canFoldAsPredicatedOp(MI.getOperand(i: 4).getReg(), MRI, this);
1389	if (!DefMI)
1390	return nullptr;
1391
1392	// Find new register class to use.
1393	MachineOperand FalseReg = MI.getOperand(i: Invert ? 5 : 4);
1394	Register DestReg = MI.getOperand(i: 0).getReg();
1395	const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
1396	if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
1397	return nullptr;
1398
1399	unsigned PredOpc = getPredicatedOpcode(Opcode: DefMI->getOpcode());
1400	assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1401
1402	// Create a new predicated version of DefMI.
1403	MachineInstrBuilder NewMI =
1404	BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1405
1406	// Copy the condition portion.
1407	NewMI.add(MO: MI.getOperand(i: 1));
1408	NewMI.add(MO: MI.getOperand(i: 2));
1409
1410	// Add condition code, inverting if necessary.
1411	auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
1412	if (Invert)
1413	CC = RISCVCC::getOppositeBranchCondition(CC);
1414	NewMI.addImm(Val: CC);
1415
1416	// Copy the false register.
1417	NewMI.add(MO: FalseReg);
1418
1419	// Copy all the DefMI operands.
1420	const MCInstrDesc &DefDesc = DefMI->getDesc();
1421	for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1422	NewMI.add(MO: DefMI->getOperand(i));
1423
1424	// Update SeenMIs set: register newly created MI and erase removed DefMI.
1425	SeenMIs.insert(Ptr: NewMI);
1426	SeenMIs.erase(Ptr: DefMI);
1427
1428	// If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1429	// DefMI would be invalid when tranferred inside the loop. Checking for a
1430	// loop is expensive, but at least remove kill flags if they are in different
1431	// BBs.
1432	if (DefMI->getParent() != MI.getParent())
1433	NewMI->clearKillInfo();
1434
1435	// The caller will erase MI, but not DefMI.
1436	DefMI->eraseFromParent();
1437	return NewMI;
1438	}
1439
1440	unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1441	if (MI.isMetaInstruction())
1442	return 0;
1443
1444	unsigned Opcode = MI.getOpcode();
1445
1446	if (Opcode == TargetOpcode::INLINEASM ||
1447	Opcode == TargetOpcode::INLINEASM_BR) {
1448	const MachineFunction &MF = *MI.getParent()->getParent();
1449	const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget());
1450	return getInlineAsmLength(MI.getOperand(i: 0).getSymbolName(),
1451	*TM.getMCAsmInfo());
1452	}
1453
1454	if (!MI.memoperands_empty()) {
1455	MachineMemOperand *MMO = *(MI.memoperands_begin());
1456	const MachineFunction &MF = *MI.getParent()->getParent();
1457	const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1458	if (ST.hasStdExtZihintntl() && MMO->isNonTemporal()) {
1459	if (ST.hasStdExtCOrZca() && ST.enableRVCHintInstrs()) {
1460	if (isCompressibleInst(MI, STI))
1461	return 4; // c.ntl.all + c.load/c.store
1462	return 6; // c.ntl.all + load/store
1463	}
1464	return 8; // ntl.all + load/store
1465	}
1466	}
1467
1468	if (Opcode == TargetOpcode::BUNDLE)
1469	return getInstBundleLength(MI);
1470
1471	if (MI.getParent() && MI.getParent()->getParent()) {
1472	if (isCompressibleInst(MI, STI))
1473	return 2;
1474	}
1475
1476	switch (Opcode) {
1477	case TargetOpcode::STACKMAP:
1478	// The upper bound for a stackmap intrinsic is the full length of its shadow
1479	return StackMapOpers(&MI).getNumPatchBytes();
1480	case TargetOpcode::PATCHPOINT:
1481	// The size of the patchpoint intrinsic is the number of bytes requested
1482	return PatchPointOpers(&MI).getNumPatchBytes();
1483	case TargetOpcode::STATEPOINT: {
1484	// The size of the statepoint intrinsic is the number of bytes requested
1485	unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
1486	// No patch bytes means at most a PseudoCall is emitted
1487	return std::max(a: NumBytes, b: 8U);
1488	}
1489	default:
1490	return get(Opcode).getSize();
1491	}
1492	}
1493
1494	unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
1495	unsigned Size = 0;
1496	MachineBasicBlock::const_instr_iterator I = MI.getIterator();
1497	MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
1498	while (++I != E && I->isInsideBundle()) {
1499	assert(!I->isBundle() && "No nested bundle!");
1500	Size += getInstSizeInBytes(MI: *I);
1501	}
1502	return Size;
1503	}
1504
1505	bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1506	const unsigned Opcode = MI.getOpcode();
1507	switch (Opcode) {
1508	default:
1509	break;
1510	case RISCV::FSGNJ_D:
1511	case RISCV::FSGNJ_S:
1512	case RISCV::FSGNJ_H:
1513	case RISCV::FSGNJ_D_INX:
1514	case RISCV::FSGNJ_D_IN32X:
1515	case RISCV::FSGNJ_S_INX:
1516	case RISCV::FSGNJ_H_INX:
1517	// The canonical floating-point move is fsgnj rd, rs, rs.
1518	return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
1519	MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
1520	case RISCV::ADDI:
1521	case RISCV::ORI:
1522	case RISCV::XORI:
1523	return (MI.getOperand(1).isReg() &&
1524	MI.getOperand(1).getReg() == RISCV::X0) ||
1525	(MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1526	}
1527	return MI.isAsCheapAsAMove();
1528	}
1529
1530	std::optional<DestSourcePair>
1531	RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
1532	if (MI.isMoveReg())
1533	return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
1534	switch (MI.getOpcode()) {
1535	default:
1536	break;
1537	case RISCV::ADDI:
1538	// Operand 1 can be a frameindex but callers expect registers
1539	if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isImm() &&
1540	MI.getOperand(i: 2).getImm() == 0)
1541	return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
1542	break;
1543	case RISCV::FSGNJ_D:
1544	case RISCV::FSGNJ_S:
1545	case RISCV::FSGNJ_H:
1546	case RISCV::FSGNJ_D_INX:
1547	case RISCV::FSGNJ_D_IN32X:
1548	case RISCV::FSGNJ_S_INX:
1549	case RISCV::FSGNJ_H_INX:
1550	// The canonical floating-point move is fsgnj rd, rs, rs.
1551	if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
1552	MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
1553	return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
1554	break;
1555	}
1556	return std::nullopt;
1557	}
1558
1559	MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {
1560	if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {
1561	// The option is unused. Choose Local strategy only for in-order cores. When
1562	// scheduling model is unspecified, use MinInstrCount strategy as more
1563	// generic one.
1564	const auto &SchedModel = STI.getSchedModel();
1565	return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
1566	? MachineTraceStrategy::TS_MinInstrCount
1567	: MachineTraceStrategy::TS_Local;
1568	}
1569	// The strategy was forced by the option.
1570	return ForceMachineCombinerStrategy;
1571	}
1572
1573	void RISCVInstrInfo::finalizeInsInstrs(
1574	MachineInstr &Root, unsigned &Pattern,
1575	SmallVectorImpl<MachineInstr *> &InsInstrs) const {
1576	int16_t FrmOpIdx =
1577	RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);
1578	if (FrmOpIdx < 0) {
1579	assert(all_of(InsInstrs,
1580	[](MachineInstr *MI) {
1581	return RISCV::getNamedOperandIdx(MI->getOpcode(),
1582	RISCV::OpName::frm) < 0;
1583	}) &&
1584	"New instructions require FRM whereas the old one does not have it");
1585	return;
1586	}
1587
1588	const MachineOperand &FRM = Root.getOperand(i: FrmOpIdx);
1589	MachineFunction &MF = *Root.getMF();
1590
1591	for (auto *NewMI : InsInstrs) {
1592	// We'd already added the FRM operand.
1593	if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
1594	NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())
1595	continue;
1596	MachineInstrBuilder MIB(MF, NewMI);
1597	MIB.add(MO: FRM);
1598	if (FRM.getImm() == RISCVFPRndMode::DYN)
1599	MIB.addUse(RISCV::FRM, RegState::Implicit);
1600	}
1601	}
1602
1603	static bool isFADD(unsigned Opc) {
1604	switch (Opc) {
1605	default:
1606	return false;
1607	case RISCV::FADD_H:
1608	case RISCV::FADD_S:
1609	case RISCV::FADD_D:
1610	return true;
1611	}
1612	}
1613
1614	static bool isFSUB(unsigned Opc) {
1615	switch (Opc) {
1616	default:
1617	return false;
1618	case RISCV::FSUB_H:
1619	case RISCV::FSUB_S:
1620	case RISCV::FSUB_D:
1621	return true;
1622	}
1623	}
1624
1625	static bool isFMUL(unsigned Opc) {
1626	switch (Opc) {
1627	default:
1628	return false;
1629	case RISCV::FMUL_H:
1630	case RISCV::FMUL_S:
1631	case RISCV::FMUL_D:
1632	return true;
1633	}
1634	}
1635
1636	bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
1637	bool &Commuted) const {
1638	if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
1639	return false;
1640
1641	const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
1642	unsigned OperandIdx = Commuted ? 2 : 1;
1643	const MachineInstr &Sibling =
1644	*MRI.getVRegDef(Reg: Inst.getOperand(i: OperandIdx).getReg());
1645
1646	int16_t InstFrmOpIdx =
1647	RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);
1648	int16_t SiblingFrmOpIdx =
1649	RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);
1650
1651	return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
1652	RISCV::hasEqualFRM(MI1: Inst, MI2: Sibling);
1653	}
1654
1655	bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
1656	bool Invert) const {
1657	unsigned Opc = Inst.getOpcode();
1658	if (Invert) {
1659	auto InverseOpcode = getInverseOpcode(Opcode: Opc);
1660	if (!InverseOpcode)
1661	return false;
1662	Opc = *InverseOpcode;
1663	}
1664
1665	if (isFADD(Opc) || isFMUL(Opc))
1666	return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
1667	Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
1668
1669	switch (Opc) {
1670	default:
1671	return false;
1672	case RISCV::ADD:
1673	case RISCV::ADDW:
1674	case RISCV::AND:
1675	case RISCV::OR:
1676	case RISCV::XOR:
1677	// From RISC-V ISA spec, if both the high and low bits of the same product
1678	// are required, then the recommended code sequence is:
1679	//
1680	// MULH[[S]U] rdh, rs1, rs2
1681	// MUL rdl, rs1, rs2
1682	// (source register specifiers must be in same order and rdh cannot be the
1683	// same as rs1 or rs2)
1684	//
1685	// Microarchitectures can then fuse these into a single multiply operation
1686	// instead of performing two separate multiplies.
1687	// MachineCombiner may reassociate MUL operands and lose the fusion
1688	// opportunity.
1689	case RISCV::MUL:
1690	case RISCV::MULW:
1691	case RISCV::MIN:
1692	case RISCV::MINU:
1693	case RISCV::MAX:
1694	case RISCV::MAXU:
1695	case RISCV::FMIN_H:
1696	case RISCV::FMIN_S:
1697	case RISCV::FMIN_D:
1698	case RISCV::FMAX_H:
1699	case RISCV::FMAX_S:
1700	case RISCV::FMAX_D:
1701	return true;
1702	}
1703
1704	return false;
1705	}
1706
1707	std::optional<unsigned>
1708	RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
1709	switch (Opcode) {
1710	default:
1711	return std::nullopt;
1712	case RISCV::FADD_H:
1713	return RISCV::FSUB_H;
1714	case RISCV::FADD_S:
1715	return RISCV::FSUB_S;
1716	case RISCV::FADD_D:
1717	return RISCV::FSUB_D;
1718	case RISCV::FSUB_H:
1719	return RISCV::FADD_H;
1720	case RISCV::FSUB_S:
1721	return RISCV::FADD_S;
1722	case RISCV::FSUB_D:
1723	return RISCV::FADD_D;
1724	case RISCV::ADD:
1725	return RISCV::SUB;
1726	case RISCV::SUB:
1727	return RISCV::ADD;
1728	case RISCV::ADDW:
1729	return RISCV::SUBW;
1730	case RISCV::SUBW:
1731	return RISCV::ADDW;
1732	}
1733	}
1734
1735	static bool canCombineFPFusedMultiply(const MachineInstr &Root,
1736	const MachineOperand &MO,
1737	bool DoRegPressureReduce) {
1738	if (!MO.isReg() || !MO.getReg().isVirtual())
1739	return false;
1740	const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1741	MachineInstr *MI = MRI.getVRegDef(Reg: MO.getReg());
1742	if (!MI || !isFMUL(Opc: MI->getOpcode()))
1743	return false;
1744
1745	if (!Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) ||
1746	!MI->getFlag(Flag: MachineInstr::MIFlag::FmContract))
1747	return false;
1748
1749	// Try combining even if fmul has more than one use as it eliminates
1750	// dependency between fadd(fsub) and fmul. However, it can extend liveranges
1751	// for fmul operands, so reject the transformation in register pressure
1752	// reduction mode.
1753	if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
1754	return false;
1755
1756	// Do not combine instructions from different basic blocks.
1757	if (Root.getParent() != MI->getParent())
1758	return false;
1759	return RISCV::hasEqualFRM(MI1: Root, MI2: *MI);
1760	}
1761
1762	static bool getFPFusedMultiplyPatterns(MachineInstr &Root,
1763	SmallVectorImpl<unsigned> &Patterns,
1764	bool DoRegPressureReduce) {
1765	unsigned Opc = Root.getOpcode();
1766	bool IsFAdd = isFADD(Opc);
1767	if (!IsFAdd && !isFSUB(Opc))
1768	return false;
1769	bool Added = false;
1770	if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 1),
1771	DoRegPressureReduce)) {
1772	Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX
1773	: RISCVMachineCombinerPattern::FMSUB);
1774	Added = true;
1775	}
1776	if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 2),
1777	DoRegPressureReduce)) {
1778	Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA
1779	: RISCVMachineCombinerPattern::FNMSUB);
1780	Added = true;
1781	}
1782	return Added;
1783	}
1784
1785	static bool getFPPatterns(MachineInstr &Root,
1786	SmallVectorImpl<unsigned> &Patterns,
1787	bool DoRegPressureReduce) {
1788	return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
1789	}
1790
1791	/// Utility routine that checks if \param MO is defined by an
1792	/// \param CombineOpc instruction in the basic block \param MBB
1793	static const MachineInstr *canCombine(const MachineBasicBlock &MBB,
1794	const MachineOperand &MO,
1795	unsigned CombineOpc) {
1796	const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1797	const MachineInstr *MI = nullptr;
1798
1799	if (MO.isReg() && MO.getReg().isVirtual())
1800	MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
1801	// And it needs to be in the trace (otherwise, it won't have a depth).
1802	if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
1803	return nullptr;
1804	// Must only used by the user we combine with.
1805	if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
1806	return nullptr;
1807
1808	return MI;
1809	}
1810
1811	/// Utility routine that checks if \param MO is defined by a SLLI in \param
1812	/// MBB that can be combined by splitting across 2 SHXADD instructions. The
1813	/// first SHXADD shift amount is given by \param OuterShiftAmt.
1814	static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,
1815	const MachineOperand &MO,
1816	unsigned OuterShiftAmt) {
1817	const MachineInstr *ShiftMI = canCombine(MBB, MO, RISCV::SLLI);
1818	if (!ShiftMI)
1819	return false;
1820
1821	unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
1822	if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
1823	return false;
1824
1825	return true;
1826	}
1827
1828	// Returns the shift amount from a SHXADD instruction. Returns 0 if the
1829	// instruction is not a SHXADD.
1830	static unsigned getSHXADDShiftAmount(unsigned Opc) {
1831	switch (Opc) {
1832	default:
1833	return 0;
1834	case RISCV::SH1ADD:
1835	return 1;
1836	case RISCV::SH2ADD:
1837	return 2;
1838	case RISCV::SH3ADD:
1839	return 3;
1840	}
1841	}
1842
1843	// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
1844	// (sh3add (sh2add Y, Z), X).
1845	static bool getSHXADDPatterns(const MachineInstr &Root,
1846	SmallVectorImpl<unsigned> &Patterns) {
1847	unsigned ShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
1848	if (!ShiftAmt)
1849	return false;
1850
1851	const MachineBasicBlock &MBB = *Root.getParent();
1852
1853	const MachineInstr *AddMI = canCombine(MBB, Root.getOperand(2), RISCV::ADD);
1854	if (!AddMI)
1855	return false;
1856
1857	bool Found = false;
1858	if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 1), OuterShiftAmt: ShiftAmt)) {
1859	Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);
1860	Found = true;
1861	}
1862	if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 2), OuterShiftAmt: ShiftAmt)) {
1863	Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);
1864	Found = true;
1865	}
1866
1867	return Found;
1868	}
1869
1870	CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
1871	switch (Pattern) {
1872	case RISCVMachineCombinerPattern::FMADD_AX:
1873	case RISCVMachineCombinerPattern::FMADD_XA:
1874	case RISCVMachineCombinerPattern::FMSUB:
1875	case RISCVMachineCombinerPattern::FNMSUB:
1876	return CombinerObjective::MustReduceDepth;
1877	default:
1878	return TargetInstrInfo::getCombinerObjective(Pattern);
1879	}
1880	}
1881
1882	bool RISCVInstrInfo::getMachineCombinerPatterns(
1883	MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
1884	bool DoRegPressureReduce) const {
1885
1886	if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
1887	return true;
1888
1889	if (getSHXADDPatterns(Root, Patterns))
1890	return true;
1891
1892	return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
1893	DoRegPressureReduce);
1894	}
1895
1896	static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
1897	switch (RootOpc) {
1898	default:
1899	llvm_unreachable("Unexpected opcode");
1900	case RISCV::FADD_H:
1901	return RISCV::FMADD_H;
1902	case RISCV::FADD_S:
1903	return RISCV::FMADD_S;
1904	case RISCV::FADD_D:
1905	return RISCV::FMADD_D;
1906	case RISCV::FSUB_H:
1907	return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
1908	: RISCV::FNMSUB_H;
1909	case RISCV::FSUB_S:
1910	return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
1911	: RISCV::FNMSUB_S;
1912	case RISCV::FSUB_D:
1913	return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
1914	: RISCV::FNMSUB_D;
1915	}
1916	}
1917
1918	static unsigned getAddendOperandIdx(unsigned Pattern) {
1919	switch (Pattern) {
1920	default:
1921	llvm_unreachable("Unexpected pattern");
1922	case RISCVMachineCombinerPattern::FMADD_AX:
1923	case RISCVMachineCombinerPattern::FMSUB:
1924	return 2;
1925	case RISCVMachineCombinerPattern::FMADD_XA:
1926	case RISCVMachineCombinerPattern::FNMSUB:
1927	return 1;
1928	}
1929	}
1930
1931	static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
1932	unsigned Pattern,
1933	SmallVectorImpl<MachineInstr *> &InsInstrs,
1934	SmallVectorImpl<MachineInstr *> &DelInstrs) {
1935	MachineFunction *MF = Root.getMF();
1936	MachineRegisterInfo &MRI = MF->getRegInfo();
1937	const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1938
1939	MachineOperand &Mul1 = Prev.getOperand(i: 1);
1940	MachineOperand &Mul2 = Prev.getOperand(i: 2);
1941	MachineOperand &Dst = Root.getOperand(i: 0);
1942	MachineOperand &Addend = Root.getOperand(i: getAddendOperandIdx(Pattern));
1943
1944	Register DstReg = Dst.getReg();
1945	unsigned FusedOpc = getFPFusedMultiplyOpcode(RootOpc: Root.getOpcode(), Pattern);
1946	uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
1947	DebugLoc MergedLoc =
1948	DILocation::getMergedLocation(LocA: Root.getDebugLoc(), LocB: Prev.getDebugLoc());
1949
1950	bool Mul1IsKill = Mul1.isKill();
1951	bool Mul2IsKill = Mul2.isKill();
1952	bool AddendIsKill = Addend.isKill();
1953
1954	// We need to clear kill flags since we may be extending the live range past
1955	// a kill. If the mul had kill flags, we can preserve those since we know
1956	// where the previous range stopped.
1957	MRI.clearKillFlags(Reg: Mul1.getReg());
1958	MRI.clearKillFlags(Reg: Mul2.getReg());
1959
1960	MachineInstrBuilder MIB =
1961	BuildMI(MF&: *MF, MIMD: MergedLoc, MCID: TII->get(Opcode: FusedOpc), DestReg: DstReg)
1962	.addReg(RegNo: Mul1.getReg(), flags: getKillRegState(B: Mul1IsKill))
1963	.addReg(RegNo: Mul2.getReg(), flags: getKillRegState(B: Mul2IsKill))
1964	.addReg(RegNo: Addend.getReg(), flags: getKillRegState(B: AddendIsKill))
1965	.setMIFlags(IntersectedFlags);
1966
1967	InsInstrs.push_back(Elt: MIB);
1968	if (MRI.hasOneNonDBGUse(RegNo: Prev.getOperand(i: 0).getReg()))
1969	DelInstrs.push_back(Elt: &Prev);
1970	DelInstrs.push_back(Elt: &Root);
1971	}
1972
1973	// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
1974	// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
1975	// shXadd instructions. The outer shXadd keeps its original opcode.
1976	static void
1977	genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
1978	SmallVectorImpl<MachineInstr *> &InsInstrs,
1979	SmallVectorImpl<MachineInstr *> &DelInstrs,
1980	DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) {
1981	MachineFunction *MF = Root.getMF();
1982	MachineRegisterInfo &MRI = MF->getRegInfo();
1983	const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1984
1985	unsigned OuterShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
1986	assert(OuterShiftAmt != 0 && "Unexpected opcode");
1987
1988	MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
1989	MachineInstr *ShiftMI =
1990	MRI.getUniqueVRegDef(Reg: AddMI->getOperand(i: AddOpIdx).getReg());
1991
1992	unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
1993	assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
1994
1995	unsigned InnerOpc;
1996	switch (InnerShiftAmt - OuterShiftAmt) {
1997	default:
1998	llvm_unreachable("Unexpected shift amount");
1999	case 0:
2000	InnerOpc = RISCV::ADD;
2001	break;
2002	case 1:
2003	InnerOpc = RISCV::SH1ADD;
2004	break;
2005	case 2:
2006	InnerOpc = RISCV::SH2ADD;
2007	break;
2008	case 3:
2009	InnerOpc = RISCV::SH3ADD;
2010	break;
2011	}
2012
2013	const MachineOperand &X = AddMI->getOperand(i: 3 - AddOpIdx);
2014	const MachineOperand &Y = ShiftMI->getOperand(i: 1);
2015	const MachineOperand &Z = Root.getOperand(i: 1);
2016
2017	Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2018
2019	auto MIB1 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: InnerOpc), DestReg: NewVR)
2020	.addReg(RegNo: Y.getReg(), flags: getKillRegState(B: Y.isKill()))
2021	.addReg(RegNo: Z.getReg(), flags: getKillRegState(B: Z.isKill()));
2022	auto MIB2 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Root.getOpcode()),
2023	DestReg: Root.getOperand(i: 0).getReg())
2024	.addReg(RegNo: NewVR, flags: RegState::Kill)
2025	.addReg(RegNo: X.getReg(), flags: getKillRegState(B: X.isKill()));
2026
2027	InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
2028	InsInstrs.push_back(Elt: MIB1);
2029	InsInstrs.push_back(Elt: MIB2);
2030	DelInstrs.push_back(Elt: ShiftMI);
2031	DelInstrs.push_back(Elt: AddMI);
2032	DelInstrs.push_back(Elt: &Root);
2033	}
2034
2035	void RISCVInstrInfo::genAlternativeCodeSequence(
2036	MachineInstr &Root, unsigned Pattern,
2037	SmallVectorImpl<MachineInstr *> &InsInstrs,
2038	SmallVectorImpl<MachineInstr *> &DelInstrs,
2039	DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
2040	MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2041	switch (Pattern) {
2042	default:
2043	TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
2044	DelInstrs, InstrIdxForVirtReg);
2045	return;
2046	case RISCVMachineCombinerPattern::FMADD_AX:
2047	case RISCVMachineCombinerPattern::FMSUB: {
2048	MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 1).getReg());
2049	combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2050	return;
2051	}
2052	case RISCVMachineCombinerPattern::FMADD_XA:
2053	case RISCVMachineCombinerPattern::FNMSUB: {
2054	MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 2).getReg());
2055	combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2056	return;
2057	}
2058	case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:
2059	genShXAddAddShift(Root, AddOpIdx: 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2060	return;
2061	case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:
2062	genShXAddAddShift(Root, AddOpIdx: 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2063	return;
2064	}
2065	}
2066
2067	bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
2068	StringRef &ErrInfo) const {
2069	MCInstrDesc const &Desc = MI.getDesc();
2070
2071	for (const auto &[Index, Operand] : enumerate(First: Desc.operands())) {
2072	unsigned OpType = Operand.OperandType;
2073	if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
2074	OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
2075	const MachineOperand &MO = MI.getOperand(i: Index);
2076	if (MO.isImm()) {
2077	int64_t Imm = MO.getImm();
2078	bool Ok;
2079	switch (OpType) {
2080	default:
2081	llvm_unreachable("Unexpected operand type");
2082
2083	// clang-format off
2084	#define CASE_OPERAND_UIMM(NUM) \
2085	case RISCVOp::OPERAND_UIMM##NUM: \
2086	Ok = isUInt<NUM>(Imm); \
2087	break;
2088	CASE_OPERAND_UIMM(1)
2089	CASE_OPERAND_UIMM(2)
2090	CASE_OPERAND_UIMM(3)
2091	CASE_OPERAND_UIMM(4)
2092	CASE_OPERAND_UIMM(5)
2093	CASE_OPERAND_UIMM(6)
2094	CASE_OPERAND_UIMM(7)
2095	CASE_OPERAND_UIMM(8)
2096	CASE_OPERAND_UIMM(12)
2097	CASE_OPERAND_UIMM(20)
2098	// clang-format on
2099	case RISCVOp::OPERAND_UIMM2_LSB0:
2100	Ok = isShiftedUInt<1, 1>(x: Imm);
2101	break;
2102	case RISCVOp::OPERAND_UIMM7_LSB00:
2103	Ok = isShiftedUInt<5, 2>(x: Imm);
2104	break;
2105	case RISCVOp::OPERAND_UIMM8_LSB00:
2106	Ok = isShiftedUInt<6, 2>(x: Imm);
2107	break;
2108	case RISCVOp::OPERAND_UIMM8_LSB000:
2109	Ok = isShiftedUInt<5, 3>(x: Imm);
2110	break;
2111	case RISCVOp::OPERAND_UIMM8_GE32:
2112	Ok = isUInt<8>(x: Imm) && Imm >= 32;
2113	break;
2114	case RISCVOp::OPERAND_UIMM9_LSB000:
2115	Ok = isShiftedUInt<6, 3>(x: Imm);
2116	break;
2117	case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
2118	Ok = isShiftedInt<6, 4>(x: Imm) && (Imm != 0);
2119	break;
2120	case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:
2121	Ok = isShiftedUInt<8, 2>(x: Imm) && (Imm != 0);
2122	break;
2123	case RISCVOp::OPERAND_ZERO:
2124	Ok = Imm == 0;
2125	break;
2126	case RISCVOp::OPERAND_SIMM5:
2127	Ok = isInt<5>(x: Imm);
2128	break;
2129	case RISCVOp::OPERAND_SIMM5_PLUS1:
2130	Ok = (isInt<5>(x: Imm) && Imm != -16) || Imm == 16;
2131	break;
2132	case RISCVOp::OPERAND_SIMM6:
2133	Ok = isInt<6>(x: Imm);
2134	break;
2135	case RISCVOp::OPERAND_SIMM6_NONZERO:
2136	Ok = Imm != 0 && isInt<6>(x: Imm);
2137	break;
2138	case RISCVOp::OPERAND_VTYPEI10:
2139	Ok = isUInt<10>(x: Imm);
2140	break;
2141	case RISCVOp::OPERAND_VTYPEI11:
2142	Ok = isUInt<11>(x: Imm);
2143	break;
2144	case RISCVOp::OPERAND_SIMM12:
2145	Ok = isInt<12>(x: Imm);
2146	break;
2147	case RISCVOp::OPERAND_SIMM12_LSB00000:
2148	Ok = isShiftedInt<7, 5>(x: Imm);
2149	break;
2150	case RISCVOp::OPERAND_UIMMLOG2XLEN:
2151	Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
2152	break;
2153	case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
2154	Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
2155	Ok = Ok && Imm != 0;
2156	break;
2157	case RISCVOp::OPERAND_CLUI_IMM:
2158	Ok = (isUInt<5>(x: Imm) && Imm != 0) ||
2159	(Imm >= 0xfffe0 && Imm <= 0xfffff);
2160	break;
2161	case RISCVOp::OPERAND_RVKRNUM:
2162	Ok = Imm >= 0 && Imm <= 10;
2163	break;
2164	case RISCVOp::OPERAND_RVKRNUM_0_7:
2165	Ok = Imm >= 0 && Imm <= 7;
2166	break;
2167	case RISCVOp::OPERAND_RVKRNUM_1_10:
2168	Ok = Imm >= 1 && Imm <= 10;
2169	break;
2170	case RISCVOp::OPERAND_RVKRNUM_2_14:
2171	Ok = Imm >= 2 && Imm <= 14;
2172	break;
2173	case RISCVOp::OPERAND_SPIMM:
2174	Ok = (Imm & 0xf) == 0;
2175	break;
2176	}
2177	if (!Ok) {
2178	ErrInfo = "Invalid immediate";
2179	return false;
2180	}
2181	}
2182	}
2183	}
2184
2185	const uint64_t TSFlags = Desc.TSFlags;
2186	if (RISCVII::hasVLOp(TSFlags)) {
2187	const MachineOperand &Op = MI.getOperand(i: RISCVII::getVLOpNum(Desc));
2188	if (!Op.isImm() && !Op.isReg()) {
2189	ErrInfo = "Invalid operand type for VL operand";
2190	return false;
2191	}
2192	if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
2193	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2194	auto *RC = MRI.getRegClass(Reg: Op.getReg());
2195	if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
2196	ErrInfo = "Invalid register class for VL operand";
2197	return false;
2198	}
2199	}
2200	if (!RISCVII::hasSEWOp(TSFlags)) {
2201	ErrInfo = "VL operand w/o SEW operand?";
2202	return false;
2203	}
2204	}
2205	if (RISCVII::hasSEWOp(TSFlags)) {
2206	unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
2207	if (!MI.getOperand(i: OpIdx).isImm()) {
2208	ErrInfo = "SEW value expected to be an immediate";
2209	return false;
2210	}
2211	uint64_t Log2SEW = MI.getOperand(i: OpIdx).getImm();
2212	if (Log2SEW > 31) {
2213	ErrInfo = "Unexpected SEW value";
2214	return false;
2215	}
2216	unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2217	if (!RISCVVType::isValidSEW(SEW)) {
2218	ErrInfo = "Unexpected SEW value";
2219	return false;
2220	}
2221	}
2222	if (RISCVII::hasVecPolicyOp(TSFlags)) {
2223	unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
2224	if (!MI.getOperand(i: OpIdx).isImm()) {
2225	ErrInfo = "Policy operand expected to be an immediate";
2226	return false;
2227	}
2228	uint64_t Policy = MI.getOperand(i: OpIdx).getImm();
2229	if (Policy > (RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC)) {
2230	ErrInfo = "Invalid Policy Value";
2231	return false;
2232	}
2233	if (!RISCVII::hasVLOp(TSFlags)) {
2234	ErrInfo = "policy operand w/o VL operand?";
2235	return false;
2236	}
2237
2238	// VecPolicy operands can only exist on instructions with passthru/merge
2239	// arguments. Note that not all arguments with passthru have vec policy
2240	// operands- some instructions have implicit policies.
2241	unsigned UseOpIdx;
2242	if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx)) {
2243	ErrInfo = "policy operand w/o tied operand?";
2244	return false;
2245	}
2246	}
2247
2248	return true;
2249	}
2250
2251	bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
2252	const MachineInstr &AddrI,
2253	ExtAddrMode &AM) const {
2254	switch (MemI.getOpcode()) {
2255	default:
2256	return false;
2257	case RISCV::LB:
2258	case RISCV::LBU:
2259	case RISCV::LH:
2260	case RISCV::LHU:
2261	case RISCV::LW:
2262	case RISCV::LWU:
2263	case RISCV::LD:
2264	case RISCV::FLH:
2265	case RISCV::FLW:
2266	case RISCV::FLD:
2267	case RISCV::SB:
2268	case RISCV::SH:
2269	case RISCV::SW:
2270	case RISCV::SD:
2271	case RISCV::FSH:
2272	case RISCV::FSW:
2273	case RISCV::FSD:
2274	break;
2275	}
2276
2277	if (MemI.getOperand(i: 0).getReg() == Reg)
2278	return false;
2279
2280	if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(1).isReg() ||
2281	!AddrI.getOperand(2).isImm())
2282	return false;
2283
2284	int64_t OldOffset = MemI.getOperand(i: 2).getImm();
2285	int64_t Disp = AddrI.getOperand(i: 2).getImm();
2286	int64_t NewOffset = OldOffset + Disp;
2287	if (!STI.is64Bit())
2288	NewOffset = SignExtend64<32>(x: NewOffset);
2289
2290	if (!isInt<12>(x: NewOffset))
2291	return false;
2292
2293	AM.BaseReg = AddrI.getOperand(i: 1).getReg();
2294	AM.ScaledReg = 0;
2295	AM.Scale = 0;
2296	AM.Displacement = NewOffset;
2297	AM.Form = ExtAddrMode::Formula::Basic;
2298	return true;
2299	}
2300
2301	MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
2302	const ExtAddrMode &AM) const {
2303
2304	const DebugLoc &DL = MemI.getDebugLoc();
2305	MachineBasicBlock &MBB = *MemI.getParent();
2306
2307	assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
2308	"Addressing mode not supported for folding");
2309
2310	return BuildMI(MBB, MemI, DL, get(MemI.getOpcode()))
2311	.addReg(MemI.getOperand(i: 0).getReg(),
2312	MemI.mayLoad() ? RegState::Define : 0)
2313	.addReg(AM.BaseReg)
2314	.addImm(AM.Displacement)
2315	.setMemRefs(MemI.memoperands())
2316	.setMIFlags(MemI.getFlags());
2317	}
2318
2319	bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
2320	const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
2321	int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
2322	const TargetRegisterInfo *TRI) const {
2323	if (!LdSt.mayLoadOrStore())
2324	return false;
2325
2326	// Conservatively, only handle scalar loads/stores for now.
2327	switch (LdSt.getOpcode()) {
2328	case RISCV::LB:
2329	case RISCV::LBU:
2330	case RISCV::SB:
2331	case RISCV::LH:
2332	case RISCV::LHU:
2333	case RISCV::FLH:
2334	case RISCV::SH:
2335	case RISCV::FSH:
2336	case RISCV::LW:
2337	case RISCV::LWU:
2338	case RISCV::FLW:
2339	case RISCV::SW:
2340	case RISCV::FSW:
2341	case RISCV::LD:
2342	case RISCV::FLD:
2343	case RISCV::SD:
2344	case RISCV::FSD:
2345	break;
2346	default:
2347	return false;
2348	}
2349	const MachineOperand *BaseOp;
2350	OffsetIsScalable = false;
2351	if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2352	return false;
2353	BaseOps.push_back(Elt: BaseOp);
2354	return true;
2355	}
2356
2357	// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
2358	// helper?
2359	static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
2360	ArrayRef<const MachineOperand *> BaseOps1,
2361	const MachineInstr &MI2,
2362	ArrayRef<const MachineOperand *> BaseOps2) {
2363	// Only examine the first "base" operand of each instruction, on the
2364	// assumption that it represents the real base address of the memory access.
2365	// Other operands are typically offsets or indices from this base address.
2366	if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
2367	return true;
2368
2369	if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
2370	return false;
2371
2372	auto MO1 = *MI1.memoperands_begin();
2373	auto MO2 = *MI2.memoperands_begin();
2374	if (MO1->getAddrSpace() != MO2->getAddrSpace())
2375	return false;
2376
2377	auto Base1 = MO1->getValue();
2378	auto Base2 = MO2->getValue();
2379	if (!Base1 || !Base2)
2380	return false;
2381	Base1 = getUnderlyingObject(V: Base1);
2382	Base2 = getUnderlyingObject(V: Base2);
2383
2384	if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
2385	return false;
2386
2387	return Base1 == Base2;
2388	}
2389
2390	bool RISCVInstrInfo::shouldClusterMemOps(
2391	ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
2392	bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
2393	int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
2394	unsigned NumBytes) const {
2395	// If the mem ops (to be clustered) do not have the same base ptr, then they
2396	// should not be clustered
2397	if (!BaseOps1.empty() && !BaseOps2.empty()) {
2398	const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
2399	const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
2400	if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
2401	return false;
2402	} else if (!BaseOps1.empty() || !BaseOps2.empty()) {
2403	// If only one base op is empty, they do not have the same base ptr
2404	return false;
2405	}
2406
2407	unsigned CacheLineSize =
2408	BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
2409	// Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
2410	CacheLineSize = CacheLineSize ? CacheLineSize : 64;
2411	// Cluster if the memory operations are on the same or a neighbouring cache
2412	// line, but limit the maximum ClusterSize to avoid creating too much
2413	// additional register pressure.
2414	return ClusterSize <= 4 && std::abs(i: Offset1 - Offset2) < CacheLineSize;
2415	}
2416
2417	// Set BaseReg (the base register operand), Offset (the byte offset being
2418	// accessed) and the access Width of the passed instruction that reads/writes
2419	// memory. Returns false if the instruction does not read/write memory or the
2420	// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
2421	// recognise base operands and offsets in all cases.
2422	// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
2423	// function) and set it as appropriate.
2424	bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
2425	const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
2426	LocationSize &Width, const TargetRegisterInfo *TRI) const {
2427	if (!LdSt.mayLoadOrStore())
2428	return false;
2429
2430	// Here we assume the standard RISC-V ISA, which uses a base+offset
2431	// addressing mode. You'll need to relax these conditions to support custom
2432	// load/store instructions.
2433	if (LdSt.getNumExplicitOperands() != 3)
2434	return false;
2435	if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
2436	!LdSt.getOperand(i: 2).isImm())
2437	return false;
2438
2439	if (!LdSt.hasOneMemOperand())
2440	return false;
2441
2442	Width = (*LdSt.memoperands_begin())->getSize();
2443	BaseReg = &LdSt.getOperand(i: 1);
2444	Offset = LdSt.getOperand(i: 2).getImm();
2445	return true;
2446	}
2447
2448	bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
2449	const MachineInstr &MIa, const MachineInstr &MIb) const {
2450	assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
2451	assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
2452
2453	if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
2454	MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2455	return false;
2456
2457	// Retrieve the base register, offset from the base register and width. Width
2458	// is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
2459	// base registers are identical, and the offset of a lower memory access +
2460	// the width doesn't overlap the offset of a higher memory access,
2461	// then the memory accesses are different.
2462	const TargetRegisterInfo *TRI = STI.getRegisterInfo();
2463	const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
2464	int64_t OffsetA = 0, OffsetB = 0;
2465	LocationSize WidthA = 0, WidthB = 0;
2466	if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA, TRI) &&
2467	getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB, TRI)) {
2468	if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
2469	int LowOffset = std::min(a: OffsetA, b: OffsetB);
2470	int HighOffset = std::max(a: OffsetA, b: OffsetB);
2471	LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2472	if (LowWidth.hasValue() &&
2473	LowOffset + (int)LowWidth.getValue() <= HighOffset)
2474	return true;
2475	}
2476	}
2477	return false;
2478	}
2479
2480	std::pair<unsigned, unsigned>
2481	RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
2482	const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
2483	return std::make_pair(x: TF & Mask, y: TF & ~Mask);
2484	}
2485
2486	ArrayRef<std::pair<unsigned, const char *>>
2487	RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
2488	using namespace RISCVII;
2489	static const std::pair<unsigned, const char *> TargetFlags[] = {
2490	{MO_CALL, "riscv-call"},
2491	{MO_LO, "riscv-lo"},
2492	{MO_HI, "riscv-hi"},
2493	{MO_PCREL_LO, "riscv-pcrel-lo"},
2494	{MO_PCREL_HI, "riscv-pcrel-hi"},
2495	{MO_GOT_HI, "riscv-got-hi"},
2496	{MO_TPREL_LO, "riscv-tprel-lo"},
2497	{MO_TPREL_HI, "riscv-tprel-hi"},
2498	{MO_TPREL_ADD, "riscv-tprel-add"},
2499	{MO_TLS_GOT_HI, "riscv-tls-got-hi"},
2500	{MO_TLS_GD_HI, "riscv-tls-gd-hi"},
2501	{MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
2502	{MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
2503	{MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
2504	{MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
2505	return ArrayRef(TargetFlags);
2506	}
2507	bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
2508	MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
2509	const Function &F = MF.getFunction();
2510
2511	// Can F be deduplicated by the linker? If it can, don't outline from it.
2512	if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
2513	return false;
2514
2515	// Don't outline from functions with section markings; the program could
2516	// expect that all the code is in the named section.
2517	if (F.hasSection())
2518	return false;
2519
2520	// It's safe to outline from MF.
2521	return true;
2522	}
2523
2524	bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
2525	unsigned &Flags) const {
2526	// More accurate safety checking is done in getOutliningCandidateInfo.
2527	return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
2528	}
2529
2530	// Enum values indicating how an outlined call should be constructed.
2531	enum MachineOutlinerConstructionID {
2532	MachineOutlinerDefault
2533	};
2534
2535	bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
2536	MachineFunction &MF) const {
2537	return MF.getFunction().hasMinSize();
2538	}
2539
2540	std::optional<outliner::OutlinedFunction>
2541	RISCVInstrInfo::getOutliningCandidateInfo(
2542	std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
2543
2544	// First we need to filter out candidates where the X5 register (IE t0) can't
2545	// be used to setup the function call.
2546	auto CannotInsertCall = [](outliner::Candidate &C) {
2547	const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();
2548	return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
2549	};
2550
2551	llvm::erase_if(C&: RepeatedSequenceLocs, P: CannotInsertCall);
2552
2553	// If the sequence doesn't have enough candidates left, then we're done.
2554	if (RepeatedSequenceLocs.size() < 2)
2555	return std::nullopt;
2556
2557	unsigned SequenceSize = 0;
2558
2559	for (auto &MI : RepeatedSequenceLocs[0])
2560	SequenceSize += getInstSizeInBytes(MI);
2561
2562	// call t0, function = 8 bytes.
2563	unsigned CallOverhead = 8;
2564	for (auto &C : RepeatedSequenceLocs)
2565	C.setCallInfo(CID: MachineOutlinerDefault, CO: CallOverhead);
2566
2567	// jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
2568	unsigned FrameOverhead = 4;
2569	if (RepeatedSequenceLocs[0]
2570	.getMF()
2571	->getSubtarget<RISCVSubtarget>()
2572	.hasStdExtCOrZca())
2573	FrameOverhead = 2;
2574
2575	return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
2576	FrameOverhead, MachineOutlinerDefault);
2577	}
2578
2579	outliner::InstrType
2580	RISCVInstrInfo::getOutliningTypeImpl(MachineBasicBlock::iterator &MBBI,
2581	unsigned Flags) const {
2582	MachineInstr &MI = *MBBI;
2583	MachineBasicBlock *MBB = MI.getParent();
2584	const TargetRegisterInfo *TRI =
2585	MBB->getParent()->getSubtarget().getRegisterInfo();
2586	const auto &F = MI.getMF()->getFunction();
2587
2588	// We can manually strip out CFI instructions later.
2589	if (MI.isCFIInstruction())
2590	// If current function has exception handling code, we can't outline &
2591	// strip these CFI instructions since it may break .eh_frame section
2592	// needed in unwinding.
2593	return F.needsUnwindTableEntry() ? outliner::InstrType::Illegal
2594	: outliner::InstrType::Invisible;
2595
2596	// We need support for tail calls to outlined functions before return
2597	// statements can be allowed.
2598	if (MI.isReturn())
2599	return outliner::InstrType::Illegal;
2600
2601	// Don't allow modifying the X5 register which we use for return addresses for
2602	// these outlined functions.
2603	if (MI.modifiesRegister(RISCV::X5, TRI) ||
2604	MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5))
2605	return outliner::InstrType::Illegal;
2606
2607	// Make sure the operands don't reference something unsafe.
2608	for (const auto &MO : MI.operands()) {
2609
2610	// pcrel-hi and pcrel-lo can't put in separate sections, filter that out
2611	// if any possible.
2612	if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
2613	(MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
2614	F.hasSection()))
2615	return outliner::InstrType::Illegal;
2616	}
2617
2618	return outliner::InstrType::Legal;
2619	}
2620
2621	void RISCVInstrInfo::buildOutlinedFrame(
2622	MachineBasicBlock &MBB, MachineFunction &MF,
2623	const outliner::OutlinedFunction &OF) const {
2624
2625	// Strip out any CFI instructions
2626	bool Changed = true;
2627	while (Changed) {
2628	Changed = false;
2629	auto I = MBB.begin();
2630	auto E = MBB.end();
2631	for (; I != E; ++I) {
2632	if (I->isCFIInstruction()) {
2633	I->removeFromParent();
2634	Changed = true;
2635	break;
2636	}
2637	}
2638	}
2639
2640	MBB.addLiveIn(RISCV::X5);
2641
2642	// Add in a return instruction to the end of the outlined frame.
2643	MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))
2644	.addReg(RISCV::X0, RegState::Define)
2645	.addReg(RISCV::X5)
2646	.addImm(0));
2647	}
2648
2649	MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
2650	Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
2651	MachineFunction &MF, outliner::Candidate &C) const {
2652
2653	// Add in a call instruction to the outlined function at the given location.
2654	It = MBB.insert(It,
2655	BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
2656	.addGlobalAddress(M.getNamedValue(MF.getName()), 0,
2657	RISCVII::MO_CALL));
2658	return It;
2659	}
2660
2661	std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
2662	Register Reg) const {
2663	// TODO: Handle cases where Reg is a super- or sub-register of the
2664	// destination register.
2665	const MachineOperand &Op0 = MI.getOperand(i: 0);
2666	if (!Op0.isReg() || Reg != Op0.getReg())
2667	return std::nullopt;
2668
2669	// Don't consider ADDIW as a candidate because the caller may not be aware
2670	// of its sign extension behaviour.
2671	if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&
2672	MI.getOperand(2).isImm())
2673	return RegImmPair{MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getImm()};
2674
2675	return std::nullopt;
2676	}
2677
2678	// MIR printer helper function to annotate Operands with a comment.
2679	std::string RISCVInstrInfo::createMIROperandComment(
2680	const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
2681	const TargetRegisterInfo *TRI) const {
2682	// Print a generic comment for this operand if there is one.
2683	std::string GenericComment =
2684	TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
2685	if (!GenericComment.empty())
2686	return GenericComment;
2687
2688	// If not, we must have an immediate operand.
2689	if (!Op.isImm())
2690	return std::string();
2691
2692	std::string Comment;
2693	raw_string_ostream OS(Comment);
2694
2695	uint64_t TSFlags = MI.getDesc().TSFlags;
2696
2697	// Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
2698	// operand of vector codegen pseudos.
2699	if ((MI.getOpcode() == RISCV::VSETVLI || MI.getOpcode() == RISCV::VSETIVLI ||
2700	MI.getOpcode() == RISCV::PseudoVSETVLI ||
2701	MI.getOpcode() == RISCV::PseudoVSETIVLI ||
2702	MI.getOpcode() == RISCV::PseudoVSETVLIX0) &&
2703	OpIdx == 2) {
2704	unsigned Imm = MI.getOperand(i: OpIdx).getImm();
2705	RISCVVType::printVType(VType: Imm, OS);
2706	} else if (RISCVII::hasSEWOp(TSFlags) &&
2707	OpIdx == RISCVII::getSEWOpNum(Desc: MI.getDesc())) {
2708	unsigned Log2SEW = MI.getOperand(i: OpIdx).getImm();
2709	unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2710	assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
2711	OS << "e" << SEW;
2712	} else if (RISCVII::hasVecPolicyOp(TSFlags) &&
2713	OpIdx == RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())) {
2714	unsigned Policy = MI.getOperand(i: OpIdx).getImm();
2715	assert(Policy <= (RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC) &&
2716	"Invalid Policy Value");
2717	OS << (Policy & RISCVII::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
2718	<< (Policy & RISCVII::MASK_AGNOSTIC ? "ma" : "mu");
2719	}
2720
2721	OS.flush();
2722	return Comment;
2723	}
2724
2725	// clang-format off
2726	#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
2727	RISCV::Pseudo##OP##_##LMUL
2728
2729	#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
2730	RISCV::Pseudo##OP##_##LMUL##_MASK
2731
2732	#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
2733	CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
2734	case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
2735
2736	#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
2737	CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
2738	case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
2739	case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
2740	case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
2741	case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
2742	case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
2743
2744	#define CASE_RVV_OPCODE_UNMASK(OP) \
2745	CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
2746	case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
2747
2748	#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
2749	CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
2750	case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
2751	case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
2752	case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
2753	case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
2754	case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
2755
2756	#define CASE_RVV_OPCODE_MASK(OP) \
2757	CASE_RVV_OPCODE_MASK_WIDEN(OP): \
2758	case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
2759
2760	#define CASE_RVV_OPCODE_WIDEN(OP) \
2761	CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
2762	case CASE_RVV_OPCODE_MASK_WIDEN(OP)
2763
2764	#define CASE_RVV_OPCODE(OP) \
2765	CASE_RVV_OPCODE_UNMASK(OP): \
2766	case CASE_RVV_OPCODE_MASK(OP)
2767	// clang-format on
2768
2769	// clang-format off
2770	#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
2771	RISCV::PseudoV##OP##_##TYPE##_##LMUL
2772
2773	#define CASE_VMA_OPCODE_LMULS_M1(OP, TYPE) \
2774	CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
2775	case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
2776	case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
2777	case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
2778
2779	#define CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE) \
2780	CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
2781	case CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)
2782
2783	#define CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE) \
2784	CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
2785	case CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)
2786
2787	#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
2788	CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
2789	case CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)
2790
2791	// VFMA instructions are SEW specific.
2792	#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
2793	RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
2794
2795	#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
2796	CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
2797	case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
2798	case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
2799	case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
2800
2801	#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
2802	CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
2803	case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
2804
2805	#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
2806	CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
2807	case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
2808
2809	#define CASE_VFMA_OPCODE_VV(OP) \
2810	CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
2811	case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
2812	case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
2813
2814	#define CASE_VFMA_SPLATS(OP) \
2815	CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
2816	case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
2817	case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
2818	// clang-format on
2819
2820	bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2821	unsigned &SrcOpIdx1,
2822	unsigned &SrcOpIdx2) const {
2823	const MCInstrDesc &Desc = MI.getDesc();
2824	if (!Desc.isCommutable())
2825	return false;
2826
2827	switch (MI.getOpcode()) {
2828	case RISCV::TH_MVEQZ:
2829	case RISCV::TH_MVNEZ:
2830	// We can't commute operands if operand 2 (i.e., rs1 in
2831	// mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
2832	// not valid as the in/out-operand 1).
2833	if (MI.getOperand(2).getReg() == RISCV::X0)
2834	return false;
2835	// Operands 1 and 2 are commutable, if we switch the opcode.
2836	return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
2837	case RISCV::TH_MULA:
2838	case RISCV::TH_MULAW:
2839	case RISCV::TH_MULAH:
2840	case RISCV::TH_MULS:
2841	case RISCV::TH_MULSW:
2842	case RISCV::TH_MULSH:
2843	// Operands 2 and 3 are commutable.
2844	return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
2845	case RISCV::PseudoCCMOVGPRNoX0:
2846	case RISCV::PseudoCCMOVGPR:
2847	// Operands 4 and 5 are commutable.
2848	return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
2849	case CASE_RVV_OPCODE(VADD_VV):
2850	case CASE_RVV_OPCODE(VAND_VV):
2851	case CASE_RVV_OPCODE(VOR_VV):
2852	case CASE_RVV_OPCODE(VXOR_VV):
2853	case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
2854	case CASE_RVV_OPCODE_MASK(VMSNE_VV):
2855	case CASE_RVV_OPCODE(VMIN_VV):
2856	case CASE_RVV_OPCODE(VMINU_VV):
2857	case CASE_RVV_OPCODE(VMAX_VV):
2858	case CASE_RVV_OPCODE(VMAXU_VV):
2859	case CASE_RVV_OPCODE(VMUL_VV):
2860	case CASE_RVV_OPCODE(VMULH_VV):
2861	case CASE_RVV_OPCODE(VMULHU_VV):
2862	case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
2863	case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
2864	case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
2865	case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
2866	case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
2867	case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
2868	case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
2869	// Operands 2 and 3 are commutable.
2870	return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
2871	case CASE_VFMA_SPLATS(FMADD):
2872	case CASE_VFMA_SPLATS(FMSUB):
2873	case CASE_VFMA_SPLATS(FMACC):
2874	case CASE_VFMA_SPLATS(FMSAC):
2875	case CASE_VFMA_SPLATS(FNMADD):
2876	case CASE_VFMA_SPLATS(FNMSUB):
2877	case CASE_VFMA_SPLATS(FNMACC):
2878	case CASE_VFMA_SPLATS(FNMSAC):
2879	case CASE_VFMA_OPCODE_VV(FMACC):
2880	case CASE_VFMA_OPCODE_VV(FMSAC):
2881	case CASE_VFMA_OPCODE_VV(FNMACC):
2882	case CASE_VFMA_OPCODE_VV(FNMSAC):
2883	case CASE_VMA_OPCODE_LMULS(MADD, VX):
2884	case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
2885	case CASE_VMA_OPCODE_LMULS(MACC, VX):
2886	case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
2887	case CASE_VMA_OPCODE_LMULS(MACC, VV):
2888	case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
2889	// If the tail policy is undisturbed we can't commute.
2890	assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
2891	if ((MI.getOperand(i: MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
2892	return false;
2893
2894	// For these instructions we can only swap operand 1 and operand 3 by
2895	// changing the opcode.
2896	unsigned CommutableOpIdx1 = 1;
2897	unsigned CommutableOpIdx2 = 3;
2898	if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2899	CommutableOpIdx2))
2900	return false;
2901	return true;
2902	}
2903	case CASE_VFMA_OPCODE_VV(FMADD):
2904	case CASE_VFMA_OPCODE_VV(FMSUB):
2905	case CASE_VFMA_OPCODE_VV(FNMADD):
2906	case CASE_VFMA_OPCODE_VV(FNMSUB):
2907	case CASE_VMA_OPCODE_LMULS(MADD, VV):
2908	case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
2909	// If the tail policy is undisturbed we can't commute.
2910	assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
2911	if ((MI.getOperand(i: MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
2912	return false;
2913
2914	// For these instructions we have more freedom. We can commute with the
2915	// other multiplicand or with the addend/subtrahend/minuend.
2916
2917	// Any fixed operand must be from source 1, 2 or 3.
2918	if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
2919	return false;
2920	if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
2921	return false;
2922
2923	// It both ops are fixed one must be the tied source.
2924	if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2925	SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
2926	return false;
2927
2928	// Look for two different register operands assumed to be commutable
2929	// regardless of the FMA opcode. The FMA opcode is adjusted later if
2930	// needed.
2931	if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2932	SrcOpIdx2 == CommuteAnyOperandIndex) {
2933	// At least one of operands to be commuted is not specified and
2934	// this method is free to choose appropriate commutable operands.
2935	unsigned CommutableOpIdx1 = SrcOpIdx1;
2936	if (SrcOpIdx1 == SrcOpIdx2) {
2937	// Both of operands are not fixed. Set one of commutable
2938	// operands to the tied source.
2939	CommutableOpIdx1 = 1;
2940	} else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
2941	// Only one of the operands is not fixed.
2942	CommutableOpIdx1 = SrcOpIdx2;
2943	}
2944
2945	// CommutableOpIdx1 is well defined now. Let's choose another commutable
2946	// operand and assign its index to CommutableOpIdx2.
2947	unsigned CommutableOpIdx2;
2948	if (CommutableOpIdx1 != 1) {
2949	// If we haven't already used the tied source, we must use it now.
2950	CommutableOpIdx2 = 1;
2951	} else {
2952	Register Op1Reg = MI.getOperand(i: CommutableOpIdx1).getReg();
2953
2954	// The commuted operands should have different registers.
2955	// Otherwise, the commute transformation does not change anything and
2956	// is useless. We use this as a hint to make our decision.
2957	if (Op1Reg != MI.getOperand(i: 2).getReg())
2958	CommutableOpIdx2 = 2;
2959	else
2960	CommutableOpIdx2 = 3;
2961	}
2962
2963	// Assign the found pair of commutable indices to SrcOpIdx1 and
2964	// SrcOpIdx2 to return those values.
2965	if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2966	CommutableOpIdx2))
2967	return false;
2968	}
2969
2970	return true;
2971	}
2972	}
2973
2974	return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2975	}
2976
2977	// clang-format off
2978	#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
2979	case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
2980	Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
2981	break;
2982
2983	#define CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \
2984	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
2985	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
2986	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
2987	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
2988
2989	#define CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \
2990	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
2991	CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
2992
2993	#define CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \
2994	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
2995	CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
2996
2997	#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
2998	CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
2999	CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
3000
3001	#define CASE_VMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3002	CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16) \
3003	CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32) \
3004	CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
3005
3006	// VFMA depends on SEW.
3007	#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
3008	case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
3009	Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
3010	break;
3011
3012	#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
3013	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
3014	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
3015	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
3016	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
3017
3018	#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
3019	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
3020	CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
3021
3022	#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
3023	CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
3024	CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
3025	CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
3026
3027	#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
3028	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
3029	CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
3030
3031	#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE, SEW) \
3032	CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8, SEW) \
3033	CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)
3034
3035	#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3036	CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
3037	CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
3038	CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
3039
3040	MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
3041	bool NewMI,
3042	unsigned OpIdx1,
3043	unsigned OpIdx2) const {
3044	auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
3045	if (NewMI)
3046	return *MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI);
3047	return MI;
3048	};
3049
3050	switch (MI.getOpcode()) {
3051	case RISCV::TH_MVEQZ:
3052	case RISCV::TH_MVNEZ: {
3053	auto &WorkingMI = cloneIfNew(MI);
3054	WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
3055	: RISCV::TH_MVEQZ));
3056	return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,
3057	OpIdx2);
3058	}
3059	case RISCV::PseudoCCMOVGPRNoX0:
3060	case RISCV::PseudoCCMOVGPR: {
3061	// CCMOV can be commuted by inverting the condition.
3062	auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
3063	CC = RISCVCC::getOppositeBranchCondition(CC);
3064	auto &WorkingMI = cloneIfNew(MI);
3065	WorkingMI.getOperand(i: 3).setImm(CC);
3066	return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,
3067	OpIdx1, OpIdx2);
3068	}
3069	case CASE_VFMA_SPLATS(FMACC):
3070	case CASE_VFMA_SPLATS(FMADD):
3071	case CASE_VFMA_SPLATS(FMSAC):
3072	case CASE_VFMA_SPLATS(FMSUB):
3073	case CASE_VFMA_SPLATS(FNMACC):
3074	case CASE_VFMA_SPLATS(FNMADD):
3075	case CASE_VFMA_SPLATS(FNMSAC):
3076	case CASE_VFMA_SPLATS(FNMSUB):
3077	case CASE_VFMA_OPCODE_VV(FMACC):
3078	case CASE_VFMA_OPCODE_VV(FMSAC):
3079	case CASE_VFMA_OPCODE_VV(FNMACC):
3080	case CASE_VFMA_OPCODE_VV(FNMSAC):
3081	case CASE_VMA_OPCODE_LMULS(MADD, VX):
3082	case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
3083	case CASE_VMA_OPCODE_LMULS(MACC, VX):
3084	case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
3085	case CASE_VMA_OPCODE_LMULS(MACC, VV):
3086	case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
3087	// It only make sense to toggle these between clobbering the
3088	// addend/subtrahend/minuend one of the multiplicands.
3089	assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3090	assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
3091	unsigned Opc;
3092	switch (MI.getOpcode()) {
3093	default:
3094	llvm_unreachable("Unexpected opcode");
3095	CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
3096	CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
3097	CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
3098	CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
3099	CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
3100	CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
3101	CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
3102	CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
3103	CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
3104	CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
3105	CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
3106	CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
3107	CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
3108	CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
3109	CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
3110	CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
3111	CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
3112	CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
3113	}
3114
3115	auto &WorkingMI = cloneIfNew(MI);
3116	WorkingMI.setDesc(get(Opc));
3117	return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
3118	OpIdx1, OpIdx2);
3119	}
3120	case CASE_VFMA_OPCODE_VV(FMADD):
3121	case CASE_VFMA_OPCODE_VV(FMSUB):
3122	case CASE_VFMA_OPCODE_VV(FNMADD):
3123	case CASE_VFMA_OPCODE_VV(FNMSUB):
3124	case CASE_VMA_OPCODE_LMULS(MADD, VV):
3125	case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
3126	assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3127	// If one of the operands, is the addend we need to change opcode.
3128	// Otherwise we're just swapping 2 of the multiplicands.
3129	if (OpIdx1 == 3 || OpIdx2 == 3) {
3130	unsigned Opc;
3131	switch (MI.getOpcode()) {
3132	default:
3133	llvm_unreachable("Unexpected opcode");
3134	CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
3135	CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
3136	CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
3137	CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
3138	CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
3139	CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
3140	}
3141
3142	auto &WorkingMI = cloneIfNew(MI);
3143	WorkingMI.setDesc(get(Opc));
3144	return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
3145	OpIdx1, OpIdx2);
3146	}
3147	// Let the default code handle it.
3148	break;
3149	}
3150	}
3151
3152	return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
3153	}
3154
3155	#undef CASE_VMA_OPCODE_COMMON
3156	#undef CASE_VMA_OPCODE_LMULS_M1
3157	#undef CASE_VMA_OPCODE_LMULS_MF2
3158	#undef CASE_VMA_OPCODE_LMULS_MF4
3159	#undef CASE_VMA_OPCODE_LMULS
3160	#undef CASE_VFMA_OPCODE_COMMON
3161	#undef CASE_VFMA_OPCODE_LMULS_M1
3162	#undef CASE_VFMA_OPCODE_LMULS_MF2
3163	#undef CASE_VFMA_OPCODE_LMULS_MF4
3164	#undef CASE_VFMA_OPCODE_VV
3165	#undef CASE_VFMA_SPLATS
3166
3167	// clang-format off
3168	#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
3169	RISCV::PseudoV##OP##_##LMUL##_TIED
3170
3171	#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \
3172	CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
3173	case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
3174	case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
3175	case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
3176	case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
3177
3178	#define CASE_WIDEOP_OPCODE_LMULS(OP) \
3179	CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
3180	case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
3181
3182	#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
3183	case RISCV::PseudoV##OP##_##LMUL##_TIED: \
3184	NewOpc = RISCV::PseudoV##OP##_##LMUL; \
3185	break;
3186
3187	#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3188	CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
3189	CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
3190	CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
3191	CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
3192	CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
3193
3194	#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3195	CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
3196	CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3197
3198	// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
3199	#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
3200	RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
3201
3202	#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP) \
3203	CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
3204	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
3205	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
3206	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
3207	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
3208	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
3209	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
3210	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
3211	case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
3212
3213	#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
3214	case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
3215	NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
3216	break;
3217
3218	#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3219	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
3220	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
3221	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
3222	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
3223	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
3224	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
3225	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
3226	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
3227	CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
3228
3229	#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3230	CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3231	// clang-format on
3232
3233	MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
3234	LiveVariables *LV,
3235	LiveIntervals *LIS) const {
3236	MachineInstrBuilder MIB;
3237	switch (MI.getOpcode()) {
3238	default:
3239	return nullptr;
3240	case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
3241	case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): {
3242	assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
3243	MI.getNumExplicitOperands() == 7 &&
3244	"Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
3245	// If the tail policy is undisturbed we can't convert.
3246	if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
3247	1) == 0)
3248	return nullptr;
3249	// clang-format off
3250	unsigned NewOpc;
3251	switch (MI.getOpcode()) {
3252	default:
3253	llvm_unreachable("Unexpected opcode");
3254	CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV)
3255	CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV)
3256	}
3257	// clang-format on
3258
3259	MachineBasicBlock &MBB = *MI.getParent();
3260	MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3261	.add(MI.getOperand(i: 0))
3262	.addReg(MI.getOperand(i: 0).getReg(), RegState::Undef)
3263	.add(MI.getOperand(i: 1))
3264	.add(MI.getOperand(i: 2))
3265	.add(MI.getOperand(i: 3))
3266	.add(MI.getOperand(i: 4))
3267	.add(MI.getOperand(i: 5))
3268	.add(MI.getOperand(i: 6));
3269	break;
3270	}
3271	case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
3272	case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
3273	case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
3274	case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
3275	// If the tail policy is undisturbed we can't convert.
3276	assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
3277	MI.getNumExplicitOperands() == 6);
3278	if ((MI.getOperand(i: 5).getImm() & 1) == 0)
3279	return nullptr;
3280
3281	// clang-format off
3282	unsigned NewOpc;
3283	switch (MI.getOpcode()) {
3284	default:
3285	llvm_unreachable("Unexpected opcode");
3286	CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
3287	CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
3288	CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
3289	CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
3290	}
3291	// clang-format on
3292
3293	MachineBasicBlock &MBB = *MI.getParent();
3294	MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3295	.add(MI.getOperand(i: 0))
3296	.addReg(MI.getOperand(i: 0).getReg(), RegState::Undef)
3297	.add(MI.getOperand(i: 1))
3298	.add(MI.getOperand(i: 2))
3299	.add(MI.getOperand(i: 3))
3300	.add(MI.getOperand(i: 4))
3301	.add(MI.getOperand(i: 5));
3302	break;
3303	}
3304	}
3305	MIB.copyImplicitOps(OtherMI: MI);
3306
3307	if (LV) {
3308	unsigned NumOps = MI.getNumOperands();
3309	for (unsigned I = 1; I < NumOps; ++I) {
3310	MachineOperand &Op = MI.getOperand(i: I);
3311	if (Op.isReg() && Op.isKill())
3312	LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI&: *MIB);
3313	}
3314	}
3315
3316	if (LIS) {
3317	SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3318
3319	if (MI.getOperand(i: 0).isEarlyClobber()) {
3320	// Use operand 1 was tied to early-clobber def operand 0, so its live
3321	// interval could have ended at an early-clobber slot. Now they are not
3322	// tied we need to update it to the normal register slot.
3323	LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
3324	LiveRange::Segment *S = LI.getSegmentContaining(Idx);
3325	if (S->end == Idx.getRegSlot(EC: true))
3326	S->end = Idx.getRegSlot();
3327	}
3328	}
3329
3330	return MIB;
3331	}
3332
3333	#undef CASE_WIDEOP_OPCODE_COMMON
3334	#undef CASE_WIDEOP_OPCODE_LMULS_MF4
3335	#undef CASE_WIDEOP_OPCODE_LMULS
3336	#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
3337	#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3338	#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
3339	#undef CASE_FP_WIDEOP_OPCODE_COMMON
3340	#undef CASE_FP_WIDEOP_OPCODE_LMULS_MF4
3341	#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
3342	#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3343	#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
3344
3345	void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
3346	MachineBasicBlock::iterator II, const DebugLoc &DL,
3347	Register DestReg, uint32_t Amount,
3348	MachineInstr::MIFlag Flag) const {
3349	MachineRegisterInfo &MRI = MF.getRegInfo();
3350	if (llvm::has_single_bit<uint32_t>(Value: Amount)) {
3351	uint32_t ShiftAmount = Log2_32(Value: Amount);
3352	if (ShiftAmount == 0)
3353	return;
3354	BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
3355	.addReg(DestReg, RegState::Kill)
3356	.addImm(ShiftAmount)
3357	.setMIFlag(Flag);
3358	} else if (STI.hasStdExtZba() &&
3359	((Amount % 3 == 0 && isPowerOf2_64(Value: Amount / 3)) ||
3360	(Amount % 5 == 0 && isPowerOf2_64(Value: Amount / 5)) ||
3361	(Amount % 9 == 0 && isPowerOf2_64(Value: Amount / 9)))) {
3362	// We can use Zba SHXADD+SLLI instructions for multiply in some cases.
3363	unsigned Opc;
3364	uint32_t ShiftAmount;
3365	if (Amount % 9 == 0) {
3366	Opc = RISCV::SH3ADD;
3367	ShiftAmount = Log2_64(Value: Amount / 9);
3368	} else if (Amount % 5 == 0) {
3369	Opc = RISCV::SH2ADD;
3370	ShiftAmount = Log2_64(Value: Amount / 5);
3371	} else if (Amount % 3 == 0) {
3372	Opc = RISCV::SH1ADD;
3373	ShiftAmount = Log2_64(Value: Amount / 3);
3374	} else {
3375	llvm_unreachable("implied by if-clause");
3376	}
3377	if (ShiftAmount)
3378	BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
3379	.addReg(DestReg, RegState::Kill)
3380	.addImm(ShiftAmount)
3381	.setMIFlag(Flag);
3382	BuildMI(MBB, II, DL, get(Opc), DestReg)
3383	.addReg(DestReg, RegState::Kill)
3384	.addReg(DestReg)
3385	.setMIFlag(Flag);
3386	} else if (llvm::has_single_bit<uint32_t>(Value: Amount - 1)) {
3387	Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3388	uint32_t ShiftAmount = Log2_32(Value: Amount - 1);
3389	BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
3390	.addReg(DestReg)
3391	.addImm(ShiftAmount)
3392	.setMIFlag(Flag);
3393	BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
3394	.addReg(ScaledRegister, RegState::Kill)
3395	.addReg(DestReg, RegState::Kill)
3396	.setMIFlag(Flag);
3397	} else if (llvm::has_single_bit<uint32_t>(Value: Amount + 1)) {
3398	Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3399	uint32_t ShiftAmount = Log2_32(Value: Amount + 1);
3400	BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
3401	.addReg(DestReg)
3402	.addImm(ShiftAmount)
3403	.setMIFlag(Flag);
3404	BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)
3405	.addReg(ScaledRegister, RegState::Kill)
3406	.addReg(DestReg, RegState::Kill)
3407	.setMIFlag(Flag);
3408	} else if (STI.hasStdExtM() || STI.hasStdExtZmmul()) {
3409	Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3410	movImm(MBB, MBBI: II, DL, DstReg: N, Val: Amount, Flag);
3411	BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)
3412	.addReg(DestReg, RegState::Kill)
3413	.addReg(N, RegState::Kill)
3414	.setMIFlag(Flag);
3415	} else {
3416	Register Acc;
3417	uint32_t PrevShiftAmount = 0;
3418	for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
3419	if (Amount & (1U << ShiftAmount)) {
3420	if (ShiftAmount)
3421	BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
3422	.addReg(DestReg, RegState::Kill)
3423	.addImm(ShiftAmount - PrevShiftAmount)
3424	.setMIFlag(Flag);
3425	if (Amount >> (ShiftAmount + 1)) {
3426	// If we don't have an accmulator yet, create it and copy DestReg.
3427	if (!Acc) {
3428	Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3429	BuildMI(MBB, II, DL, get(TargetOpcode::COPY), Acc)
3430	.addReg(DestReg)
3431	.setMIFlag(Flag);
3432	} else {
3433	BuildMI(MBB, II, DL, get(RISCV::ADD), Acc)
3434	.addReg(Acc, RegState::Kill)
3435	.addReg(DestReg)
3436	.setMIFlag(Flag);
3437	}
3438	}
3439	PrevShiftAmount = ShiftAmount;
3440	}
3441	}
3442	assert(Acc && "Expected valid accumulator");
3443	BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
3444	.addReg(DestReg, RegState::Kill)
3445	.addReg(Acc, RegState::Kill)
3446	.setMIFlag(Flag);
3447	}
3448	}
3449
3450	ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
3451	RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
3452	static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
3453	{{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
3454	{MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
3455	return ArrayRef(TargetFlags);
3456	}
3457
3458	// Returns true if this is the sext.w pattern, addiw rd, rs1, 0.
3459	bool RISCV::isSEXT_W(const MachineInstr &MI) {
3460	return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
3461	MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0;
3462	}
3463
3464	// Returns true if this is the zext.w pattern, adduw rd, rs1, x0.
3465	bool RISCV::isZEXT_W(const MachineInstr &MI) {
3466	return MI.getOpcode() == RISCV::ADD_UW && MI.getOperand(1).isReg() &&
3467	MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0;
3468	}
3469
3470	// Returns true if this is the zext.b pattern, andi rd, rs1, 255.
3471	bool RISCV::isZEXT_B(const MachineInstr &MI) {
3472	return MI.getOpcode() == RISCV::ANDI && MI.getOperand(1).isReg() &&
3473	MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 255;
3474	}
3475
3476	static bool isRVVWholeLoadStore(unsigned Opcode) {
3477	switch (Opcode) {
3478	default:
3479	return false;
3480	case RISCV::VS1R_V:
3481	case RISCV::VS2R_V:
3482	case RISCV::VS4R_V:
3483	case RISCV::VS8R_V:
3484	case RISCV::VL1RE8_V:
3485	case RISCV::VL2RE8_V:
3486	case RISCV::VL4RE8_V:
3487	case RISCV::VL8RE8_V:
3488	case RISCV::VL1RE16_V:
3489	case RISCV::VL2RE16_V:
3490	case RISCV::VL4RE16_V:
3491	case RISCV::VL8RE16_V:
3492	case RISCV::VL1RE32_V:
3493	case RISCV::VL2RE32_V:
3494	case RISCV::VL4RE32_V:
3495	case RISCV::VL8RE32_V:
3496	case RISCV::VL1RE64_V:
3497	case RISCV::VL2RE64_V:
3498	case RISCV::VL4RE64_V:
3499	case RISCV::VL8RE64_V:
3500	return true;
3501	}
3502	}
3503
3504	bool RISCV::isRVVSpill(const MachineInstr &MI) {
3505	// RVV lacks any support for immediate addressing for stack addresses, so be
3506	// conservative.
3507	unsigned Opcode = MI.getOpcode();
3508	if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
3509	!isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
3510	return false;
3511	return true;
3512	}
3513
3514	std::optional<std::pair<unsigned, unsigned>>
3515	RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
3516	switch (Opcode) {
3517	default:
3518	return std::nullopt;
3519	case RISCV::PseudoVSPILL2_M1:
3520	case RISCV::PseudoVRELOAD2_M1:
3521	return std::make_pair(x: 2u, y: 1u);
3522	case RISCV::PseudoVSPILL2_M2:
3523	case RISCV::PseudoVRELOAD2_M2:
3524	return std::make_pair(x: 2u, y: 2u);
3525	case RISCV::PseudoVSPILL2_M4:
3526	case RISCV::PseudoVRELOAD2_M4:
3527	return std::make_pair(x: 2u, y: 4u);
3528	case RISCV::PseudoVSPILL3_M1:
3529	case RISCV::PseudoVRELOAD3_M1:
3530	return std::make_pair(x: 3u, y: 1u);
3531	case RISCV::PseudoVSPILL3_M2:
3532	case RISCV::PseudoVRELOAD3_M2:
3533	return std::make_pair(x: 3u, y: 2u);
3534	case RISCV::PseudoVSPILL4_M1:
3535	case RISCV::PseudoVRELOAD4_M1:
3536	return std::make_pair(x: 4u, y: 1u);
3537	case RISCV::PseudoVSPILL4_M2:
3538	case RISCV::PseudoVRELOAD4_M2:
3539	return std::make_pair(x: 4u, y: 2u);
3540	case RISCV::PseudoVSPILL5_M1:
3541	case RISCV::PseudoVRELOAD5_M1:
3542	return std::make_pair(x: 5u, y: 1u);
3543	case RISCV::PseudoVSPILL6_M1:
3544	case RISCV::PseudoVRELOAD6_M1:
3545	return std::make_pair(x: 6u, y: 1u);
3546	case RISCV::PseudoVSPILL7_M1:
3547	case RISCV::PseudoVRELOAD7_M1:
3548	return std::make_pair(x: 7u, y: 1u);
3549	case RISCV::PseudoVSPILL8_M1:
3550	case RISCV::PseudoVRELOAD8_M1:
3551	return std::make_pair(x: 8u, y: 1u);
3552	}
3553	}
3554
3555	bool RISCV::isFaultFirstLoad(const MachineInstr &MI) {
3556	return MI.getNumExplicitDefs() == 2 &&
3557	MI.modifiesRegister(RISCV::VL, /*TRI=*/nullptr) && !MI.isInlineAsm();
3558	}
3559
3560	bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
3561	int16_t MI1FrmOpIdx =
3562	RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
3563	int16_t MI2FrmOpIdx =
3564	RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);
3565	if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
3566	return false;
3567	MachineOperand FrmOp1 = MI1.getOperand(i: MI1FrmOpIdx);
3568	MachineOperand FrmOp2 = MI2.getOperand(i: MI2FrmOpIdx);
3569	return FrmOp1.getImm() == FrmOp2.getImm();
3570	}
3571
3572	std::optional<unsigned>
3573	RISCV::getVectorLowDemandedScalarBits(uint16_t Opcode, unsigned Log2SEW) {
3574	// TODO: Handle Zvbb instructions
3575	switch (Opcode) {
3576	default:
3577	return std::nullopt;
3578
3579	// 11.6. Vector Single-Width Shift Instructions
3580	case RISCV::VSLL_VX:
3581	case RISCV::VSRL_VX:
3582	case RISCV::VSRA_VX:
3583	// 12.4. Vector Single-Width Scaling Shift Instructions
3584	case RISCV::VSSRL_VX:
3585	case RISCV::VSSRA_VX:
3586	// Only the low lg2(SEW) bits of the shift-amount value are used.
3587	return Log2SEW;
3588
3589	// 11.7 Vector Narrowing Integer Right Shift Instructions
3590	case RISCV::VNSRL_WX:
3591	case RISCV::VNSRA_WX:
3592	// 12.5. Vector Narrowing Fixed-Point Clip Instructions
3593	case RISCV::VNCLIPU_WX:
3594	case RISCV::VNCLIP_WX:
3595	// Only the low lg2(2*SEW) bits of the shift-amount value are used.
3596	return Log2SEW + 1;
3597
3598	// 11.1. Vector Single-Width Integer Add and Subtract
3599	case RISCV::VADD_VX:
3600	case RISCV::VSUB_VX:
3601	case RISCV::VRSUB_VX:
3602	// 11.2. Vector Widening Integer Add/Subtract
3603	case RISCV::VWADDU_VX:
3604	case RISCV::VWSUBU_VX:
3605	case RISCV::VWADD_VX:
3606	case RISCV::VWSUB_VX:
3607	case RISCV::VWADDU_WX:
3608	case RISCV::VWSUBU_WX:
3609	case RISCV::VWADD_WX:
3610	case RISCV::VWSUB_WX:
3611	// 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
3612	case RISCV::VADC_VXM:
3613	case RISCV::VADC_VIM:
3614	case RISCV::VMADC_VXM:
3615	case RISCV::VMADC_VIM:
3616	case RISCV::VMADC_VX:
3617	case RISCV::VSBC_VXM:
3618	case RISCV::VMSBC_VXM:
3619	case RISCV::VMSBC_VX:
3620	// 11.5 Vector Bitwise Logical Instructions
3621	case RISCV::VAND_VX:
3622	case RISCV::VOR_VX:
3623	case RISCV::VXOR_VX:
3624	// 11.8. Vector Integer Compare Instructions
3625	case RISCV::VMSEQ_VX:
3626	case RISCV::VMSNE_VX:
3627	case RISCV::VMSLTU_VX:
3628	case RISCV::VMSLT_VX:
3629	case RISCV::VMSLEU_VX:
3630	case RISCV::VMSLE_VX:
3631	case RISCV::VMSGTU_VX:
3632	case RISCV::VMSGT_VX:
3633	// 11.9. Vector Integer Min/Max Instructions
3634	case RISCV::VMINU_VX:
3635	case RISCV::VMIN_VX:
3636	case RISCV::VMAXU_VX:
3637	case RISCV::VMAX_VX:
3638	// 11.10. Vector Single-Width Integer Multiply Instructions
3639	case RISCV::VMUL_VX:
3640	case RISCV::VMULH_VX:
3641	case RISCV::VMULHU_VX:
3642	case RISCV::VMULHSU_VX:
3643	// 11.11. Vector Integer Divide Instructions
3644	case RISCV::VDIVU_VX:
3645	case RISCV::VDIV_VX:
3646	case RISCV::VREMU_VX:
3647	case RISCV::VREM_VX:
3648	// 11.12. Vector Widening Integer Multiply Instructions
3649	case RISCV::VWMUL_VX:
3650	case RISCV::VWMULU_VX:
3651	case RISCV::VWMULSU_VX:
3652	// 11.13. Vector Single-Width Integer Multiply-Add Instructions
3653	case RISCV::VMACC_VX:
3654	case RISCV::VNMSAC_VX:
3655	case RISCV::VMADD_VX:
3656	case RISCV::VNMSUB_VX:
3657	// 11.14. Vector Widening Integer Multiply-Add Instructions
3658	case RISCV::VWMACCU_VX:
3659	case RISCV::VWMACC_VX:
3660	case RISCV::VWMACCSU_VX:
3661	case RISCV::VWMACCUS_VX:
3662	// 11.15. Vector Integer Merge Instructions
3663	case RISCV::VMERGE_VXM:
3664	// 11.16. Vector Integer Move Instructions
3665	case RISCV::VMV_V_X:
3666	// 12.1. Vector Single-Width Saturating Add and Subtract
3667	case RISCV::VSADDU_VX:
3668	case RISCV::VSADD_VX:
3669	case RISCV::VSSUBU_VX:
3670	case RISCV::VSSUB_VX:
3671	// 12.2. Vector Single-Width Averaging Add and Subtract
3672	case RISCV::VAADDU_VX:
3673	case RISCV::VAADD_VX:
3674	case RISCV::VASUBU_VX:
3675	case RISCV::VASUB_VX:
3676	// 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
3677	case RISCV::VSMUL_VX:
3678	// 16.1. Integer Scalar Move Instructions
3679	case RISCV::VMV_S_X:
3680	return 1U << Log2SEW;
3681	}
3682	}
3683
3684	unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
3685	const RISCVVPseudosTable::PseudoInfo *RVV =
3686	RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
3687	if (!RVV)
3688	return 0;
3689	return RVV->BaseInstr;
3690	}
3691