1	//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	/// \file
10	/// SI Implementation of TargetInstrInfo.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#include "SIInstrInfo.h"
15	#include "AMDGPU.h"
16	#include "AMDGPUInstrInfo.h"
17	#include "GCNHazardRecognizer.h"
18	#include "GCNSubtarget.h"
19	#include "SIMachineFunctionInfo.h"
20	#include "Utils/AMDGPUBaseInfo.h"
21	#include "llvm/Analysis/ValueTracking.h"
22	#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23	#include "llvm/CodeGen/LiveIntervals.h"
24	#include "llvm/CodeGen/LiveVariables.h"
25	#include "llvm/CodeGen/MachineDominators.h"
26	#include "llvm/CodeGen/MachineFrameInfo.h"
27	#include "llvm/CodeGen/MachineScheduler.h"
28	#include "llvm/CodeGen/RegisterScavenging.h"
29	#include "llvm/CodeGen/ScheduleDAG.h"
30	#include "llvm/IR/DiagnosticInfo.h"
31	#include "llvm/IR/IntrinsicsAMDGPU.h"
32	#include "llvm/MC/MCContext.h"
33	#include "llvm/Support/CommandLine.h"
34	#include "llvm/Target/TargetMachine.h"
35
36	using namespace llvm;
37
38	#define DEBUG_TYPE "si-instr-info"
39
40	#define GET_INSTRINFO_CTOR_DTOR
41	#include "AMDGPUGenInstrInfo.inc"
42
43	namespace llvm {
44	namespace AMDGPU {
45	#define GET_D16ImageDimIntrinsics_IMPL
46	#define GET_ImageDimIntrinsicTable_IMPL
47	#define GET_RsrcIntrinsics_IMPL
48	#include "AMDGPUGenSearchableTables.inc"
49	}
50	}
51
52
53	// Must be at least 4 to be able to branch over minimum unconditional branch
54	// code. This is only for making it possible to write reasonably small tests for
55	// long branches.
56	static cl::opt<unsigned>
57	BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(Val: 16),
58	cl::desc("Restrict range of branch instructions (DEBUG)"));
59
60	static cl::opt<bool> Fix16BitCopies(
61	"amdgpu-fix-16-bit-physreg-copies",
62	cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
63	cl::init(Val: true),
64	cl::ReallyHidden);
65
66	SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
67	: AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
68	RI(ST), ST(ST) {
69	SchedModel.init(&ST);
70	}
71
72	//===----------------------------------------------------------------------===//
73	// TargetInstrInfo callbacks
74	//===----------------------------------------------------------------------===//
75
76	static unsigned getNumOperandsNoGlue(SDNode *Node) {
77	unsigned N = Node->getNumOperands();
78	while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
79	--N;
80	return N;
81	}
82
83	/// Returns true if both nodes have the same value for the given
84	/// operand \p Op, or if both nodes do not have this operand.
85	static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
86	unsigned Opc0 = N0->getMachineOpcode();
87	unsigned Opc1 = N1->getMachineOpcode();
88
89	int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, NamedIdx: OpName);
90	int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, NamedIdx: OpName);
91
92	if (Op0Idx == -1 && Op1Idx == -1)
93	return true;
94
95
96	if ((Op0Idx == -1 && Op1Idx != -1) ||
97	(Op1Idx == -1 && Op0Idx != -1))
98	return false;
99
100	// getNamedOperandIdx returns the index for the MachineInstr's operands,
101	// which includes the result as the first operand. We are indexing into the
102	// MachineSDNode's operands, so we need to skip the result operand to get
103	// the real index.
104	--Op0Idx;
105	--Op1Idx;
106
107	return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
108	}
109
110	static bool canRemat(const MachineInstr &MI) {
111
112	if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
113	SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
114	SIInstrInfo::isSALU(MI))
115	return true;
116
117	if (SIInstrInfo::isSMRD(MI)) {
118	return !MI.memoperands_empty() &&
119	llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
120	return MMO->isLoad() && MMO->isInvariant();
121	});
122	}
123
124	return false;
125	}
126
127	bool SIInstrInfo::isReallyTriviallyReMaterializable(
128	const MachineInstr &MI) const {
129
130	if (canRemat(MI)) {
131	// Normally VALU use of exec would block the rematerialization, but that
132	// is OK in this case to have an implicit exec read as all VALU do.
133	// We really want all of the generic logic for this except for this.
134
135	// Another potential implicit use is mode register. The core logic of
136	// the RA will not attempt rematerialization if mode is set anywhere
137	// in the function, otherwise it is safe since mode is not changed.
138
139	// There is difference to generic method which does not allow
140	// rematerialization if there are virtual register uses. We allow this,
141	// therefore this method includes SOP instructions as well.
142	if (!MI.hasImplicitDef() &&
143	MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
144	!MI.mayRaiseFPException())
145	return true;
146	}
147
148	return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
149	}
150
151	// Returns true if the scalar result of a VALU instruction depends on exec.
152	static bool resultDependsOnExec(const MachineInstr &MI) {
153	// Ignore comparisons which are only used masked with exec.
154	// This allows some hoisting/sinking of VALU comparisons.
155	if (MI.isCompare()) {
156	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
157	Register DstReg = MI.getOperand(i: 0).getReg();
158	if (!DstReg.isVirtual())
159	return true;
160	for (MachineInstr &Use : MRI.use_nodbg_instructions(Reg: DstReg)) {
161	switch (Use.getOpcode()) {
162	case AMDGPU::S_AND_SAVEEXEC_B32:
163	case AMDGPU::S_AND_SAVEEXEC_B64:
164	break;
165	case AMDGPU::S_AND_B32:
166	case AMDGPU::S_AND_B64:
167	if (!Use.readsRegister(AMDGPU::EXEC))
168	return true;
169	break;
170	default:
171	return true;
172	}
173	}
174	return false;
175	}
176
177	switch (MI.getOpcode()) {
178	default:
179	break;
180	case AMDGPU::V_READFIRSTLANE_B32:
181	return true;
182	}
183
184	return false;
185	}
186
187	bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
188	// Any implicit use of exec by VALU is not a real register read.
189	return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
190	isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent());
191	}
192
193	bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
194	MachineBasicBlock *SuccToSinkTo,
195	MachineCycleInfo *CI) const {
196	// Allow sinking if MI edits lane mask (divergent i1 in sgpr).
197	if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
198	return true;
199
200	MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
201	// Check if sinking of MI would create temporal divergent use.
202	for (auto Op : MI.uses()) {
203	if (Op.isReg() && Op.getReg().isVirtual() &&
204	RI.isSGPRClass(MRI.getRegClass(Reg: Op.getReg()))) {
205	MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
206
207	// SgprDef defined inside cycle
208	MachineCycle *FromCycle = CI->getCycle(Block: SgprDef->getParent());
209	if (FromCycle == nullptr)
210	continue;
211
212	MachineCycle *ToCycle = CI->getCycle(Block: SuccToSinkTo);
213	// Check if there is a FromCycle that contains SgprDef's basic block but
214	// does not contain SuccToSinkTo and also has divergent exit condition.
215	while (FromCycle && !FromCycle->contains(C: ToCycle)) {
216	// After structurize-cfg, there should be exactly one cycle exit.
217	SmallVector<MachineBasicBlock *, 1> ExitBlocks;
218	FromCycle->getExitBlocks(TmpStorage&: ExitBlocks);
219	assert(ExitBlocks.size() == 1);
220	assert(ExitBlocks[0]->getSinglePredecessor());
221
222	// FromCycle has divergent exit condition.
223	if (hasDivergentBranch(MBB: ExitBlocks[0]->getSinglePredecessor())) {
224	return false;
225	}
226
227	FromCycle = FromCycle->getParentCycle();
228	}
229	}
230	}
231
232	return true;
233	}
234
235	bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
236	int64_t &Offset0,
237	int64_t &Offset1) const {
238	if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
239	return false;
240
241	unsigned Opc0 = Load0->getMachineOpcode();
242	unsigned Opc1 = Load1->getMachineOpcode();
243
244	// Make sure both are actually loads.
245	if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
246	return false;
247
248	// A mayLoad instruction without a def is not a load. Likely a prefetch.
249	if (!get(Opc0).getNumDefs() || !get(Opc1).getNumDefs())
250	return false;
251
252	if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
253
254	// FIXME: Handle this case:
255	if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
256	return false;
257
258	// Check base reg.
259	if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
260	return false;
261
262	// Skip read2 / write2 variants for simplicity.
263	// TODO: We should report true if the used offsets are adjacent (excluded
264	// st64 versions).
265	int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
266	int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
267	if (Offset0Idx == -1 || Offset1Idx == -1)
268	return false;
269
270	// XXX - be careful of dataless loads
271	// getNamedOperandIdx returns the index for MachineInstrs. Since they
272	// include the output in the operand list, but SDNodes don't, we need to
273	// subtract the index by one.
274	Offset0Idx -= get(Opc0).NumDefs;
275	Offset1Idx -= get(Opc1).NumDefs;
276	Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
277	Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
278	return true;
279	}
280
281	if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
282	// Skip time and cache invalidation instructions.
283	if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) ||
284	!AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase))
285	return false;
286
287	unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
288	if (NumOps != getNumOperandsNoGlue(Node: Load1))
289	return false;
290
291	// Check base reg.
292	if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
293	return false;
294
295	// Match register offsets, if both register and immediate offsets present.
296	assert(NumOps == 4 || NumOps == 5);
297	if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
298	return false;
299
300	const ConstantSDNode *Load0Offset =
301	dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
302	const ConstantSDNode *Load1Offset =
303	dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
304
305	if (!Load0Offset || !Load1Offset)
306	return false;
307
308	Offset0 = Load0Offset->getZExtValue();
309	Offset1 = Load1Offset->getZExtValue();
310	return true;
311	}
312
313	// MUBUF and MTBUF can access the same addresses.
314	if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
315
316	// MUBUF and MTBUF have vaddr at different indices.
317	if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
318	!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
319	!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
320	return false;
321
322	int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
323	int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
324
325	if (OffIdx0 == -1 || OffIdx1 == -1)
326	return false;
327
328	// getNamedOperandIdx returns the index for MachineInstrs. Since they
329	// include the output in the operand list, but SDNodes don't, we need to
330	// subtract the index by one.
331	OffIdx0 -= get(Opc0).NumDefs;
332	OffIdx1 -= get(Opc1).NumDefs;
333
334	SDValue Off0 = Load0->getOperand(Num: OffIdx0);
335	SDValue Off1 = Load1->getOperand(Num: OffIdx1);
336
337	// The offset might be a FrameIndexSDNode.
338	if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
339	return false;
340
341	Offset0 = Off0->getAsZExtVal();
342	Offset1 = Off1->getAsZExtVal();
343	return true;
344	}
345
346	return false;
347	}
348
349	static bool isStride64(unsigned Opc) {
350	switch (Opc) {
351	case AMDGPU::DS_READ2ST64_B32:
352	case AMDGPU::DS_READ2ST64_B64:
353	case AMDGPU::DS_WRITE2ST64_B32:
354	case AMDGPU::DS_WRITE2ST64_B64:
355	return true;
356	default:
357	return false;
358	}
359	}
360
361	bool SIInstrInfo::getMemOperandsWithOffsetWidth(
362	const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
363	int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
364	const TargetRegisterInfo *TRI) const {
365	if (!LdSt.mayLoadOrStore())
366	return false;
367
368	unsigned Opc = LdSt.getOpcode();
369	OffsetIsScalable = false;
370	const MachineOperand *BaseOp, *OffsetOp;
371	int DataOpIdx;
372
373	if (isDS(MI: LdSt)) {
374	BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
375	OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
376	if (OffsetOp) {
377	// Normal, single offset LDS instruction.
378	if (!BaseOp) {
379	// DS_CONSUME/DS_APPEND use M0 for the base address.
380	// TODO: find the implicit use operand for M0 and use that as BaseOp?
381	return false;
382	}
383	BaseOps.push_back(Elt: BaseOp);
384	Offset = OffsetOp->getImm();
385	// Get appropriate operand, and compute width accordingly.
386	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
387	if (DataOpIdx == -1)
388	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
389	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
390	} else {
391	// The 2 offset instructions use offset0 and offset1 instead. We can treat
392	// these as a load with a single offset if the 2 offsets are consecutive.
393	// We will use this for some partially aligned loads.
394	const MachineOperand *Offset0Op =
395	getNamedOperand(LdSt, AMDGPU::OpName::offset0);
396	const MachineOperand *Offset1Op =
397	getNamedOperand(LdSt, AMDGPU::OpName::offset1);
398
399	unsigned Offset0 = Offset0Op->getImm() & 0xff;
400	unsigned Offset1 = Offset1Op->getImm() & 0xff;
401	if (Offset0 + 1 != Offset1)
402	return false;
403
404	// Each of these offsets is in element sized units, so we need to convert
405	// to bytes of the individual reads.
406
407	unsigned EltSize;
408	if (LdSt.mayLoad())
409	EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
410	else {
411	assert(LdSt.mayStore());
412	int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
413	EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
414	}
415
416	if (isStride64(Opc))
417	EltSize *= 64;
418
419	BaseOps.push_back(Elt: BaseOp);
420	Offset = EltSize * Offset0;
421	// Get appropriate operand(s), and compute width accordingly.
422	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
423	if (DataOpIdx == -1) {
424	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
425	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
426	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
427	Width += getOpSize(MI: LdSt, OpNo: DataOpIdx);
428	} else {
429	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
430	}
431	}
432	return true;
433	}
434
435	if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
436	const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
437	if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
438	return false;
439	BaseOps.push_back(Elt: RSrc);
440	BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
441	if (BaseOp && !BaseOp->isFI())
442	BaseOps.push_back(Elt: BaseOp);
443	const MachineOperand *OffsetImm =
444	getNamedOperand(LdSt, AMDGPU::OpName::offset);
445	Offset = OffsetImm->getImm();
446	const MachineOperand *SOffset =
447	getNamedOperand(LdSt, AMDGPU::OpName::soffset);
448	if (SOffset) {
449	if (SOffset->isReg())
450	BaseOps.push_back(Elt: SOffset);
451	else
452	Offset += SOffset->getImm();
453	}
454	// Get appropriate operand, and compute width accordingly.
455	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
456	if (DataOpIdx == -1)
457	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
458	if (DataOpIdx == -1) // LDS DMA
459	return false;
460	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
461	return true;
462	}
463
464	if (isMIMG(MI: LdSt)) {
465	int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
466	BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
467	int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
468	if (VAddr0Idx >= 0) {
469	// GFX10 possible NSA encoding.
470	for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
471	BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
472	} else {
473	BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
474	}
475	Offset = 0;
476	// Get appropriate operand, and compute width accordingly.
477	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
478	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
479	return true;
480	}
481
482	if (isSMRD(MI: LdSt)) {
483	BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
484	if (!BaseOp) // e.g. S_MEMTIME
485	return false;
486	BaseOps.push_back(Elt: BaseOp);
487	OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
488	Offset = OffsetOp ? OffsetOp->getImm() : 0;
489	// Get appropriate operand, and compute width accordingly.
490	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
491	if (DataOpIdx == -1)
492	return false;
493	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
494	return true;
495	}
496
497	if (isFLAT(MI: LdSt)) {
498	// Instructions have either vaddr or saddr or both or none.
499	BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
500	if (BaseOp)
501	BaseOps.push_back(Elt: BaseOp);
502	BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
503	if (BaseOp)
504	BaseOps.push_back(Elt: BaseOp);
505	Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
506	// Get appropriate operand, and compute width accordingly.
507	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
508	if (DataOpIdx == -1)
509	DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
510	if (DataOpIdx == -1) // LDS DMA
511	return false;
512	Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
513	return true;
514	}
515
516	return false;
517	}
518
519	static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
520	ArrayRef<const MachineOperand *> BaseOps1,
521	const MachineInstr &MI2,
522	ArrayRef<const MachineOperand *> BaseOps2) {
523	// Only examine the first "base" operand of each instruction, on the
524	// assumption that it represents the real base address of the memory access.
525	// Other operands are typically offsets or indices from this base address.
526	if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
527	return true;
528
529	if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
530	return false;
531
532	auto MO1 = *MI1.memoperands_begin();
533	auto MO2 = *MI2.memoperands_begin();
534	if (MO1->getAddrSpace() != MO2->getAddrSpace())
535	return false;
536
537	auto Base1 = MO1->getValue();
538	auto Base2 = MO2->getValue();
539	if (!Base1 || !Base2)
540	return false;
541	Base1 = getUnderlyingObject(V: Base1);
542	Base2 = getUnderlyingObject(V: Base2);
543
544	if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
545	return false;
546
547	return Base1 == Base2;
548	}
549
550	bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
551	int64_t Offset1, bool OffsetIsScalable1,
552	ArrayRef<const MachineOperand *> BaseOps2,
553	int64_t Offset2, bool OffsetIsScalable2,
554	unsigned ClusterSize,
555	unsigned NumBytes) const {
556	// If the mem ops (to be clustered) do not have the same base ptr, then they
557	// should not be clustered
558	if (!BaseOps1.empty() && !BaseOps2.empty()) {
559	const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
560	const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
561	if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
562	return false;
563	} else if (!BaseOps1.empty() || !BaseOps2.empty()) {
564	// If only one base op is empty, they do not have the same base ptr
565	return false;
566	}
567
568	// In order to avoid register pressure, on an average, the number of DWORDS
569	// loaded together by all clustered mem ops should not exceed 8. This is an
570	// empirical value based on certain observations and performance related
571	// experiments.
572	// The good thing about this heuristic is - it avoids clustering of too many
573	// sub-word loads, and also avoids clustering of wide loads. Below is the
574	// brief summary of how the heuristic behaves for various `LoadSize`.
575	// (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
576	// (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
577	// (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
578	// (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
579	// (5) LoadSize >= 17: do not cluster
580	const unsigned LoadSize = NumBytes / ClusterSize;
581	const unsigned NumDWORDs = ((LoadSize + 3) / 4) * ClusterSize;
582	return NumDWORDs <= 8;
583	}
584
585	// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
586	// the first 16 loads will be interleaved with the stores, and the next 16 will
587	// be clustered as expected. It should really split into 2 16 store batches.
588	//
589	// Loads are clustered until this returns false, rather than trying to schedule
590	// groups of stores. This also means we have to deal with saying different
591	// address space loads should be clustered, and ones which might cause bank
592	// conflicts.
593	//
594	// This might be deprecated so it might not be worth that much effort to fix.
595	bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
596	int64_t Offset0, int64_t Offset1,
597	unsigned NumLoads) const {
598	assert(Offset1 > Offset0 &&
599	"Second offset should be larger than first offset!");
600	// If we have less than 16 loads in a row, and the offsets are within 64
601	// bytes, then schedule together.
602
603	// A cacheline is 64 bytes (for global memory).
604	return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
605	}
606
607	static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
608	MachineBasicBlock::iterator MI,
609	const DebugLoc &DL, MCRegister DestReg,
610	MCRegister SrcReg, bool KillSrc,
611	const char *Msg = "illegal VGPR to SGPR copy") {
612	MachineFunction *MF = MBB.getParent();
613	DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error);
614	LLVMContext &C = MF->getFunction().getContext();
615	C.diagnose(DI: IllegalCopy);
616
617	BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
618	.addReg(SrcReg, getKillRegState(KillSrc));
619	}
620
621	/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
622	/// possible to have a direct copy in these cases on GFX908, so an intermediate
623	/// VGPR copy is required.
624	static void indirectCopyToAGPR(const SIInstrInfo &TII,
625	MachineBasicBlock &MBB,
626	MachineBasicBlock::iterator MI,
627	const DebugLoc &DL, MCRegister DestReg,
628	MCRegister SrcReg, bool KillSrc,
629	RegScavenger &RS, bool RegsOverlap,
630	Register ImpDefSuperReg = Register(),
631	Register ImpUseSuperReg = Register()) {
632	assert((TII.getSubtarget().hasMAIInsts() &&
633	!TII.getSubtarget().hasGFX90AInsts()) &&
634	"Expected GFX908 subtarget.");
635
636	assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
637	AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
638	"Source register of the copy should be either an SGPR or an AGPR.");
639
640	assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
641	"Destination register of the copy should be an AGPR.");
642
643	const SIRegisterInfo &RI = TII.getRegisterInfo();
644
645	// First try to find defining accvgpr_write to avoid temporary registers.
646	// In the case of copies of overlapping AGPRs, we conservatively do not
647	// reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
648	// an accvgpr_write used for this same copy due to implicit-defs
649	if (!RegsOverlap) {
650	for (auto Def = MI, E = MBB.begin(); Def != E; ) {
651	--Def;
652
653	if (!Def->modifiesRegister(SrcReg, &RI))
654	continue;
655
656	if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
657	Def->getOperand(0).getReg() != SrcReg)
658	break;
659
660	MachineOperand &DefOp = Def->getOperand(i: 1);
661	assert(DefOp.isReg() || DefOp.isImm());
662
663	if (DefOp.isReg()) {
664	bool SafeToPropagate = true;
665	// Check that register source operand is not clobbered before MI.
666	// Immediate operands are always safe to propagate.
667	for (auto I = Def; I != MI && SafeToPropagate; ++I)
668	if (I->modifiesRegister(DefOp.getReg(), &RI))
669	SafeToPropagate = false;
670
671	if (!SafeToPropagate)
672	break;
673
674	DefOp.setIsKill(false);
675	}
676
677	MachineInstrBuilder Builder =
678	BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
679	.add(DefOp);
680	if (ImpDefSuperReg)
681	Builder.addReg(RegNo: ImpDefSuperReg, flags: RegState::Define | RegState::Implicit);
682
683	if (ImpUseSuperReg) {
684	Builder.addReg(RegNo: ImpUseSuperReg,
685	flags: getKillRegState(B: KillSrc) | RegState::Implicit);
686	}
687
688	return;
689	}
690	}
691
692	RS.enterBasicBlockEnd(MBB);
693	RS.backward(I: std::next(x: MI));
694
695	// Ideally we want to have three registers for a long reg_sequence copy
696	// to hide 2 waitstates between v_mov_b32 and accvgpr_write.
697	unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
698	*MBB.getParent());
699
700	// Registers in the sequence are allocated contiguously so we can just
701	// use register number to pick one of three round-robin temps.
702	unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
703	Register Tmp =
704	MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
705	assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
706	"VGPR used for an intermediate copy should have been reserved.");
707
708	// Only loop through if there are any free registers left. We don't want to
709	// spill.
710	while (RegNo--) {
711	Register Tmp2 = RS.scavengeRegisterBackwards(AMDGPU::VGPR_32RegClass, MI,
712	/* RestoreAfter */ false, 0,
713	/* AllowSpill */ false);
714	if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
715	break;
716	Tmp = Tmp2;
717	RS.setRegUsed(Reg: Tmp);
718	}
719
720	// Insert copy to temporary VGPR.
721	unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
722	if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
723	TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
724	} else {
725	assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
726	}
727
728	MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)
729	.addReg(SrcReg, getKillRegState(B: KillSrc));
730	if (ImpUseSuperReg) {
731	UseBuilder.addReg(RegNo: ImpUseSuperReg,
732	flags: getKillRegState(B: KillSrc) | RegState::Implicit);
733	}
734
735	MachineInstrBuilder DefBuilder
736	= BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
737	.addReg(Tmp, RegState::Kill);
738
739	if (ImpDefSuperReg)
740	DefBuilder.addReg(RegNo: ImpDefSuperReg, flags: RegState::Define | RegState::Implicit);
741	}
742
743	static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
744	MachineBasicBlock::iterator MI, const DebugLoc &DL,
745	MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
746	const TargetRegisterClass *RC, bool Forward) {
747	const SIRegisterInfo &RI = TII.getRegisterInfo();
748	ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4);
749	MachineBasicBlock::iterator I = MI;
750	MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
751
752	for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
753	int16_t SubIdx = BaseIndices[Idx];
754	Register DestSubReg = RI.getSubReg(DestReg, SubIdx);
755	Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
756	assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
757	unsigned Opcode = AMDGPU::S_MOV_B32;
758
759	// Is SGPR aligned? If so try to combine with next.
760	bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
761	bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
762	if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
763	// Can use SGPR64 copy
764	unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
765	SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
766	DestSubReg = RI.getSubReg(DestReg, SubIdx);
767	SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
768	assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
769	Opcode = AMDGPU::S_MOV_B64;
770	Idx++;
771	}
772
773	LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), DestSubReg)
774	.addReg(SrcSubReg)
775	.addReg(SrcReg, RegState::Implicit);
776
777	if (!FirstMI)
778	FirstMI = LastMI;
779
780	if (!Forward)
781	I--;
782	}
783
784	assert(FirstMI && LastMI);
785	if (!Forward)
786	std::swap(a&: FirstMI, b&: LastMI);
787
788	FirstMI->addOperand(
789	Op: MachineOperand::CreateReg(Reg: DestReg, isDef: true /*IsDef*/, isImp: true /*IsImp*/));
790
791	if (KillSrc)
792	LastMI->addRegisterKilled(SrcReg, &RI);
793	}
794
795	void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
796	MachineBasicBlock::iterator MI,
797	const DebugLoc &DL, MCRegister DestReg,
798	MCRegister SrcReg, bool KillSrc) const {
799	const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg);
800	unsigned Size = RI.getRegSizeInBits(*RC);
801	const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg);
802	unsigned SrcSize = RI.getRegSizeInBits(*SrcRC);
803
804	// The rest of copyPhysReg assumes Src and Dst size are the same size.
805	// TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
806	// we remove Fix16BitCopies and this code block?
807	if (Fix16BitCopies) {
808	if (((Size == 16) != (SrcSize == 16))) {
809	// Non-VGPR Src and Dst will later be expanded back to 32 bits.
810	assert(ST.hasTrue16BitInsts());
811	MCRegister &RegToFix = (Size == 32) ? DestReg : SrcReg;
812	MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16);
813	RegToFix = SubReg;
814
815	if (DestReg == SrcReg) {
816	// Identity copy. Insert empty bundle since ExpandPostRA expects an
817	// instruction here.
818	BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
819	return;
820	}
821	RC = RI.getPhysRegBaseClass(DestReg);
822	Size = RI.getRegSizeInBits(*RC);
823	SrcRC = RI.getPhysRegBaseClass(SrcReg);
824	SrcSize = RI.getRegSizeInBits(*SrcRC);
825	}
826	}
827
828	if (RC == &AMDGPU::VGPR_32RegClass) {
829	assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
830	AMDGPU::SReg_32RegClass.contains(SrcReg) ||
831	AMDGPU::AGPR_32RegClass.contains(SrcReg));
832	unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
833	AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
834	BuildMI(MBB, MI, DL, get(Opc), DestReg)
835	.addReg(SrcReg, getKillRegState(B: KillSrc));
836	return;
837	}
838
839	if (RC == &AMDGPU::SReg_32_XM0RegClass ||
840	RC == &AMDGPU::SReg_32RegClass) {
841	if (SrcReg == AMDGPU::SCC) {
842	BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
843	.addImm(1)
844	.addImm(0);
845	return;
846	}
847
848	if (DestReg == AMDGPU::VCC_LO) {
849	if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
850	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
851	.addReg(SrcReg, getKillRegState(KillSrc));
852	} else {
853	// FIXME: Hack until VReg_1 removed.
854	assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
855	BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
856	.addImm(0)
857	.addReg(SrcReg, getKillRegState(KillSrc));
858	}
859
860	return;
861	}
862
863	if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
864	reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
865	return;
866	}
867
868	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
869	.addReg(SrcReg, getKillRegState(KillSrc));
870	return;
871	}
872
873	if (RC == &AMDGPU::SReg_64RegClass) {
874	if (SrcReg == AMDGPU::SCC) {
875	BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
876	.addImm(1)
877	.addImm(0);
878	return;
879	}
880
881	if (DestReg == AMDGPU::VCC) {
882	if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
883	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
884	.addReg(SrcReg, getKillRegState(KillSrc));
885	} else {
886	// FIXME: Hack until VReg_1 removed.
887	assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
888	BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
889	.addImm(0)
890	.addReg(SrcReg, getKillRegState(KillSrc));
891	}
892
893	return;
894	}
895
896	if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
897	reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
898	return;
899	}
900
901	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
902	.addReg(SrcReg, getKillRegState(KillSrc));
903	return;
904	}
905
906	if (DestReg == AMDGPU::SCC) {
907	// Copying 64-bit or 32-bit sources to SCC barely makes sense,
908	// but SelectionDAG emits such copies for i1 sources.
909	if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
910	// This copy can only be produced by patterns
911	// with explicit SCC, which are known to be enabled
912	// only for subtargets with S_CMP_LG_U64 present.
913	assert(ST.hasScalarCompareEq64());
914	BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))
915	.addReg(SrcReg, getKillRegState(KillSrc))
916	.addImm(0);
917	} else {
918	assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
919	BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
920	.addReg(SrcReg, getKillRegState(KillSrc))
921	.addImm(0);
922	}
923
924	return;
925	}
926
927	if (RC == &AMDGPU::AGPR_32RegClass) {
928	if (AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
929	(ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {
930	BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
931	.addReg(SrcReg, getKillRegState(KillSrc));
932	return;
933	}
934
935	if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {
936	BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
937	.addReg(SrcReg, getKillRegState(KillSrc));
938	return;
939	}
940
941	// FIXME: Pass should maintain scavenger to avoid scan through the block on
942	// every AGPR spill.
943	RegScavenger RS;
944	const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
945	indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
946	return;
947	}
948
949	if (Size == 16) {
950	assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
951	AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
952	AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
953
954	bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
955	bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
956	bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
957	bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
958	bool DstLow = !AMDGPU::isHi(DestReg, RI);
959	bool SrcLow = !AMDGPU::isHi(SrcReg, RI);
960	MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
961	MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
962
963	if (IsSGPRDst) {
964	if (!IsSGPRSrc) {
965	reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
966	return;
967	}
968
969	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)
970	.addReg(NewSrcReg, getKillRegState(KillSrc));
971	return;
972	}
973
974	if (IsAGPRDst || IsAGPRSrc) {
975	if (!DstLow || !SrcLow) {
976	reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
977	Msg: "Cannot use hi16 subreg with an AGPR!");
978	}
979
980	copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
981	return;
982	}
983
984	if (ST.hasTrue16BitInsts()) {
985	if (IsSGPRSrc) {
986	assert(SrcLow);
987	SrcReg = NewSrcReg;
988	}
989	// Use the smaller instruction encoding if possible.
990	if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) &&
991	(IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) {
992	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg)
993	.addReg(SrcReg);
994	} else {
995	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg)
996	.addImm(0) // src0_modifiers
997	.addReg(SrcReg)
998	.addImm(0); // op_sel
999	}
1000	return;
1001	}
1002
1003	if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1004	if (!DstLow || !SrcLow) {
1005	reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1006	Msg: "Cannot use hi16 subreg on VI!");
1007	}
1008
1009	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
1010	.addReg(NewSrcReg, getKillRegState(KillSrc));
1011	return;
1012	}
1013
1014	auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
1015	.addImm(0) // src0_modifiers
1016	.addReg(NewSrcReg)
1017	.addImm(0) // clamp
1018	.addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1019	: AMDGPU::SDWA::SdwaSel::WORD_1)
1020	.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1021	.addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1022	: AMDGPU::SDWA::SdwaSel::WORD_1)
1023	.addReg(NewDestReg, RegState::Implicit | RegState::Undef);
1024	// First implicit operand is $exec.
1025	MIB->tieOperands(0, MIB->getNumOperands() - 1);
1026	return;
1027	}
1028
1029	if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
1030	if (ST.hasMovB64()) {
1031	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)
1032	.addReg(SrcReg, getKillRegState(KillSrc));
1033	return;
1034	}
1035	if (ST.hasPkMovB32()) {
1036	BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
1037	.addImm(SISrcMods::OP_SEL_1)
1038	.addReg(SrcReg)
1039	.addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1040	.addReg(SrcReg)
1041	.addImm(0) // op_sel_lo
1042	.addImm(0) // op_sel_hi
1043	.addImm(0) // neg_lo
1044	.addImm(0) // neg_hi
1045	.addImm(0) // clamp
1046	.addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);
1047	return;
1048	}
1049	}
1050
1051	const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1052	if (RI.isSGPRClass(RC)) {
1053	if (!RI.isSGPRClass(SrcRC)) {
1054	reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1055	return;
1056	}
1057	const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
1058	expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1059	Forward);
1060	return;
1061	}
1062
1063	unsigned EltSize = 4;
1064	unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1065	if (RI.isAGPRClass(RC)) {
1066	if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))
1067	Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1068	else if (RI.hasVGPRs(SrcRC) ||
1069	(ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC)))
1070	Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1071	else
1072	Opcode = AMDGPU::INSTRUCTION_LIST_END;
1073	} else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {
1074	Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1075	} else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
1076	(RI.isProperlyAlignedRC(*RC) &&
1077	(SrcRC == RC || RI.isSGPRClass(SrcRC)))) {
1078	// TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
1079	if (ST.hasMovB64()) {
1080	Opcode = AMDGPU::V_MOV_B64_e32;
1081	EltSize = 8;
1082	} else if (ST.hasPkMovB32()) {
1083	Opcode = AMDGPU::V_PK_MOV_B32;
1084	EltSize = 8;
1085	}
1086	}
1087
1088	// For the cases where we need an intermediate instruction/temporary register
1089	// (destination is an AGPR), we need a scavenger.
1090	//
1091	// FIXME: The pass should maintain this for us so we don't have to re-scan the
1092	// whole block for every handled copy.
1093	std::unique_ptr<RegScavenger> RS;
1094	if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1095	RS.reset(p: new RegScavenger());
1096
1097	ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
1098
1099	// If there is an overlap, we can't kill the super-register on the last
1100	// instruction, since it will also kill the components made live by this def.
1101	const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
1102	const bool CanKillSuperReg = KillSrc && !Overlap;
1103
1104	for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1105	unsigned SubIdx;
1106	if (Forward)
1107	SubIdx = SubIndices[Idx];
1108	else
1109	SubIdx = SubIndices[SubIndices.size() - Idx - 1];
1110	Register DestSubReg = RI.getSubReg(DestReg, SubIdx);
1111	Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
1112	assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1113
1114	bool IsFirstSubreg = Idx == 0;
1115	bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
1116
1117	if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1118	Register ImpDefSuper = IsFirstSubreg ? Register(DestReg) : Register();
1119	Register ImpUseSuper = SrcReg;
1120	indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1121	RS&: *RS, RegsOverlap: Overlap, ImpDefSuperReg: ImpDefSuper, ImpUseSuperReg: ImpUseSuper);
1122	} else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1123	MachineInstrBuilder MIB =
1124	BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestSubReg)
1125	.addImm(SISrcMods::OP_SEL_1)
1126	.addReg(SrcSubReg)
1127	.addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1128	.addReg(SrcSubReg)
1129	.addImm(0) // op_sel_lo
1130	.addImm(0) // op_sel_hi
1131	.addImm(0) // neg_lo
1132	.addImm(0) // neg_hi
1133	.addImm(0) // clamp
1134	.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1135	if (IsFirstSubreg)
1136	MIB.addReg(RegNo: DestReg, flags: RegState::Define | RegState::Implicit);
1137	} else {
1138	MachineInstrBuilder Builder =
1139	BuildMI(MBB, MI, DL, get(Opcode), DestSubReg).addReg(SrcSubReg);
1140	if (IsFirstSubreg)
1141	Builder.addReg(RegNo: DestReg, flags: RegState::Define | RegState::Implicit);
1142
1143	Builder.addReg(RegNo: SrcReg, flags: getKillRegState(B: UseKill) | RegState::Implicit);
1144	}
1145	}
1146	}
1147
1148	int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1149	int NewOpc;
1150
1151	// Try to map original to commuted opcode
1152	NewOpc = AMDGPU::getCommuteRev(Opcode);
1153	if (NewOpc != -1)
1154	// Check if the commuted (REV) opcode exists on the target.
1155	return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1156
1157	// Try to map commuted to original opcode
1158	NewOpc = AMDGPU::getCommuteOrig(Opcode);
1159	if (NewOpc != -1)
1160	// Check if the original (non-REV) opcode exists on the target.
1161	return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1162
1163	return Opcode;
1164	}
1165
1166	void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
1167	MachineBasicBlock::iterator MI,
1168	const DebugLoc &DL, Register DestReg,
1169	int64_t Value) const {
1170	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1171	const TargetRegisterClass *RegClass = MRI.getRegClass(Reg: DestReg);
1172	if (RegClass == &AMDGPU::SReg_32RegClass ||
1173	RegClass == &AMDGPU::SGPR_32RegClass ||
1174	RegClass == &AMDGPU::SReg_32_XM0RegClass ||
1175	RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
1176	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
1177	.addImm(Value);
1178	return;
1179	}
1180
1181	if (RegClass == &AMDGPU::SReg_64RegClass ||
1182	RegClass == &AMDGPU::SGPR_64RegClass ||
1183	RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
1184	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
1185	.addImm(Value);
1186	return;
1187	}
1188
1189	if (RegClass == &AMDGPU::VGPR_32RegClass) {
1190	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
1191	.addImm(Value);
1192	return;
1193	}
1194	if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) {
1195	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
1196	.addImm(Value);
1197	return;
1198	}
1199
1200	unsigned EltSize = 4;
1201	unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1202	if (RI.isSGPRClass(RegClass)) {
1203	if (RI.getRegSizeInBits(*RegClass) > 32) {
1204	Opcode = AMDGPU::S_MOV_B64;
1205	EltSize = 8;
1206	} else {
1207	Opcode = AMDGPU::S_MOV_B32;
1208	EltSize = 4;
1209	}
1210	}
1211
1212	ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
1213	for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1214	int64_t IdxValue = Idx == 0 ? Value : 0;
1215
1216	MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
1217	get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx]));
1218	Builder.addImm(Val: IdxValue);
1219	}
1220	}
1221
1222	const TargetRegisterClass *
1223	SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
1224	return &AMDGPU::VGPR_32RegClass;
1225	}
1226
1227	void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
1228	MachineBasicBlock::iterator I,
1229	const DebugLoc &DL, Register DstReg,
1230	ArrayRef<MachineOperand> Cond,
1231	Register TrueReg,
1232	Register FalseReg) const {
1233	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1234	const TargetRegisterClass *BoolXExecRC =
1235	RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
1236	assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
1237	"Not a VGPR32 reg");
1238
1239	if (Cond.size() == 1) {
1240	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1241	BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1242	.add(Cond[0]);
1243	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1244	.addImm(0)
1245	.addReg(FalseReg)
1246	.addImm(0)
1247	.addReg(TrueReg)
1248	.addReg(SReg);
1249	} else if (Cond.size() == 2) {
1250	assert(Cond[0].isImm() && "Cond[0] is not an immediate");
1251	switch (Cond[0].getImm()) {
1252	case SIInstrInfo::SCC_TRUE: {
1253	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1254	BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1255	: AMDGPU::S_CSELECT_B64), SReg)
1256	.addImm(1)
1257	.addImm(0);
1258	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1259	.addImm(0)
1260	.addReg(FalseReg)
1261	.addImm(0)
1262	.addReg(TrueReg)
1263	.addReg(SReg);
1264	break;
1265	}
1266	case SIInstrInfo::SCC_FALSE: {
1267	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1268	BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1269	: AMDGPU::S_CSELECT_B64), SReg)
1270	.addImm(0)
1271	.addImm(1);
1272	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1273	.addImm(0)
1274	.addReg(FalseReg)
1275	.addImm(0)
1276	.addReg(TrueReg)
1277	.addReg(SReg);
1278	break;
1279	}
1280	case SIInstrInfo::VCCNZ: {
1281	MachineOperand RegOp = Cond[1];
1282	RegOp.setImplicit(false);
1283	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1284	BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1285	.add(RegOp);
1286	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1287	.addImm(0)
1288	.addReg(FalseReg)
1289	.addImm(0)
1290	.addReg(TrueReg)
1291	.addReg(SReg);
1292	break;
1293	}
1294	case SIInstrInfo::VCCZ: {
1295	MachineOperand RegOp = Cond[1];
1296	RegOp.setImplicit(false);
1297	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1298	BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1299	.add(RegOp);
1300	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1301	.addImm(0)
1302	.addReg(TrueReg)
1303	.addImm(0)
1304	.addReg(FalseReg)
1305	.addReg(SReg);
1306	break;
1307	}
1308	case SIInstrInfo::EXECNZ: {
1309	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1310	Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
1311	BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1312	: AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1313	.addImm(0);
1314	BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1315	: AMDGPU::S_CSELECT_B64), SReg)
1316	.addImm(1)
1317	.addImm(0);
1318	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1319	.addImm(0)
1320	.addReg(FalseReg)
1321	.addImm(0)
1322	.addReg(TrueReg)
1323	.addReg(SReg);
1324	break;
1325	}
1326	case SIInstrInfo::EXECZ: {
1327	Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1328	Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
1329	BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1330	: AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1331	.addImm(0);
1332	BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1333	: AMDGPU::S_CSELECT_B64), SReg)
1334	.addImm(0)
1335	.addImm(1);
1336	BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1337	.addImm(0)
1338	.addReg(FalseReg)
1339	.addImm(0)
1340	.addReg(TrueReg)
1341	.addReg(SReg);
1342	llvm_unreachable("Unhandled branch predicate EXECZ");
1343	break;
1344	}
1345	default:
1346	llvm_unreachable("invalid branch predicate");
1347	}
1348	} else {
1349	llvm_unreachable("Can only handle Cond size 1 or 2");
1350	}
1351	}
1352
1353	Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
1354	MachineBasicBlock::iterator I,
1355	const DebugLoc &DL,
1356	Register SrcReg, int Value) const {
1357	MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1358	Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
1359	BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
1360	.addImm(Value)
1361	.addReg(SrcReg);
1362
1363	return Reg;
1364	}
1365
1366	Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
1367	MachineBasicBlock::iterator I,
1368	const DebugLoc &DL,
1369	Register SrcReg, int Value) const {
1370	MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1371	Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
1372	BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
1373	.addImm(Value)
1374	.addReg(SrcReg);
1375
1376	return Reg;
1377	}
1378
1379	unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1380
1381	if (RI.isAGPRClass(DstRC))
1382	return AMDGPU::COPY;
1383	if (RI.getRegSizeInBits(*DstRC) == 16) {
1384	// Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1385	// before RA.
1386	return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1387	} else if (RI.getRegSizeInBits(*DstRC) == 32) {
1388	return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1389	} else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
1390	return AMDGPU::S_MOV_B64;
1391	} else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
1392	return AMDGPU::V_MOV_B64_PSEUDO;
1393	}
1394	return AMDGPU::COPY;
1395	}
1396
1397	const MCInstrDesc &
1398	SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1399	bool IsIndirectSrc) const {
1400	if (IsIndirectSrc) {
1401	if (VecSize <= 32) // 4 bytes
1402	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1403	if (VecSize <= 64) // 8 bytes
1404	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1405	if (VecSize <= 96) // 12 bytes
1406	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1407	if (VecSize <= 128) // 16 bytes
1408	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1409	if (VecSize <= 160) // 20 bytes
1410	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1411	if (VecSize <= 256) // 32 bytes
1412	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1413	if (VecSize <= 288) // 36 bytes
1414	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1415	if (VecSize <= 320) // 40 bytes
1416	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1417	if (VecSize <= 352) // 44 bytes
1418	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1419	if (VecSize <= 384) // 48 bytes
1420	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1421	if (VecSize <= 512) // 64 bytes
1422	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1423	if (VecSize <= 1024) // 128 bytes
1424	return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1425
1426	llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1427	}
1428
1429	if (VecSize <= 32) // 4 bytes
1430	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1431	if (VecSize <= 64) // 8 bytes
1432	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1433	if (VecSize <= 96) // 12 bytes
1434	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1435	if (VecSize <= 128) // 16 bytes
1436	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1437	if (VecSize <= 160) // 20 bytes
1438	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1439	if (VecSize <= 256) // 32 bytes
1440	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1441	if (VecSize <= 288) // 36 bytes
1442	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1443	if (VecSize <= 320) // 40 bytes
1444	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1445	if (VecSize <= 352) // 44 bytes
1446	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1447	if (VecSize <= 384) // 48 bytes
1448	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1449	if (VecSize <= 512) // 64 bytes
1450	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1451	if (VecSize <= 1024) // 128 bytes
1452	return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1453
1454	llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1455	}
1456
1457	static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1458	if (VecSize <= 32) // 4 bytes
1459	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1460	if (VecSize <= 64) // 8 bytes
1461	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1462	if (VecSize <= 96) // 12 bytes
1463	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1464	if (VecSize <= 128) // 16 bytes
1465	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1466	if (VecSize <= 160) // 20 bytes
1467	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1468	if (VecSize <= 256) // 32 bytes
1469	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1470	if (VecSize <= 288) // 36 bytes
1471	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1472	if (VecSize <= 320) // 40 bytes
1473	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1474	if (VecSize <= 352) // 44 bytes
1475	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1476	if (VecSize <= 384) // 48 bytes
1477	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1478	if (VecSize <= 512) // 64 bytes
1479	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1480	if (VecSize <= 1024) // 128 bytes
1481	return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1482
1483	llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1484	}
1485
1486	static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1487	if (VecSize <= 32) // 4 bytes
1488	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1489	if (VecSize <= 64) // 8 bytes
1490	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1491	if (VecSize <= 96) // 12 bytes
1492	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1493	if (VecSize <= 128) // 16 bytes
1494	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1495	if (VecSize <= 160) // 20 bytes
1496	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1497	if (VecSize <= 256) // 32 bytes
1498	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1499	if (VecSize <= 288) // 36 bytes
1500	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1501	if (VecSize <= 320) // 40 bytes
1502	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1503	if (VecSize <= 352) // 44 bytes
1504	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1505	if (VecSize <= 384) // 48 bytes
1506	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1507	if (VecSize <= 512) // 64 bytes
1508	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1509	if (VecSize <= 1024) // 128 bytes
1510	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1511
1512	llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1513	}
1514
1515	static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1516	if (VecSize <= 64) // 8 bytes
1517	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1518	if (VecSize <= 128) // 16 bytes
1519	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1520	if (VecSize <= 256) // 32 bytes
1521	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1522	if (VecSize <= 512) // 64 bytes
1523	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1524	if (VecSize <= 1024) // 128 bytes
1525	return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1526
1527	llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1528	}
1529
1530	const MCInstrDesc &
1531	SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1532	bool IsSGPR) const {
1533	if (IsSGPR) {
1534	switch (EltSize) {
1535	case 32:
1536	return get(getIndirectSGPRWriteMovRelPseudo32(VecSize));
1537	case 64:
1538	return get(getIndirectSGPRWriteMovRelPseudo64(VecSize));
1539	default:
1540	llvm_unreachable("invalid reg indexing elt size");
1541	}
1542	}
1543
1544	assert(EltSize == 32 && "invalid reg indexing elt size");
1545	return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1546	}
1547
1548	static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
1549	switch (Size) {
1550	case 4:
1551	return AMDGPU::SI_SPILL_S32_SAVE;
1552	case 8:
1553	return AMDGPU::SI_SPILL_S64_SAVE;
1554	case 12:
1555	return AMDGPU::SI_SPILL_S96_SAVE;
1556	case 16:
1557	return AMDGPU::SI_SPILL_S128_SAVE;
1558	case 20:
1559	return AMDGPU::SI_SPILL_S160_SAVE;
1560	case 24:
1561	return AMDGPU::SI_SPILL_S192_SAVE;
1562	case 28:
1563	return AMDGPU::SI_SPILL_S224_SAVE;
1564	case 32:
1565	return AMDGPU::SI_SPILL_S256_SAVE;
1566	case 36:
1567	return AMDGPU::SI_SPILL_S288_SAVE;
1568	case 40:
1569	return AMDGPU::SI_SPILL_S320_SAVE;
1570	case 44:
1571	return AMDGPU::SI_SPILL_S352_SAVE;
1572	case 48:
1573	return AMDGPU::SI_SPILL_S384_SAVE;
1574	case 64:
1575	return AMDGPU::SI_SPILL_S512_SAVE;
1576	case 128:
1577	return AMDGPU::SI_SPILL_S1024_SAVE;
1578	default:
1579	llvm_unreachable("unknown register size");
1580	}
1581	}
1582
1583	static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1584	switch (Size) {
1585	case 4:
1586	return AMDGPU::SI_SPILL_V32_SAVE;
1587	case 8:
1588	return AMDGPU::SI_SPILL_V64_SAVE;
1589	case 12:
1590	return AMDGPU::SI_SPILL_V96_SAVE;
1591	case 16:
1592	return AMDGPU::SI_SPILL_V128_SAVE;
1593	case 20:
1594	return AMDGPU::SI_SPILL_V160_SAVE;
1595	case 24:
1596	return AMDGPU::SI_SPILL_V192_SAVE;
1597	case 28:
1598	return AMDGPU::SI_SPILL_V224_SAVE;
1599	case 32:
1600	return AMDGPU::SI_SPILL_V256_SAVE;
1601	case 36:
1602	return AMDGPU::SI_SPILL_V288_SAVE;
1603	case 40:
1604	return AMDGPU::SI_SPILL_V320_SAVE;
1605	case 44:
1606	return AMDGPU::SI_SPILL_V352_SAVE;
1607	case 48:
1608	return AMDGPU::SI_SPILL_V384_SAVE;
1609	case 64:
1610	return AMDGPU::SI_SPILL_V512_SAVE;
1611	case 128:
1612	return AMDGPU::SI_SPILL_V1024_SAVE;
1613	default:
1614	llvm_unreachable("unknown register size");
1615	}
1616	}
1617
1618	static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1619	switch (Size) {
1620	case 4:
1621	return AMDGPU::SI_SPILL_A32_SAVE;
1622	case 8:
1623	return AMDGPU::SI_SPILL_A64_SAVE;
1624	case 12:
1625	return AMDGPU::SI_SPILL_A96_SAVE;
1626	case 16:
1627	return AMDGPU::SI_SPILL_A128_SAVE;
1628	case 20:
1629	return AMDGPU::SI_SPILL_A160_SAVE;
1630	case 24:
1631	return AMDGPU::SI_SPILL_A192_SAVE;
1632	case 28:
1633	return AMDGPU::SI_SPILL_A224_SAVE;
1634	case 32:
1635	return AMDGPU::SI_SPILL_A256_SAVE;
1636	case 36:
1637	return AMDGPU::SI_SPILL_A288_SAVE;
1638	case 40:
1639	return AMDGPU::SI_SPILL_A320_SAVE;
1640	case 44:
1641	return AMDGPU::SI_SPILL_A352_SAVE;
1642	case 48:
1643	return AMDGPU::SI_SPILL_A384_SAVE;
1644	case 64:
1645	return AMDGPU::SI_SPILL_A512_SAVE;
1646	case 128:
1647	return AMDGPU::SI_SPILL_A1024_SAVE;
1648	default:
1649	llvm_unreachable("unknown register size");
1650	}
1651	}
1652
1653	static unsigned getAVSpillSaveOpcode(unsigned Size) {
1654	switch (Size) {
1655	case 4:
1656	return AMDGPU::SI_SPILL_AV32_SAVE;
1657	case 8:
1658	return AMDGPU::SI_SPILL_AV64_SAVE;
1659	case 12:
1660	return AMDGPU::SI_SPILL_AV96_SAVE;
1661	case 16:
1662	return AMDGPU::SI_SPILL_AV128_SAVE;
1663	case 20:
1664	return AMDGPU::SI_SPILL_AV160_SAVE;
1665	case 24:
1666	return AMDGPU::SI_SPILL_AV192_SAVE;
1667	case 28:
1668	return AMDGPU::SI_SPILL_AV224_SAVE;
1669	case 32:
1670	return AMDGPU::SI_SPILL_AV256_SAVE;
1671	case 36:
1672	return AMDGPU::SI_SPILL_AV288_SAVE;
1673	case 40:
1674	return AMDGPU::SI_SPILL_AV320_SAVE;
1675	case 44:
1676	return AMDGPU::SI_SPILL_AV352_SAVE;
1677	case 48:
1678	return AMDGPU::SI_SPILL_AV384_SAVE;
1679	case 64:
1680	return AMDGPU::SI_SPILL_AV512_SAVE;
1681	case 128:
1682	return AMDGPU::SI_SPILL_AV1024_SAVE;
1683	default:
1684	llvm_unreachable("unknown register size");
1685	}
1686	}
1687
1688	static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1689	bool IsVectorSuperClass) {
1690	// Currently, there is only 32-bit WWM register spills needed.
1691	if (Size != 4)
1692	llvm_unreachable("unknown wwm register spill size");
1693
1694	if (IsVectorSuperClass)
1695	return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1696
1697	return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1698	}
1699
1700	static unsigned getVectorRegSpillSaveOpcode(Register Reg,
1701	const TargetRegisterClass *RC,
1702	unsigned Size,
1703	const SIRegisterInfo &TRI,
1704	const SIMachineFunctionInfo &MFI) {
1705	bool IsVectorSuperClass = TRI.isVectorSuperClass(RC);
1706
1707	// Choose the right opcode if spilling a WWM register.
1708	if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1709	return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1710
1711	if (IsVectorSuperClass)
1712	return getAVSpillSaveOpcode(Size);
1713
1714	return TRI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(Size)
1715	: getVGPRSpillSaveOpcode(Size);
1716	}
1717
1718	void SIInstrInfo::storeRegToStackSlot(
1719	MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1720	bool isKill, int FrameIndex, const TargetRegisterClass *RC,
1721	const TargetRegisterInfo *TRI, Register VReg) const {
1722	MachineFunction *MF = MBB.getParent();
1723	SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1724	MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1725	const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1726
1727	MachinePointerInfo PtrInfo
1728	= MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1729	MachineMemOperand *MMO = MF->getMachineMemOperand(
1730	PtrInfo, f: MachineMemOperand::MOStore, s: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1731	base_alignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1732	unsigned SpillSize = TRI->getSpillSize(RC: *RC);
1733
1734	MachineRegisterInfo &MRI = MF->getRegInfo();
1735	if (RI.isSGPRClass(RC)) {
1736	MFI->setHasSpilledSGPRs();
1737	assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1738	assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1739	SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1740
1741	// We are only allowed to create one new instruction when spilling
1742	// registers, so we need to use pseudo instruction for spilling SGPRs.
1743	const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(Size: SpillSize));
1744
1745	// The SGPR spill/restore instructions only work on number sgprs, so we need
1746	// to make sure we are using the correct register class.
1747	if (SrcReg.isVirtual() && SpillSize == 4) {
1748	MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1749	}
1750
1751	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1752	.addReg(RegNo: SrcReg, flags: getKillRegState(B: isKill)) // data
1753	.addFrameIndex(Idx: FrameIndex) // addr
1754	.addMemOperand(MMO)
1755	.addReg(RegNo: MFI->getStackPtrOffsetReg(), flags: RegState::Implicit);
1756
1757	if (RI.spillSGPRToVGPR())
1758	FrameInfo.setStackID(ObjectIdx: FrameIndex, ID: TargetStackID::SGPRSpill);
1759	return;
1760	}
1761
1762	unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1763	Size: SpillSize, TRI: RI, MFI: *MFI);
1764	MFI->setHasSpilledVGPRs();
1765
1766	BuildMI(MBB, MI, DL, get(Opcode))
1767	.addReg(SrcReg, getKillRegState(B: isKill)) // data
1768	.addFrameIndex(FrameIndex) // addr
1769	.addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1770	.addImm(0) // offset
1771	.addMemOperand(MMO);
1772	}
1773
1774	static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1775	switch (Size) {
1776	case 4:
1777	return AMDGPU::SI_SPILL_S32_RESTORE;
1778	case 8:
1779	return AMDGPU::SI_SPILL_S64_RESTORE;
1780	case 12:
1781	return AMDGPU::SI_SPILL_S96_RESTORE;
1782	case 16:
1783	return AMDGPU::SI_SPILL_S128_RESTORE;
1784	case 20:
1785	return AMDGPU::SI_SPILL_S160_RESTORE;
1786	case 24:
1787	return AMDGPU::SI_SPILL_S192_RESTORE;
1788	case 28:
1789	return AMDGPU::SI_SPILL_S224_RESTORE;
1790	case 32:
1791	return AMDGPU::SI_SPILL_S256_RESTORE;
1792	case 36:
1793	return AMDGPU::SI_SPILL_S288_RESTORE;
1794	case 40:
1795	return AMDGPU::SI_SPILL_S320_RESTORE;
1796	case 44:
1797	return AMDGPU::SI_SPILL_S352_RESTORE;
1798	case 48:
1799	return AMDGPU::SI_SPILL_S384_RESTORE;
1800	case 64:
1801	return AMDGPU::SI_SPILL_S512_RESTORE;
1802	case 128:
1803	return AMDGPU::SI_SPILL_S1024_RESTORE;
1804	default:
1805	llvm_unreachable("unknown register size");
1806	}
1807	}
1808
1809	static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1810	switch (Size) {
1811	case 4:
1812	return AMDGPU::SI_SPILL_V32_RESTORE;
1813	case 8:
1814	return AMDGPU::SI_SPILL_V64_RESTORE;
1815	case 12:
1816	return AMDGPU::SI_SPILL_V96_RESTORE;
1817	case 16:
1818	return AMDGPU::SI_SPILL_V128_RESTORE;
1819	case 20:
1820	return AMDGPU::SI_SPILL_V160_RESTORE;
1821	case 24:
1822	return AMDGPU::SI_SPILL_V192_RESTORE;
1823	case 28:
1824	return AMDGPU::SI_SPILL_V224_RESTORE;
1825	case 32:
1826	return AMDGPU::SI_SPILL_V256_RESTORE;
1827	case 36:
1828	return AMDGPU::SI_SPILL_V288_RESTORE;
1829	case 40:
1830	return AMDGPU::SI_SPILL_V320_RESTORE;
1831	case 44:
1832	return AMDGPU::SI_SPILL_V352_RESTORE;
1833	case 48:
1834	return AMDGPU::SI_SPILL_V384_RESTORE;
1835	case 64:
1836	return AMDGPU::SI_SPILL_V512_RESTORE;
1837	case 128:
1838	return AMDGPU::SI_SPILL_V1024_RESTORE;
1839	default:
1840	llvm_unreachable("unknown register size");
1841	}
1842	}
1843
1844	static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1845	switch (Size) {
1846	case 4:
1847	return AMDGPU::SI_SPILL_A32_RESTORE;
1848	case 8:
1849	return AMDGPU::SI_SPILL_A64_RESTORE;
1850	case 12:
1851	return AMDGPU::SI_SPILL_A96_RESTORE;
1852	case 16:
1853	return AMDGPU::SI_SPILL_A128_RESTORE;
1854	case 20:
1855	return AMDGPU::SI_SPILL_A160_RESTORE;
1856	case 24:
1857	return AMDGPU::SI_SPILL_A192_RESTORE;
1858	case 28:
1859	return AMDGPU::SI_SPILL_A224_RESTORE;
1860	case 32:
1861	return AMDGPU::SI_SPILL_A256_RESTORE;
1862	case 36:
1863	return AMDGPU::SI_SPILL_A288_RESTORE;
1864	case 40:
1865	return AMDGPU::SI_SPILL_A320_RESTORE;
1866	case 44:
1867	return AMDGPU::SI_SPILL_A352_RESTORE;
1868	case 48:
1869	return AMDGPU::SI_SPILL_A384_RESTORE;
1870	case 64:
1871	return AMDGPU::SI_SPILL_A512_RESTORE;
1872	case 128:
1873	return AMDGPU::SI_SPILL_A1024_RESTORE;
1874	default:
1875	llvm_unreachable("unknown register size");
1876	}
1877	}
1878
1879	static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1880	switch (Size) {
1881	case 4:
1882	return AMDGPU::SI_SPILL_AV32_RESTORE;
1883	case 8:
1884	return AMDGPU::SI_SPILL_AV64_RESTORE;
1885	case 12:
1886	return AMDGPU::SI_SPILL_AV96_RESTORE;
1887	case 16:
1888	return AMDGPU::SI_SPILL_AV128_RESTORE;
1889	case 20:
1890	return AMDGPU::SI_SPILL_AV160_RESTORE;
1891	case 24:
1892	return AMDGPU::SI_SPILL_AV192_RESTORE;
1893	case 28:
1894	return AMDGPU::SI_SPILL_AV224_RESTORE;
1895	case 32:
1896	return AMDGPU::SI_SPILL_AV256_RESTORE;
1897	case 36:
1898	return AMDGPU::SI_SPILL_AV288_RESTORE;
1899	case 40:
1900	return AMDGPU::SI_SPILL_AV320_RESTORE;
1901	case 44:
1902	return AMDGPU::SI_SPILL_AV352_RESTORE;
1903	case 48:
1904	return AMDGPU::SI_SPILL_AV384_RESTORE;
1905	case 64:
1906	return AMDGPU::SI_SPILL_AV512_RESTORE;
1907	case 128:
1908	return AMDGPU::SI_SPILL_AV1024_RESTORE;
1909	default:
1910	llvm_unreachable("unknown register size");
1911	}
1912	}
1913
1914	static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1915	bool IsVectorSuperClass) {
1916	// Currently, there is only 32-bit WWM register spills needed.
1917	if (Size != 4)
1918	llvm_unreachable("unknown wwm register spill size");
1919
1920	if (IsVectorSuperClass)
1921	return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1922
1923	return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1924	}
1925
1926	static unsigned
1927	getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
1928	unsigned Size, const SIRegisterInfo &TRI,
1929	const SIMachineFunctionInfo &MFI) {
1930	bool IsVectorSuperClass = TRI.isVectorSuperClass(RC);
1931
1932	// Choose the right opcode if restoring a WWM register.
1933	if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1934	return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1935
1936	if (IsVectorSuperClass)
1937	return getAVSpillRestoreOpcode(Size);
1938
1939	return TRI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(Size)
1940	: getVGPRSpillRestoreOpcode(Size);
1941	}
1942
1943	void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1944	MachineBasicBlock::iterator MI,
1945	Register DestReg, int FrameIndex,
1946	const TargetRegisterClass *RC,
1947	const TargetRegisterInfo *TRI,
1948	Register VReg) const {
1949	MachineFunction *MF = MBB.getParent();
1950	SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1951	MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1952	const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1953	unsigned SpillSize = TRI->getSpillSize(RC: *RC);
1954
1955	MachinePointerInfo PtrInfo
1956	= MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1957
1958	MachineMemOperand *MMO = MF->getMachineMemOperand(
1959	PtrInfo, f: MachineMemOperand::MOLoad, s: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1960	base_alignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1961
1962	if (RI.isSGPRClass(RC)) {
1963	MFI->setHasSpilledSGPRs();
1964	assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1965	assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1966	DestReg != AMDGPU::EXEC && "exec should not be spilled");
1967
1968	// FIXME: Maybe this should not include a memoperand because it will be
1969	// lowered to non-memory instructions.
1970	const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(Size: SpillSize));
1971	if (DestReg.isVirtual() && SpillSize == 4) {
1972	MachineRegisterInfo &MRI = MF->getRegInfo();
1973	MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1974	}
1975
1976	if (RI.spillSGPRToVGPR())
1977	FrameInfo.setStackID(ObjectIdx: FrameIndex, ID: TargetStackID::SGPRSpill);
1978	BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
1979	.addFrameIndex(Idx: FrameIndex) // addr
1980	.addMemOperand(MMO)
1981	.addReg(RegNo: MFI->getStackPtrOffsetReg(), flags: RegState::Implicit);
1982
1983	return;
1984	}
1985
1986	unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
1987	Size: SpillSize, TRI: RI, MFI: *MFI);
1988	BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1989	.addFrameIndex(FrameIndex) // vaddr
1990	.addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1991	.addImm(0) // offset
1992	.addMemOperand(MMO);
1993	}
1994
1995	void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1996	MachineBasicBlock::iterator MI) const {
1997	insertNoops(MBB, MI, Quantity: 1);
1998	}
1999
2000	void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
2001	MachineBasicBlock::iterator MI,
2002	unsigned Quantity) const {
2003	DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2004	while (Quantity > 0) {
2005	unsigned Arg = std::min(a: Quantity, b: 8u);
2006	Quantity -= Arg;
2007	BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);
2008	}
2009	}
2010
2011	void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
2012	auto MF = MBB.getParent();
2013	SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
2014
2015	assert(Info->isEntryFunction());
2016
2017	if (MBB.succ_empty()) {
2018	bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
2019	if (HasNoTerminator) {
2020	if (Info->returnsVoid()) {
2021	BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
2022	} else {
2023	BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
2024	}
2025	}
2026	}
2027	}
2028
2029	unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
2030	switch (MI.getOpcode()) {
2031	default:
2032	if (MI.isMetaInstruction())
2033	return 0;
2034	return 1; // FIXME: Do wait states equal cycles?
2035
2036	case AMDGPU::S_NOP:
2037	return MI.getOperand(i: 0).getImm() + 1;
2038	// SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2039	// hazard, even if one exist, won't really be visible. Should we handle it?
2040	}
2041	}
2042
2043	bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2044	const SIRegisterInfo *TRI = ST.getRegisterInfo();
2045	MachineBasicBlock &MBB = *MI.getParent();
2046	DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2047	switch (MI.getOpcode()) {
2048	default: return TargetInstrInfo::expandPostRAPseudo(MI);
2049	case AMDGPU::S_MOV_B64_term:
2050	// This is only a terminator to get the correct spill code placement during
2051	// register allocation.
2052	MI.setDesc(get(AMDGPU::S_MOV_B64));
2053	break;
2054
2055	case AMDGPU::S_MOV_B32_term:
2056	// This is only a terminator to get the correct spill code placement during
2057	// register allocation.
2058	MI.setDesc(get(AMDGPU::S_MOV_B32));
2059	break;
2060
2061	case AMDGPU::S_XOR_B64_term:
2062	// This is only a terminator to get the correct spill code placement during
2063	// register allocation.
2064	MI.setDesc(get(AMDGPU::S_XOR_B64));
2065	break;
2066
2067	case AMDGPU::S_XOR_B32_term:
2068	// This is only a terminator to get the correct spill code placement during
2069	// register allocation.
2070	MI.setDesc(get(AMDGPU::S_XOR_B32));
2071	break;
2072	case AMDGPU::S_OR_B64_term:
2073	// This is only a terminator to get the correct spill code placement during
2074	// register allocation.
2075	MI.setDesc(get(AMDGPU::S_OR_B64));
2076	break;
2077	case AMDGPU::S_OR_B32_term:
2078	// This is only a terminator to get the correct spill code placement during
2079	// register allocation.
2080	MI.setDesc(get(AMDGPU::S_OR_B32));
2081	break;
2082
2083	case AMDGPU::S_ANDN2_B64_term:
2084	// This is only a terminator to get the correct spill code placement during
2085	// register allocation.
2086	MI.setDesc(get(AMDGPU::S_ANDN2_B64));
2087	break;
2088
2089	case AMDGPU::S_ANDN2_B32_term:
2090	// This is only a terminator to get the correct spill code placement during
2091	// register allocation.
2092	MI.setDesc(get(AMDGPU::S_ANDN2_B32));
2093	break;
2094
2095	case AMDGPU::S_AND_B64_term:
2096	// This is only a terminator to get the correct spill code placement during
2097	// register allocation.
2098	MI.setDesc(get(AMDGPU::S_AND_B64));
2099	break;
2100
2101	case AMDGPU::S_AND_B32_term:
2102	// This is only a terminator to get the correct spill code placement during
2103	// register allocation.
2104	MI.setDesc(get(AMDGPU::S_AND_B32));
2105	break;
2106
2107	case AMDGPU::S_AND_SAVEEXEC_B64_term:
2108	// This is only a terminator to get the correct spill code placement during
2109	// register allocation.
2110	MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B64));
2111	break;
2112
2113	case AMDGPU::S_AND_SAVEEXEC_B32_term:
2114	// This is only a terminator to get the correct spill code placement during
2115	// register allocation.
2116	MI.setDesc(get(AMDGPU::S_AND_SAVEEXEC_B32));
2117	break;
2118
2119	case AMDGPU::SI_SPILL_S32_TO_VGPR:
2120	MI.setDesc(get(AMDGPU::V_WRITELANE_B32));
2121	break;
2122
2123	case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2124	MI.setDesc(get(AMDGPU::V_READLANE_B32));
2125	break;
2126
2127	case AMDGPU::V_MOV_B64_PSEUDO: {
2128	Register Dst = MI.getOperand(i: 0).getReg();
2129	Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2130	Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2131
2132	const MachineOperand &SrcOp = MI.getOperand(i: 1);
2133	// FIXME: Will this work for 64-bit floating point immediates?
2134	assert(!SrcOp.isFPImm());
2135	if (ST.hasMovB64()) {
2136	MI.setDesc(get(AMDGPU::V_MOV_B64_e32));
2137	if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2138	isUInt<32>(x: SrcOp.getImm()))
2139	break;
2140	}
2141	if (SrcOp.isImm()) {
2142	APInt Imm(64, SrcOp.getImm());
2143	APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2144	APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2145	if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo)) {
2146	BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
2147	.addImm(SISrcMods::OP_SEL_1)
2148	.addImm(Lo.getSExtValue())
2149	.addImm(SISrcMods::OP_SEL_1)
2150	.addImm(Lo.getSExtValue())
2151	.addImm(0) // op_sel_lo
2152	.addImm(0) // op_sel_hi
2153	.addImm(0) // neg_lo
2154	.addImm(0) // neg_hi
2155	.addImm(0); // clamp
2156	} else {
2157	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2158	.addImm(Lo.getSExtValue())
2159	.addReg(Dst, RegState::Implicit | RegState::Define);
2160	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2161	.addImm(Hi.getSExtValue())
2162	.addReg(Dst, RegState::Implicit | RegState::Define);
2163	}
2164	} else {
2165	assert(SrcOp.isReg());
2166	if (ST.hasPkMovB32() &&
2167	!RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) {
2168	BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
2169	.addImm(SISrcMods::OP_SEL_1) // src0_mod
2170	.addReg(SrcOp.getReg())
2171	.addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2172	.addReg(SrcOp.getReg())
2173	.addImm(0) // op_sel_lo
2174	.addImm(0) // op_sel_hi
2175	.addImm(0) // neg_lo
2176	.addImm(0) // neg_hi
2177	.addImm(0); // clamp
2178	} else {
2179	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
2180	.addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
2181	.addReg(Dst, RegState::Implicit | RegState::Define);
2182	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
2183	.addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
2184	.addReg(Dst, RegState::Implicit | RegState::Define);
2185	}
2186	}
2187	MI.eraseFromParent();
2188	break;
2189	}
2190	case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2191	expandMovDPP64(MI);
2192	break;
2193	}
2194	case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2195	const MachineOperand &SrcOp = MI.getOperand(i: 1);
2196	assert(!SrcOp.isFPImm());
2197	APInt Imm(64, SrcOp.getImm());
2198	if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2199	MI.setDesc(get(AMDGPU::S_MOV_B64));
2200	break;
2201	}
2202
2203	Register Dst = MI.getOperand(i: 0).getReg();
2204	Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
2205	Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2206
2207	APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2208	APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2209	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)
2210	.addImm(Lo.getSExtValue())
2211	.addReg(Dst, RegState::Implicit | RegState::Define);
2212	BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)
2213	.addImm(Hi.getSExtValue())
2214	.addReg(Dst, RegState::Implicit | RegState::Define);
2215	MI.eraseFromParent();
2216	break;
2217	}
2218	case AMDGPU::V_SET_INACTIVE_B32: {
2219	unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
2220	unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2221	// FIXME: We may possibly optimize the COPY once we find ways to make LLVM
2222	// optimizations (mainly Register Coalescer) aware of WWM register liveness.
2223	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
2224	.add(MI.getOperand(1));
2225	auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
2226	FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
2227	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
2228	.add(MI.getOperand(2));
2229	BuildMI(MBB, MI, DL, get(NotOpc), Exec)
2230	.addReg(Exec);
2231	MI.eraseFromParent();
2232	break;
2233	}
2234	case AMDGPU::V_SET_INACTIVE_B64: {
2235	unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
2236	unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2237	MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
2238	MI.getOperand(0).getReg())
2239	.add(MI.getOperand(1));
2240	expandPostRAPseudo(MI&: *Copy);
2241	auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
2242	FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
2243	Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
2244	MI.getOperand(0).getReg())
2245	.add(MI.getOperand(2));
2246	expandPostRAPseudo(MI&: *Copy);
2247	BuildMI(MBB, MI, DL, get(NotOpc), Exec)
2248	.addReg(Exec);
2249	MI.eraseFromParent();
2250	break;
2251	}
2252	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2253	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2254	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2255	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2256	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2257	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2258	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2259	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2260	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2261	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2262	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2263	case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2264	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2265	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2266	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2267	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2268	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2269	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2270	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2271	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2272	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2273	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2274	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2275	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2276	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2277	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2278	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2279	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2280	case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2281	const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2282
2283	unsigned Opc;
2284	if (RI.hasVGPRs(EltRC)) {
2285	Opc = AMDGPU::V_MOVRELD_B32_e32;
2286	} else {
2287	Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2288	: AMDGPU::S_MOVRELD_B32;
2289	}
2290
2291	const MCInstrDesc &OpDesc = get(Opc);
2292	Register VecReg = MI.getOperand(i: 0).getReg();
2293	bool IsUndef = MI.getOperand(i: 1).isUndef();
2294	unsigned SubReg = MI.getOperand(i: 3).getImm();
2295	assert(VecReg == MI.getOperand(1).getReg());
2296
2297	MachineInstrBuilder MIB =
2298	BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2299	.addReg(RegNo: RI.getSubReg(VecReg, SubReg), flags: RegState::Undef)
2300	.add(MI.getOperand(i: 2))
2301	.addReg(VecReg, RegState::ImplicitDefine)
2302	.addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2303
2304	const int ImpDefIdx =
2305	OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2306	const int ImpUseIdx = ImpDefIdx + 1;
2307	MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2308	MI.eraseFromParent();
2309	break;
2310	}
2311	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2312	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2313	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2314	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2315	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2316	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2317	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2318	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2319	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2320	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2321	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2322	case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2323	assert(ST.useVGPRIndexMode());
2324	Register VecReg = MI.getOperand(i: 0).getReg();
2325	bool IsUndef = MI.getOperand(i: 1).isUndef();
2326	Register Idx = MI.getOperand(i: 3).getReg();
2327	Register SubReg = MI.getOperand(i: 4).getImm();
2328
2329	MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2330	.addReg(Idx)
2331	.addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);
2332	SetOn->getOperand(i: 3).setIsUndef();
2333
2334	const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write);
2335	MachineInstrBuilder MIB =
2336	BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2337	.addReg(RegNo: RI.getSubReg(VecReg, SubReg), flags: RegState::Undef)
2338	.add(MI.getOperand(i: 2))
2339	.addReg(VecReg, RegState::ImplicitDefine)
2340	.addReg(VecReg,
2341	RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2342
2343	const int ImpDefIdx =
2344	OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2345	const int ImpUseIdx = ImpDefIdx + 1;
2346	MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2347
2348	MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2349
2350	finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2351
2352	MI.eraseFromParent();
2353	break;
2354	}
2355	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2356	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2357	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2358	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2359	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2360	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2361	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2362	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2363	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2364	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2365	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2366	case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2367	assert(ST.useVGPRIndexMode());
2368	Register Dst = MI.getOperand(i: 0).getReg();
2369	Register VecReg = MI.getOperand(i: 1).getReg();
2370	bool IsUndef = MI.getOperand(i: 1).isUndef();
2371	Register Idx = MI.getOperand(i: 2).getReg();
2372	Register SubReg = MI.getOperand(i: 3).getImm();
2373
2374	MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2375	.addReg(Idx)
2376	.addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2377	SetOn->getOperand(i: 3).setIsUndef();
2378
2379	BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read))
2380	.addDef(Dst)
2381	.addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2382	.addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2383
2384	MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2385
2386	finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2387
2388	MI.eraseFromParent();
2389	break;
2390	}
2391	case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2392	MachineFunction &MF = *MBB.getParent();
2393	Register Reg = MI.getOperand(i: 0).getReg();
2394	Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
2395	Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
2396	MachineOperand OpLo = MI.getOperand(i: 1);
2397	MachineOperand OpHi = MI.getOperand(i: 2);
2398
2399	// Create a bundle so these instructions won't be re-ordered by the
2400	// post-RA scheduler.
2401	MIBundleBuilder Bundler(MBB, MI);
2402	Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
2403
2404	// What we want here is an offset from the value returned by s_getpc (which
2405	// is the address of the s_add_u32 instruction) to the global variable, but
2406	// since the encoding of $symbol starts 4 bytes after the start of the
2407	// s_add_u32 instruction, we end up with an offset that is 4 bytes too
2408	// small. This requires us to add 4 to the global variable offset in order
2409	// to compute the correct address. Similarly for the s_addc_u32 instruction,
2410	// the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2411	// instruction.
2412
2413	int64_t Adjust = 0;
2414	if (ST.hasGetPCZeroExtension()) {
2415	// Fix up hardware that does not sign-extend the 48-bit PC value by
2416	// inserting: s_sext_i32_i16 reghi, reghi
2417	Bundler.append(
2418	BuildMI(MF, DL, get(AMDGPU::S_SEXT_I32_I16), RegHi).addReg(RegHi));
2419	Adjust += 4;
2420	}
2421
2422	if (OpLo.isGlobal())
2423	OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2424	Bundler.append(
2425	BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo).addReg(RegLo).add(OpLo));
2426
2427	if (OpHi.isGlobal())
2428	OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2429	Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
2430	.addReg(RegHi)
2431	.add(OpHi));
2432
2433	finalizeBundle(MBB, FirstMI: Bundler.begin());
2434
2435	MI.eraseFromParent();
2436	break;
2437	}
2438	case AMDGPU::ENTER_STRICT_WWM: {
2439	// This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2440	// Whole Wave Mode is entered.
2441	MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
2442	: AMDGPU::S_OR_SAVEEXEC_B64));
2443	break;
2444	}
2445	case AMDGPU::ENTER_STRICT_WQM: {
2446	// This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2447	// STRICT_WQM is entered.
2448	const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2449	const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
2450	const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
2451	BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(i: 0).getReg()).addReg(Exec);
2452	BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec);
2453
2454	MI.eraseFromParent();
2455	break;
2456	}
2457	case AMDGPU::EXIT_STRICT_WWM:
2458	case AMDGPU::EXIT_STRICT_WQM: {
2459	// This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2460	// WWM/STICT_WQM is exited.
2461	MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
2462	break;
2463	}
2464	case AMDGPU::ENTER_PSEUDO_WM:
2465	case AMDGPU::EXIT_PSEUDO_WM: {
2466	// These do nothing.
2467	MI.eraseFromParent();
2468	break;
2469	}
2470	case AMDGPU::SI_RETURN: {
2471	const MachineFunction *MF = MBB.getParent();
2472	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2473	const SIRegisterInfo *TRI = ST.getRegisterInfo();
2474	// Hiding the return address use with SI_RETURN may lead to extra kills in
2475	// the function and missing live-ins. We are fine in practice because callee
2476	// saved register handling ensures the register value is restored before
2477	// RET, but we need the undef flag here to appease the MachineVerifier
2478	// liveness checks.
2479	MachineInstrBuilder MIB =
2480	BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return))
2481	.addReg(TRI->getReturnAddressReg(*MF), RegState::Undef);
2482
2483	MIB.copyImplicitOps(OtherMI: MI);
2484	MI.eraseFromParent();
2485	break;
2486	}
2487
2488	case AMDGPU::S_MUL_U64_U32_PSEUDO:
2489	case AMDGPU::S_MUL_I64_I32_PSEUDO:
2490	MI.setDesc(get(AMDGPU::S_MUL_U64));
2491	break;
2492
2493	case AMDGPU::S_GETPC_B64_pseudo:
2494	MI.setDesc(get(AMDGPU::S_GETPC_B64));
2495	if (ST.hasGetPCZeroExtension()) {
2496	Register Dst = MI.getOperand(i: 0).getReg();
2497	Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
2498	// Fix up hardware that does not sign-extend the 48-bit PC value by
2499	// inserting: s_sext_i32_i16 dsthi, dsthi
2500	BuildMI(MBB, std::next(MI.getIterator()), DL, get(AMDGPU::S_SEXT_I32_I16),
2501	DstHi)
2502	.addReg(DstHi);
2503	}
2504	break;
2505	}
2506	return true;
2507	}
2508
2509	void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2510	MachineBasicBlock::iterator I, Register DestReg,
2511	unsigned SubIdx, const MachineInstr &Orig,
2512	const TargetRegisterInfo &RI) const {
2513
2514	// Try shrinking the instruction to remat only the part needed for current
2515	// context.
2516	// TODO: Handle more cases.
2517	unsigned Opcode = Orig.getOpcode();
2518	switch (Opcode) {
2519	case AMDGPU::S_LOAD_DWORDX16_IMM:
2520	case AMDGPU::S_LOAD_DWORDX8_IMM: {
2521	if (SubIdx != 0)
2522	break;
2523
2524	if (I == MBB.end())
2525	break;
2526
2527	if (I->isBundled())
2528	break;
2529
2530	// Look for a single use of the register that is also a subreg.
2531	Register RegToFind = Orig.getOperand(i: 0).getReg();
2532	MachineOperand *UseMO = nullptr;
2533	for (auto &CandMO : I->operands()) {
2534	if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())
2535	continue;
2536	if (UseMO) {
2537	UseMO = nullptr;
2538	break;
2539	}
2540	UseMO = &CandMO;
2541	}
2542	if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2543	break;
2544
2545	unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2546	unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2547
2548	MachineFunction *MF = MBB.getParent();
2549	MachineRegisterInfo &MRI = MF->getRegInfo();
2550	assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2551
2552	unsigned NewOpcode = -1;
2553	if (SubregSize == 256)
2554	NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2555	else if (SubregSize == 128)
2556	NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2557	else
2558	break;
2559
2560	const MCInstrDesc &TID = get(NewOpcode);
2561	const TargetRegisterClass *NewRC =
2562	RI.getAllocatableClass(RC: getRegClass(TID, OpNum: 0, TRI: &RI, MF: *MF));
2563	MRI.setRegClass(Reg: DestReg, RC: NewRC);
2564
2565	UseMO->setReg(DestReg);
2566	UseMO->setSubReg(AMDGPU::NoSubRegister);
2567
2568	// Use a smaller load with the desired size, possibly with updated offset.
2569	MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2570	MI->setDesc(TID);
2571	MI->getOperand(i: 0).setReg(DestReg);
2572	MI->getOperand(0).setSubReg(AMDGPU::NoSubRegister);
2573	if (Offset) {
2574	MachineOperand *OffsetMO = getNamedOperand(*MI, AMDGPU::OpName::offset);
2575	int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2576	OffsetMO->setImm(FinalOffset);
2577	}
2578	SmallVector<MachineMemOperand *> NewMMOs;
2579	for (const MachineMemOperand *MemOp : Orig.memoperands())
2580	NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2581	Size: SubregSize / 8));
2582	MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2583
2584	MBB.insert(I, MI);
2585	return;
2586	}
2587
2588	default:
2589	break;
2590	}
2591
2592	TargetInstrInfo::reMaterialize(MBB, I, DestReg, SubIdx, Orig, RI);
2593	}
2594
2595	std::pair<MachineInstr*, MachineInstr*>
2596	SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2597	assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2598
2599	if (ST.hasMovB64() &&
2600	AMDGPU::isLegalDPALU_DPPControl(
2601	getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
2602	MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
2603	return std::pair(&MI, nullptr);
2604	}
2605
2606	MachineBasicBlock &MBB = *MI.getParent();
2607	DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2608	MachineFunction *MF = MBB.getParent();
2609	MachineRegisterInfo &MRI = MF->getRegInfo();
2610	Register Dst = MI.getOperand(i: 0).getReg();
2611	unsigned Part = 0;
2612	MachineInstr *Split[2];
2613
2614	for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2615	auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
2616	if (Dst.isPhysical()) {
2617	MovDPP.addDef(RI.getSubReg(Dst, Sub));
2618	} else {
2619	assert(MRI.isSSA());
2620	auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2621	MovDPP.addDef(Tmp);
2622	}
2623
2624	for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2625	const MachineOperand &SrcOp = MI.getOperand(I);
2626	assert(!SrcOp.isFPImm());
2627	if (SrcOp.isImm()) {
2628	APInt Imm(64, SrcOp.getImm());
2629	Imm.ashrInPlace(Part * 32);
2630	MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
2631	} else {
2632	assert(SrcOp.isReg());
2633	Register Src = SrcOp.getReg();
2634	if (Src.isPhysical())
2635	MovDPP.addReg(RI.getSubReg(Src, Sub));
2636	else
2637	MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);
2638	}
2639	}
2640
2641	for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3))
2642	MovDPP.addImm(MO.getImm());
2643
2644	Split[Part] = MovDPP;
2645	++Part;
2646	}
2647
2648	if (Dst.isVirtual())
2649	BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
2650	.addReg(Split[0]->getOperand(0).getReg())
2651	.addImm(AMDGPU::sub0)
2652	.addReg(Split[1]->getOperand(0).getReg())
2653	.addImm(AMDGPU::sub1);
2654
2655	MI.eraseFromParent();
2656	return std::pair(Split[0], Split[1]);
2657	}
2658
2659	std::optional<DestSourcePair>
2660	SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2661	if (MI.getOpcode() == AMDGPU::WWM_COPY)
2662	return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2663
2664	return std::nullopt;
2665	}
2666
2667	bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
2668	MachineOperand &Src0,
2669	unsigned Src0OpName,
2670	MachineOperand &Src1,
2671	unsigned Src1OpName) const {
2672	MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2673	if (!Src0Mods)
2674	return false;
2675
2676	MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2677	assert(Src1Mods &&
2678	"All commutable instructions have both src0 and src1 modifiers");
2679
2680	int Src0ModsVal = Src0Mods->getImm();
2681	int Src1ModsVal = Src1Mods->getImm();
2682
2683	Src1Mods->setImm(Src0ModsVal);
2684	Src0Mods->setImm(Src1ModsVal);
2685	return true;
2686	}
2687
2688	static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2689	MachineOperand &RegOp,
2690	MachineOperand &NonRegOp) {
2691	Register Reg = RegOp.getReg();
2692	unsigned SubReg = RegOp.getSubReg();
2693	bool IsKill = RegOp.isKill();
2694	bool IsDead = RegOp.isDead();
2695	bool IsUndef = RegOp.isUndef();
2696	bool IsDebug = RegOp.isDebug();
2697
2698	if (NonRegOp.isImm())
2699	RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2700	else if (NonRegOp.isFI())
2701	RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2702	else if (NonRegOp.isGlobal()) {
2703	RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2704	TargetFlags: NonRegOp.getTargetFlags());
2705	} else
2706	return nullptr;
2707
2708	// Make sure we don't reinterpret a subreg index in the target flags.
2709	RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2710
2711	NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
2712	NonRegOp.setSubReg(SubReg);
2713
2714	return &MI;
2715	}
2716
2717	MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
2718	unsigned Src0Idx,
2719	unsigned Src1Idx) const {
2720	assert(!NewMI && "this should never be used");
2721
2722	unsigned Opc = MI.getOpcode();
2723	int CommutedOpcode = commuteOpcode(Opcode: Opc);
2724	if (CommutedOpcode == -1)
2725	return nullptr;
2726
2727	if (Src0Idx > Src1Idx)
2728	std::swap(a&: Src0Idx, b&: Src1Idx);
2729
2730	assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2731	static_cast<int>(Src0Idx) &&
2732	AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2733	static_cast<int>(Src1Idx) &&
2734	"inconsistency with findCommutedOpIndices");
2735
2736	MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
2737	MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
2738
2739	MachineInstr *CommutedMI = nullptr;
2740	if (Src0.isReg() && Src1.isReg()) {
2741	if (isOperandLegal(MI, OpIdx: Src1Idx, MO: &Src0)) {
2742	// Be sure to copy the source modifiers to the right place.
2743	CommutedMI
2744	= TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
2745	}
2746
2747	} else if (Src0.isReg() && !Src1.isReg()) {
2748	// src0 should always be able to support any operand type, so no need to
2749	// check operand legality.
2750	CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
2751	} else if (!Src0.isReg() && Src1.isReg()) {
2752	if (isOperandLegal(MI, OpIdx: Src1Idx, MO: &Src0))
2753	CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
2754	} else {
2755	// FIXME: Found two non registers to commute. This does happen.
2756	return nullptr;
2757	}
2758
2759	if (CommutedMI) {
2760	swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
2761	Src1, AMDGPU::OpName::src1_modifiers);
2762
2763	CommutedMI->setDesc(get(CommutedOpcode));
2764	}
2765
2766	return CommutedMI;
2767	}
2768
2769	// This needs to be implemented because the source modifiers may be inserted
2770	// between the true commutable operands, and the base
2771	// TargetInstrInfo::commuteInstruction uses it.
2772	bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2773	unsigned &SrcOpIdx0,
2774	unsigned &SrcOpIdx1) const {
2775	return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2776	}
2777
2778	bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
2779	unsigned &SrcOpIdx0,
2780	unsigned &SrcOpIdx1) const {
2781	if (!Desc.isCommutable())
2782	return false;
2783
2784	unsigned Opc = Desc.getOpcode();
2785	int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2786	if (Src0Idx == -1)
2787	return false;
2788
2789	int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
2790	if (Src1Idx == -1)
2791	return false;
2792
2793	return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
2794	}
2795
2796	bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2797	int64_t BrOffset) const {
2798	// BranchRelaxation should never have to check s_setpc_b64 because its dest
2799	// block is unanalyzable.
2800	assert(BranchOp != AMDGPU::S_SETPC_B64);
2801
2802	// Convert to dwords.
2803	BrOffset /= 4;
2804
2805	// The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
2806	// from the next instruction.
2807	BrOffset -= 1;
2808
2809	return isIntN(N: BranchOffsetBits, x: BrOffset);
2810	}
2811
2812	MachineBasicBlock *
2813	SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
2814	return MI.getOperand(i: 0).getMBB();
2815	}
2816
2817	bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
2818	for (const MachineInstr &MI : MBB->terminators()) {
2819	if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO ||
2820	MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
2821	MI.getOpcode() == AMDGPU::SI_LOOP)
2822	return true;
2823	}
2824	return false;
2825	}
2826
2827	void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
2828	MachineBasicBlock &DestBB,
2829	MachineBasicBlock &RestoreBB,
2830	const DebugLoc &DL, int64_t BrOffset,
2831	RegScavenger *RS) const {
2832	assert(RS && "RegScavenger required for long branching");
2833	assert(MBB.empty() &&
2834	"new block should be inserted for expanding unconditional branch");
2835	assert(MBB.pred_size() == 1);
2836	assert(RestoreBB.empty() &&
2837	"restore block should be inserted for restoring clobbered registers");
2838
2839	MachineFunction *MF = MBB.getParent();
2840	MachineRegisterInfo &MRI = MF->getRegInfo();
2841	const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
2842
2843	// FIXME: Virtual register workaround for RegScavenger not working with empty
2844	// blocks.
2845	Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
2846
2847	auto I = MBB.end();
2848
2849	// We need to compute the offset relative to the instruction immediately after
2850	// s_getpc_b64. Insert pc arithmetic code before last terminator.
2851	MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
2852
2853	auto &MCCtx = MF->getContext();
2854	MCSymbol *PostGetPCLabel =
2855	MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
2856	GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
2857
2858	MCSymbol *OffsetLo =
2859	MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
2860	MCSymbol *OffsetHi =
2861	MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
2862	BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
2863	.addReg(PCReg, RegState::Define, AMDGPU::sub0)
2864	.addReg(PCReg, 0, AMDGPU::sub0)
2865	.addSym(OffsetLo, MO_FAR_BRANCH_OFFSET);
2866	BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
2867	.addReg(PCReg, RegState::Define, AMDGPU::sub1)
2868	.addReg(PCReg, 0, AMDGPU::sub1)
2869	.addSym(OffsetHi, MO_FAR_BRANCH_OFFSET);
2870
2871	// Insert the indirect branch after the other terminator.
2872	BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
2873	.addReg(PCReg);
2874
2875	// If a spill is needed for the pc register pair, we need to insert a spill
2876	// restore block right before the destination block, and insert a short branch
2877	// into the old destination block's fallthrough predecessor.
2878	// e.g.:
2879	//
2880	// s_cbranch_scc0 skip_long_branch:
2881	//
2882	// long_branch_bb:
2883	// spill s[8:9]
2884	// s_getpc_b64 s[8:9]
2885	// s_add_u32 s8, s8, restore_bb
2886	// s_addc_u32 s9, s9, 0
2887	// s_setpc_b64 s[8:9]
2888	//
2889	// skip_long_branch:
2890	// foo;
2891	//
2892	// .....
2893	//
2894	// dest_bb_fallthrough_predecessor:
2895	// bar;
2896	// s_branch dest_bb
2897	//
2898	// restore_bb:
2899	// restore s[8:9]
2900	// fallthrough dest_bb
2901	///
2902	// dest_bb:
2903	// buzz;
2904
2905	Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
2906	Register Scav;
2907
2908	// If we've previously reserved a register for long branches
2909	// avoid running the scavenger and just use those registers
2910	if (LongBranchReservedReg) {
2911	RS->enterBasicBlock(MBB);
2912	Scav = LongBranchReservedReg;
2913	} else {
2914	RS->enterBasicBlockEnd(MBB);
2915	Scav = RS->scavengeRegisterBackwards(
2916	AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC),
2917	/* RestoreAfter */ false, 0, /* AllowSpill */ false);
2918	}
2919	if (Scav) {
2920	RS->setRegUsed(Reg: Scav);
2921	MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
2922	MRI.clearVirtRegs();
2923	} else {
2924	// As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
2925	// SGPR spill.
2926	const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2927	const SIRegisterInfo *TRI = ST.getRegisterInfo();
2928	TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);
2929	MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);
2930	MRI.clearVirtRegs();
2931	}
2932
2933	MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
2934	// Now, the distance could be defined.
2935	auto *Offset = MCBinaryExpr::createSub(
2936	LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
2937	RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
2938	// Add offset assignments.
2939	auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
2940	OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
2941	auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
2942	OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
2943	}
2944
2945	unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
2946	switch (Cond) {
2947	case SIInstrInfo::SCC_TRUE:
2948	return AMDGPU::S_CBRANCH_SCC1;
2949	case SIInstrInfo::SCC_FALSE:
2950	return AMDGPU::S_CBRANCH_SCC0;
2951	case SIInstrInfo::VCCNZ:
2952	return AMDGPU::S_CBRANCH_VCCNZ;
2953	case SIInstrInfo::VCCZ:
2954	return AMDGPU::S_CBRANCH_VCCZ;
2955	case SIInstrInfo::EXECNZ:
2956	return AMDGPU::S_CBRANCH_EXECNZ;
2957	case SIInstrInfo::EXECZ:
2958	return AMDGPU::S_CBRANCH_EXECZ;
2959	default:
2960	llvm_unreachable("invalid branch predicate");
2961	}
2962	}
2963
2964	SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
2965	switch (Opcode) {
2966	case AMDGPU::S_CBRANCH_SCC0:
2967	return SCC_FALSE;
2968	case AMDGPU::S_CBRANCH_SCC1:
2969	return SCC_TRUE;
2970	case AMDGPU::S_CBRANCH_VCCNZ:
2971	return VCCNZ;
2972	case AMDGPU::S_CBRANCH_VCCZ:
2973	return VCCZ;
2974	case AMDGPU::S_CBRANCH_EXECNZ:
2975	return EXECNZ;
2976	case AMDGPU::S_CBRANCH_EXECZ:
2977	return EXECZ;
2978	default:
2979	return INVALID_BR;
2980	}
2981	}
2982
2983	bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
2984	MachineBasicBlock::iterator I,
2985	MachineBasicBlock *&TBB,
2986	MachineBasicBlock *&FBB,
2987	SmallVectorImpl<MachineOperand> &Cond,
2988	bool AllowModify) const {
2989	if (I->getOpcode() == AMDGPU::S_BRANCH) {
2990	// Unconditional Branch
2991	TBB = I->getOperand(i: 0).getMBB();
2992	return false;
2993	}
2994
2995	MachineBasicBlock *CondBB = nullptr;
2996
2997	if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
2998	CondBB = I->getOperand(i: 1).getMBB();
2999	Cond.push_back(Elt: I->getOperand(i: 0));
3000	} else {
3001	BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3002	if (Pred == INVALID_BR)
3003	return true;
3004
3005	CondBB = I->getOperand(i: 0).getMBB();
3006	Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3007	Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3008	}
3009	++I;
3010
3011	if (I == MBB.end()) {
3012	// Conditional branch followed by fall-through.
3013	TBB = CondBB;
3014	return false;
3015	}
3016
3017	if (I->getOpcode() == AMDGPU::S_BRANCH) {
3018	TBB = CondBB;
3019	FBB = I->getOperand(i: 0).getMBB();
3020	return false;
3021	}
3022
3023	return true;
3024	}
3025
3026	bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3027	MachineBasicBlock *&FBB,
3028	SmallVectorImpl<MachineOperand> &Cond,
3029	bool AllowModify) const {
3030	MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3031	auto E = MBB.end();
3032	if (I == E)
3033	return false;
3034
3035	// Skip over the instructions that are artificially terminators for special
3036	// exec management.
3037	while (I != E && !I->isBranch() && !I->isReturn()) {
3038	switch (I->getOpcode()) {
3039	case AMDGPU::S_MOV_B64_term:
3040	case AMDGPU::S_XOR_B64_term:
3041	case AMDGPU::S_OR_B64_term:
3042	case AMDGPU::S_ANDN2_B64_term:
3043	case AMDGPU::S_AND_B64_term:
3044	case AMDGPU::S_AND_SAVEEXEC_B64_term:
3045	case AMDGPU::S_MOV_B32_term:
3046	case AMDGPU::S_XOR_B32_term:
3047	case AMDGPU::S_OR_B32_term:
3048	case AMDGPU::S_ANDN2_B32_term:
3049	case AMDGPU::S_AND_B32_term:
3050	case AMDGPU::S_AND_SAVEEXEC_B32_term:
3051	break;
3052	case AMDGPU::SI_IF:
3053	case AMDGPU::SI_ELSE:
3054	case AMDGPU::SI_KILL_I1_TERMINATOR:
3055	case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3056	// FIXME: It's messy that these need to be considered here at all.
3057	return true;
3058	default:
3059	llvm_unreachable("unexpected non-branch terminator inst");
3060	}
3061
3062	++I;
3063	}
3064
3065	if (I == E)
3066	return false;
3067
3068	return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3069	}
3070
3071	unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3072	int *BytesRemoved) const {
3073	unsigned Count = 0;
3074	unsigned RemovedSize = 0;
3075	for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3076	// Skip over artificial terminators when removing instructions.
3077	if (MI.isBranch() || MI.isReturn()) {
3078	RemovedSize += getInstSizeInBytes(MI);
3079	MI.eraseFromParent();
3080	++Count;
3081	}
3082	}
3083
3084	if (BytesRemoved)
3085	*BytesRemoved = RemovedSize;
3086
3087	return Count;
3088	}
3089
3090	// Copy the flags onto the implicit condition register operand.
3091	static void preserveCondRegFlags(MachineOperand &CondReg,
3092	const MachineOperand &OrigCond) {
3093	CondReg.setIsUndef(OrigCond.isUndef());
3094	CondReg.setIsKill(OrigCond.isKill());
3095	}
3096
3097	unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3098	MachineBasicBlock *TBB,
3099	MachineBasicBlock *FBB,
3100	ArrayRef<MachineOperand> Cond,
3101	const DebugLoc &DL,
3102	int *BytesAdded) const {
3103	if (!FBB && Cond.empty()) {
3104	BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
3105	.addMBB(TBB);
3106	if (BytesAdded)
3107	*BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3108	return 1;
3109	}
3110
3111	if(Cond.size() == 1 && Cond[0].isReg()) {
3112	BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
3113	.add(Cond[0])
3114	.addMBB(TBB);
3115	return 1;
3116	}
3117
3118	assert(TBB && Cond[0].isImm());
3119
3120	unsigned Opcode
3121	= getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3122
3123	if (!FBB) {
3124	MachineInstr *CondBr =
3125	BuildMI(&MBB, DL, get(Opcode))
3126	.addMBB(TBB);
3127
3128	// Copy the flags onto the implicit condition register operand.
3129	preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3130	fixImplicitOperands(MI&: *CondBr);
3131
3132	if (BytesAdded)
3133	*BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3134	return 1;
3135	}
3136
3137	assert(TBB && FBB);
3138
3139	MachineInstr *CondBr =
3140	BuildMI(&MBB, DL, get(Opcode))
3141	.addMBB(TBB);
3142	fixImplicitOperands(MI&: *CondBr);
3143	BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
3144	.addMBB(FBB);
3145
3146	MachineOperand &CondReg = CondBr->getOperand(i: 1);
3147	CondReg.setIsUndef(Cond[1].isUndef());
3148	CondReg.setIsKill(Cond[1].isKill());
3149
3150	if (BytesAdded)
3151	*BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3152
3153	return 2;
3154	}
3155
3156	bool SIInstrInfo::reverseBranchCondition(
3157	SmallVectorImpl<MachineOperand> &Cond) const {
3158	if (Cond.size() != 2) {
3159	return true;
3160	}
3161
3162	if (Cond[0].isImm()) {
3163	Cond[0].setImm(-Cond[0].getImm());
3164	return false;
3165	}
3166
3167	return true;
3168	}
3169
3170	bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3171	ArrayRef<MachineOperand> Cond,
3172	Register DstReg, Register TrueReg,
3173	Register FalseReg, int &CondCycles,
3174	int &TrueCycles, int &FalseCycles) const {
3175	switch (Cond[0].getImm()) {
3176	case VCCNZ:
3177	case VCCZ: {
3178	const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3179	const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3180	if (MRI.getRegClass(Reg: FalseReg) != RC)
3181	return false;
3182
3183	int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
3184	CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3185
3186	// Limit to equal cost for branch vs. N v_cndmask_b32s.
3187	return RI.hasVGPRs(RC) && NumInsts <= 6;
3188	}
3189	case SCC_TRUE:
3190	case SCC_FALSE: {
3191	// FIXME: We could insert for VGPRs if we could replace the original compare
3192	// with a vector one.
3193	const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3194	const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3195	if (MRI.getRegClass(Reg: FalseReg) != RC)
3196	return false;
3197
3198	int NumInsts = AMDGPU::getRegBitWidth(*RC) / 32;
3199
3200	// Multiples of 8 can do s_cselect_b64
3201	if (NumInsts % 2 == 0)
3202	NumInsts /= 2;
3203
3204	CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3205	return RI.isSGPRClass(RC);
3206	}
3207	default:
3208	return false;
3209	}
3210	}
3211
3212	void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3213	MachineBasicBlock::iterator I, const DebugLoc &DL,
3214	Register DstReg, ArrayRef<MachineOperand> Cond,
3215	Register TrueReg, Register FalseReg) const {
3216	BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3217	if (Pred == VCCZ || Pred == SCC_FALSE) {
3218	Pred = static_cast<BranchPredicate>(-Pred);
3219	std::swap(a&: TrueReg, b&: FalseReg);
3220	}
3221
3222	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3223	const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3224	unsigned DstSize = RI.getRegSizeInBits(*DstRC);
3225
3226	if (DstSize == 32) {
3227	MachineInstr *Select;
3228	if (Pred == SCC_TRUE) {
3229	Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)
3230	.addReg(TrueReg)
3231	.addReg(FalseReg);
3232	} else {
3233	// Instruction's operands are backwards from what is expected.
3234	Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)
3235	.addReg(FalseReg)
3236	.addReg(TrueReg);
3237	}
3238
3239	preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3240	return;
3241	}
3242
3243	if (DstSize == 64 && Pred == SCC_TRUE) {
3244	MachineInstr *Select =
3245	BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
3246	.addReg(TrueReg)
3247	.addReg(FalseReg);
3248
3249	preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3250	return;
3251	}
3252
3253	static const int16_t Sub0_15[] = {
3254	AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3255	AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3256	AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3257	AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3258	};
3259
3260	static const int16_t Sub0_15_64[] = {
3261	AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3262	AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3263	AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3264	AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3265	};
3266
3267	unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3268	const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3269	const int16_t *SubIndices = Sub0_15;
3270	int NElts = DstSize / 32;
3271
3272	// 64-bit select is only available for SALU.
3273	// TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3274	if (Pred == SCC_TRUE) {
3275	if (NElts % 2) {
3276	SelOp = AMDGPU::S_CSELECT_B32;
3277	EltRC = &AMDGPU::SGPR_32RegClass;
3278	} else {
3279	SelOp = AMDGPU::S_CSELECT_B64;
3280	EltRC = &AMDGPU::SGPR_64RegClass;
3281	SubIndices = Sub0_15_64;
3282	NElts /= 2;
3283	}
3284	}
3285
3286	MachineInstrBuilder MIB = BuildMI(
3287	MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
3288
3289	I = MIB->getIterator();
3290
3291	SmallVector<Register, 8> Regs;
3292	for (int Idx = 0; Idx != NElts; ++Idx) {
3293	Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3294	Regs.push_back(Elt: DstElt);
3295
3296	unsigned SubIdx = SubIndices[Idx];
3297
3298	MachineInstr *Select;
3299	if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3300	Select =
3301	BuildMI(MBB, I, DL, get(SelOp), DstElt)
3302	.addReg(FalseReg, 0, SubIdx)
3303	.addReg(TrueReg, 0, SubIdx);
3304	} else {
3305	Select =
3306	BuildMI(MBB, I, DL, get(SelOp), DstElt)
3307	.addReg(TrueReg, 0, SubIdx)
3308	.addReg(FalseReg, 0, SubIdx);
3309	}
3310
3311	preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3312	fixImplicitOperands(MI&: *Select);
3313
3314	MIB.addReg(RegNo: DstElt)
3315	.addImm(Val: SubIdx);
3316	}
3317	}
3318
3319	bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3320	switch (MI.getOpcode()) {
3321	case AMDGPU::V_MOV_B32_e32:
3322	case AMDGPU::V_MOV_B32_e64:
3323	case AMDGPU::V_MOV_B64_PSEUDO:
3324	case AMDGPU::V_MOV_B64_e32:
3325	case AMDGPU::V_MOV_B64_e64:
3326	case AMDGPU::S_MOV_B32:
3327	case AMDGPU::S_MOV_B64:
3328	case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3329	case AMDGPU::COPY:
3330	case AMDGPU::WWM_COPY:
3331	case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3332	case AMDGPU::V_ACCVGPR_READ_B32_e64:
3333	case AMDGPU::V_ACCVGPR_MOV_B32:
3334	return true;
3335	default:
3336	return false;
3337	}
3338	}
3339
3340	static constexpr unsigned ModifierOpNames[] = {
3341	AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3342	AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3343	AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3344
3345	void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3346	unsigned Opc = MI.getOpcode();
3347	for (unsigned Name : reverse(ModifierOpNames)) {
3348	int Idx = AMDGPU::getNamedOperandIdx(Opc, Name);
3349	if (Idx >= 0)
3350	MI.removeOperand(Idx);
3351	}
3352	}
3353
3354	bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3355	Register Reg, MachineRegisterInfo *MRI) const {
3356	if (!MRI->hasOneNonDBGUse(RegNo: Reg))
3357	return false;
3358
3359	switch (DefMI.getOpcode()) {
3360	default:
3361	return false;
3362	case AMDGPU::V_MOV_B64_e32:
3363	case AMDGPU::S_MOV_B64:
3364	case AMDGPU::V_MOV_B64_PSEUDO:
3365	case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3366	case AMDGPU::V_MOV_B32_e32:
3367	case AMDGPU::S_MOV_B32:
3368	case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3369	break;
3370	}
3371
3372	const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
3373	assert(ImmOp);
3374	// FIXME: We could handle FrameIndex values here.
3375	if (!ImmOp->isImm())
3376	return false;
3377
3378	auto getImmFor = [ImmOp](const MachineOperand &UseOp) -> int64_t {
3379	int64_t Imm = ImmOp->getImm();
3380	switch (UseOp.getSubReg()) {
3381	default:
3382	return Imm;
3383	case AMDGPU::sub0:
3384	return Lo_32(Value: Imm);
3385	case AMDGPU::sub1:
3386	return Hi_32(Value: Imm);
3387	case AMDGPU::lo16:
3388	return APInt(16, Imm).getSExtValue();
3389	case AMDGPU::hi16:
3390	return APInt(32, Imm).ashr(ShiftAmt: 16).getSExtValue();
3391	case AMDGPU::sub1_lo16:
3392	return APInt(16, Hi_32(Value: Imm)).getSExtValue();
3393	case AMDGPU::sub1_hi16:
3394	return APInt(32, Hi_32(Value: Imm)).ashr(ShiftAmt: 16).getSExtValue();
3395	}
3396	};
3397
3398	assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
3399
3400	unsigned Opc = UseMI.getOpcode();
3401	if (Opc == AMDGPU::COPY) {
3402	assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
3403
3404	Register DstReg = UseMI.getOperand(i: 0).getReg();
3405	unsigned OpSize = getOpSize(MI: UseMI, OpNo: 0);
3406	bool Is16Bit = OpSize == 2;
3407	bool Is64Bit = OpSize == 8;
3408	bool isVGPRCopy = RI.isVGPR(*MRI, DstReg);
3409	unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO
3410	: AMDGPU::V_MOV_B32_e32
3411	: Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO
3412	: AMDGPU::S_MOV_B32;
3413	APInt Imm(Is64Bit ? 64 : 32, getImmFor(UseMI.getOperand(i: 1)));
3414
3415	if (RI.isAGPR(*MRI, DstReg)) {
3416	if (Is64Bit || !isInlineConstant(Imm))
3417	return false;
3418	NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
3419	}
3420
3421	if (Is16Bit) {
3422	if (isVGPRCopy)
3423	return false; // Do not clobber vgpr_hi16
3424
3425	if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16)
3426	return false;
3427
3428	UseMI.getOperand(i: 0).setSubReg(0);
3429	if (DstReg.isPhysical()) {
3430	DstReg = RI.get32BitRegister(Reg: DstReg);
3431	UseMI.getOperand(i: 0).setReg(DstReg);
3432	}
3433	assert(UseMI.getOperand(1).getReg().isVirtual());
3434	}
3435
3436	const MCInstrDesc &NewMCID = get(NewOpc);
3437	if (DstReg.isPhysical() &&
3438	!RI.getRegClass(NewMCID.operands()[0].RegClass)->contains(DstReg))
3439	return false;
3440
3441	UseMI.setDesc(NewMCID);
3442	UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: Imm.getSExtValue());
3443	UseMI.addImplicitDefUseOperands(MF&: *UseMI.getParent()->getParent());
3444	return true;
3445	}
3446
3447	if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3448	Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3449	Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3450	Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3451	Opc == AMDGPU::V_FMAC_F16_t16_e64) {
3452	// Don't fold if we are using source or output modifiers. The new VOP2
3453	// instructions don't have them.
3454	if (hasAnyModifiersSet(MI: UseMI))
3455	return false;
3456
3457	// If this is a free constant, there's no reason to do this.
3458	// TODO: We could fold this here instead of letting SIFoldOperands do it
3459	// later.
3460	MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
3461
3462	// Any src operand can be used for the legality check.
3463	if (isInlineConstant(MI: UseMI, UseMO: *Src0, DefMO: *ImmOp))
3464	return false;
3465
3466	bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3467	Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64;
3468	bool IsFMA =
3469	Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3470	Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3471	Opc == AMDGPU::V_FMAC_F16_t16_e64;
3472	MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
3473	MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
3474
3475	// Multiplied part is the constant: Use v_madmk_{f16, f32}.
3476	if ((Src0->isReg() && Src0->getReg() == Reg) ||
3477	(Src1->isReg() && Src1->getReg() == Reg)) {
3478	MachineOperand *RegSrc =
3479	Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
3480	if (!RegSrc->isReg())
3481	return false;
3482	if (RI.isSGPRClass(MRI->getRegClass(Reg: RegSrc->getReg())) &&
3483	ST.getConstantBusLimit(Opcode: Opc) < 2)
3484	return false;
3485
3486	if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Reg: Src2->getReg())))
3487	return false;
3488
3489	// If src2 is also a literal constant then we have to choose which one to
3490	// fold. In general it is better to choose madak so that the other literal
3491	// can be materialized in an sgpr instead of a vgpr:
3492	// s_mov_b32 s0, literal
3493	// v_madak_f32 v0, s0, v0, literal
3494	// Instead of:
3495	// v_mov_b32 v1, literal
3496	// v_madmk_f32 v0, v0, literal, v1
3497	MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
3498	if (Def && Def->isMoveImmediate() &&
3499	!isInlineConstant(MO: Def->getOperand(i: 1)))
3500	return false;
3501
3502	unsigned NewOpc =
3503	IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32
3504	: ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16
3505	: AMDGPU::V_FMAMK_F16)
3506	: (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
3507	if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3508	return false;
3509
3510	// V_FMAMK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite
3511	// would also require restricting their register classes. For now
3512	// just bail out.
3513	if (NewOpc == AMDGPU::V_FMAMK_F16_t16)
3514	return false;
3515
3516	const int64_t Imm = getImmFor(RegSrc == Src1 ? *Src0 : *Src1);
3517
3518	// FIXME: This would be a lot easier if we could return a new instruction
3519	// instead of having to modify in place.
3520
3521	Register SrcReg = RegSrc->getReg();
3522	unsigned SrcSubReg = RegSrc->getSubReg();
3523	Src0->setReg(SrcReg);
3524	Src0->setSubReg(SrcSubReg);
3525	Src0->setIsKill(RegSrc->isKill());
3526
3527	if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3528	Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3529	Opc == AMDGPU::V_FMAC_F16_e64)
3530	UseMI.untieRegOperand(
3531	AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3532
3533	Src1->ChangeToImmediate(ImmVal: Imm);
3534
3535	removeModOperands(MI&: UseMI);
3536	UseMI.setDesc(get(NewOpc));
3537
3538	bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
3539	if (DeleteDef)
3540	DefMI.eraseFromParent();
3541
3542	return true;
3543	}
3544
3545	// Added part is the constant: Use v_madak_{f16, f32}.
3546	if (Src2->isReg() && Src2->getReg() == Reg) {
3547	if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
3548	// Not allowed to use constant bus for another operand.
3549	// We can however allow an inline immediate as src0.
3550	bool Src0Inlined = false;
3551	if (Src0->isReg()) {
3552	// Try to inline constant if possible.
3553	// If the Def moves immediate and the use is single
3554	// We are saving VGPR here.
3555	MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
3556	if (Def && Def->isMoveImmediate() &&
3557	isInlineConstant(MO: Def->getOperand(i: 1)) &&
3558	MRI->hasOneUse(RegNo: Src0->getReg())) {
3559	Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
3560	Src0Inlined = true;
3561	} else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
3562	RI.isSGPRReg(*MRI, Src0->getReg())) {
3563	return false;
3564	}
3565	// VGPR is okay as Src0 - fallthrough
3566	}
3567
3568	if (Src1->isReg() && !Src0Inlined) {
3569	// We have one slot for inlinable constant so far - try to fill it
3570	MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
3571	if (Def && Def->isMoveImmediate() &&
3572	isInlineConstant(MO: Def->getOperand(i: 1)) &&
3573	MRI->hasOneUse(RegNo: Src1->getReg()) && commuteInstruction(UseMI))
3574	Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
3575	else if (RI.isSGPRReg(*MRI, Src1->getReg()))
3576	return false;
3577	// VGPR is okay as Src1 - fallthrough
3578	}
3579	}
3580
3581	unsigned NewOpc =
3582	IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32
3583	: ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16
3584	: AMDGPU::V_FMAAK_F16)
3585	: (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
3586	if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3587	return false;
3588
3589	// V_FMAAK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite
3590	// would also require restricting their register classes. For now
3591	// just bail out.
3592	if (NewOpc == AMDGPU::V_FMAAK_F16_t16)
3593	return false;
3594
3595	// FIXME: This would be a lot easier if we could return a new instruction
3596	// instead of having to modify in place.
3597
3598	if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3599	Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3600	Opc == AMDGPU::V_FMAC_F16_e64)
3601	UseMI.untieRegOperand(
3602	AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3603
3604	// ChangingToImmediate adds Src2 back to the instruction.
3605	Src2->ChangeToImmediate(ImmVal: getImmFor(*Src2));
3606
3607	// These come before src2.
3608	removeModOperands(MI&: UseMI);
3609	UseMI.setDesc(get(NewOpc));
3610	// It might happen that UseMI was commuted
3611	// and we now have SGPR as SRC1. If so 2 inlined
3612	// constant and SGPR are illegal.
3613	legalizeOperands(MI&: UseMI);
3614
3615	bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
3616	if (DeleteDef)
3617	DefMI.eraseFromParent();
3618
3619	return true;
3620	}
3621	}
3622
3623	return false;
3624	}
3625
3626	static bool
3627	memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
3628	ArrayRef<const MachineOperand *> BaseOps2) {
3629	if (BaseOps1.size() != BaseOps2.size())
3630	return false;
3631	for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
3632	if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
3633	return false;
3634	}
3635	return true;
3636	}
3637
3638	static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
3639	int WidthB, int OffsetB) {
3640	int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
3641	int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
3642	int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3643	return LowOffset + LowWidth <= HighOffset;
3644	}
3645
3646	bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
3647	const MachineInstr &MIb) const {
3648	SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
3649	int64_t Offset0, Offset1;
3650	unsigned Dummy0, Dummy1;
3651	bool Offset0IsScalable, Offset1IsScalable;
3652	if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
3653	Dummy0, &RI) ||
3654	!getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,
3655	Dummy1, &RI))
3656	return false;
3657
3658	if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
3659	return false;
3660
3661	if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
3662	// FIXME: Handle ds_read2 / ds_write2.
3663	return false;
3664	}
3665	unsigned Width0 = MIa.memoperands().front()->getSize();
3666	unsigned Width1 = MIb.memoperands().front()->getSize();
3667	return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
3668	}
3669
3670	bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
3671	const MachineInstr &MIb) const {
3672	assert(MIa.mayLoadOrStore() &&
3673	"MIa must load from or modify a memory location");
3674	assert(MIb.mayLoadOrStore() &&
3675	"MIb must load from or modify a memory location");
3676
3677	if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
3678	return false;
3679
3680	// XXX - Can we relax this between address spaces?
3681	if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3682	return false;
3683
3684	if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
3685	return false;
3686
3687	// TODO: Should we check the address space from the MachineMemOperand? That
3688	// would allow us to distinguish objects we know don't alias based on the
3689	// underlying address space, even if it was lowered to a different one,
3690	// e.g. private accesses lowered to use MUBUF instructions on a scratch
3691	// buffer.
3692	if (isDS(MI: MIa)) {
3693	if (isDS(MI: MIb))
3694	return checkInstOffsetsDoNotOverlap(MIa, MIb);
3695
3696	return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
3697	}
3698
3699	if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
3700	if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
3701	return checkInstOffsetsDoNotOverlap(MIa, MIb);
3702
3703	if (isFLAT(MI: MIb))
3704	return isFLATScratch(MI: MIb);
3705
3706	return !isSMRD(MI: MIb);
3707	}
3708
3709	if (isSMRD(MI: MIa)) {
3710	if (isSMRD(MI: MIb))
3711	return checkInstOffsetsDoNotOverlap(MIa, MIb);
3712
3713	if (isFLAT(MI: MIb))
3714	return isFLATScratch(MI: MIb);
3715
3716	return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
3717	}
3718
3719	if (isFLAT(MI: MIa)) {
3720	if (isFLAT(MI: MIb)) {
3721	if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
3722	(isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
3723	return true;
3724
3725	return checkInstOffsetsDoNotOverlap(MIa, MIb);
3726	}
3727
3728	return false;
3729	}
3730
3731	return false;
3732	}
3733
3734	static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI,
3735	int64_t &Imm, MachineInstr **DefMI = nullptr) {
3736	if (Reg.isPhysical())
3737	return false;
3738	auto *Def = MRI.getUniqueVRegDef(Reg);
3739	if (Def && SIInstrInfo::isFoldableCopy(MI: *Def) && Def->getOperand(i: 1).isImm()) {
3740	Imm = Def->getOperand(i: 1).getImm();
3741	if (DefMI)
3742	*DefMI = Def;
3743	return true;
3744	}
3745	return false;
3746	}
3747
3748	static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,
3749	MachineInstr **DefMI = nullptr) {
3750	if (!MO->isReg())
3751	return false;
3752	const MachineFunction *MF = MO->getParent()->getParent()->getParent();
3753	const MachineRegisterInfo &MRI = MF->getRegInfo();
3754	return getFoldableImm(Reg: MO->getReg(), MRI, Imm, DefMI);
3755	}
3756
3757	static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
3758	MachineInstr &NewMI) {
3759	if (LV) {
3760	unsigned NumOps = MI.getNumOperands();
3761	for (unsigned I = 1; I < NumOps; ++I) {
3762	MachineOperand &Op = MI.getOperand(i: I);
3763	if (Op.isReg() && Op.isKill())
3764	LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI);
3765	}
3766	}
3767	}
3768
3769	MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
3770	LiveVariables *LV,
3771	LiveIntervals *LIS) const {
3772	MachineBasicBlock &MBB = *MI.getParent();
3773	unsigned Opc = MI.getOpcode();
3774
3775	// Handle MFMA.
3776	int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
3777	if (NewMFMAOpc != -1) {
3778	MachineInstrBuilder MIB =
3779	BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc));
3780	for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3781	MIB.add(MO: MI.getOperand(i: I));
3782	updateLiveVariables(LV, MI, NewMI&: *MIB);
3783	if (LIS)
3784	LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3785	return MIB;
3786	}
3787
3788	if (SIInstrInfo::isWMMA(MI)) {
3789	unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
3790	MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3791	.setMIFlags(MI.getFlags());
3792	for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3793	MIB->addOperand(Op: MI.getOperand(i: I));
3794
3795	updateLiveVariables(LV, MI, NewMI&: *MIB);
3796	if (LIS)
3797	LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3798
3799	return MIB;
3800	}
3801
3802	assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
3803	"V_FMAC_F16_t16_e32 is not supported and not expected to be present "
3804	"pre-RA");
3805
3806	// Handle MAC/FMAC.
3807	bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 ||
3808	Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3809	Opc == AMDGPU::V_FMAC_F16_t16_e64;
3810	bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3811	Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3812	Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 ||
3813	Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3814	Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3815	Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3816	bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3817	bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
3818	Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
3819	Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3820	Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
3821	bool Src0Literal = false;
3822
3823	switch (Opc) {
3824	default:
3825	return nullptr;
3826	case AMDGPU::V_MAC_F16_e64:
3827	case AMDGPU::V_FMAC_F16_e64:
3828	case AMDGPU::V_FMAC_F16_t16_e64:
3829	case AMDGPU::V_MAC_F32_e64:
3830	case AMDGPU::V_MAC_LEGACY_F32_e64:
3831	case AMDGPU::V_FMAC_F32_e64:
3832	case AMDGPU::V_FMAC_LEGACY_F32_e64:
3833	case AMDGPU::V_FMAC_F64_e64:
3834	break;
3835	case AMDGPU::V_MAC_F16_e32:
3836	case AMDGPU::V_FMAC_F16_e32:
3837	case AMDGPU::V_MAC_F32_e32:
3838	case AMDGPU::V_MAC_LEGACY_F32_e32:
3839	case AMDGPU::V_FMAC_F32_e32:
3840	case AMDGPU::V_FMAC_LEGACY_F32_e32:
3841	case AMDGPU::V_FMAC_F64_e32: {
3842	int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3843	AMDGPU::OpName::src0);
3844	const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
3845	if (!Src0->isReg() && !Src0->isImm())
3846	return nullptr;
3847
3848	if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
3849	Src0Literal = true;
3850
3851	break;
3852	}
3853	}
3854
3855	MachineInstrBuilder MIB;
3856	const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3857	const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
3858	const MachineOperand *Src0Mods =
3859	getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
3860	const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3861	const MachineOperand *Src1Mods =
3862	getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
3863	const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3864	const MachineOperand *Src2Mods =
3865	getNamedOperand(MI, AMDGPU::OpName::src2_modifiers);
3866	const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3867	const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
3868	const MachineOperand *OpSel = getNamedOperand(MI, AMDGPU::OpName::op_sel);
3869
3870	if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 &&
3871	!IsLegacy &&
3872	// If we have an SGPR input, we will violate the constant bus restriction.
3873	(ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
3874	!RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) {
3875	MachineInstr *DefMI;
3876	const auto killDef = [&]() -> void {
3877	const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3878	// The only user is the instruction which will be killed.
3879	Register DefReg = DefMI->getOperand(i: 0).getReg();
3880	if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
3881	return;
3882	// We cannot just remove the DefMI here, calling pass will crash.
3883	DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF));
3884	for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I)
3885	DefMI->removeOperand(OpNo: I);
3886	if (LV)
3887	LV->getVarInfo(Reg: DefReg).AliveBlocks.clear();
3888	};
3889
3890	int64_t Imm;
3891	if (!Src0Literal && getFoldableImm(MO: Src2, Imm, DefMI: &DefMI)) {
3892	unsigned NewOpc =
3893	IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16
3894	: AMDGPU::V_FMAAK_F16)
3895	: AMDGPU::V_FMAAK_F32)
3896	: (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
3897	if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
3898	MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3899	.add(*Dst)
3900	.add(*Src0)
3901	.add(*Src1)
3902	.addImm(Imm);
3903	updateLiveVariables(LV, MI, NewMI&: *MIB);
3904	if (LIS)
3905	LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3906	killDef();
3907	return MIB;
3908	}
3909	}
3910	unsigned NewOpc =
3911	IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16
3912	: AMDGPU::V_FMAMK_F16)
3913	: AMDGPU::V_FMAMK_F32)
3914	: (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
3915	if (!Src0Literal && getFoldableImm(MO: Src1, Imm, DefMI: &DefMI)) {
3916	if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
3917	MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3918	.add(*Dst)
3919	.add(*Src0)
3920	.addImm(Imm)
3921	.add(*Src2);
3922	updateLiveVariables(LV, MI, NewMI&: *MIB);
3923	if (LIS)
3924	LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3925	killDef();
3926	return MIB;
3927	}
3928	}
3929	if (Src0Literal || getFoldableImm(MO: Src0, Imm, DefMI: &DefMI)) {
3930	if (Src0Literal) {
3931	Imm = Src0->getImm();
3932	DefMI = nullptr;
3933	}
3934	if (pseudoToMCOpcode(NewOpc) != -1 &&
3935	isOperandLegal(
3936	MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
3937	Src1)) {
3938	MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3939	.add(*Dst)
3940	.add(*Src1)
3941	.addImm(Imm)
3942	.add(*Src2);
3943	updateLiveVariables(LV, MI, NewMI&: *MIB);
3944	if (LIS)
3945	LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3946	if (DefMI)
3947	killDef();
3948	return MIB;
3949	}
3950	}
3951	}
3952
3953	// VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
3954	// if VOP3 does not allow a literal operand.
3955	if (Src0Literal && !ST.hasVOP3Literal())
3956	return nullptr;
3957
3958	unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64
3959	: IsF64 ? AMDGPU::V_FMA_F64_e64
3960	: IsLegacy
3961	? AMDGPU::V_FMA_LEGACY_F32_e64
3962	: AMDGPU::V_FMA_F32_e64
3963	: IsF16 ? AMDGPU::V_MAD_F16_e64
3964	: IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64
3965	: AMDGPU::V_MAD_F32_e64;
3966	if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3967	return nullptr;
3968
3969	MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3970	.add(*Dst)
3971	.addImm(Src0Mods ? Src0Mods->getImm() : 0)
3972	.add(*Src0)
3973	.addImm(Src1Mods ? Src1Mods->getImm() : 0)
3974	.add(*Src1)
3975	.addImm(Src2Mods ? Src2Mods->getImm() : 0)
3976	.add(*Src2)
3977	.addImm(Clamp ? Clamp->getImm() : 0)
3978	.addImm(Omod ? Omod->getImm() : 0);
3979	if (AMDGPU::hasNamedOperand(NewOpc, AMDGPU::OpName::op_sel))
3980	MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
3981	updateLiveVariables(LV, MI, NewMI&: *MIB);
3982	if (LIS)
3983	LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3984	return MIB;
3985	}
3986
3987	// It's not generally safe to move VALU instructions across these since it will
3988	// start using the register as a base index rather than directly.
3989	// XXX - Why isn't hasSideEffects sufficient for these?
3990	static bool changesVGPRIndexingMode(const MachineInstr &MI) {
3991	switch (MI.getOpcode()) {
3992	case AMDGPU::S_SET_GPR_IDX_ON:
3993	case AMDGPU::S_SET_GPR_IDX_MODE:
3994	case AMDGPU::S_SET_GPR_IDX_OFF:
3995	return true;
3996	default:
3997	return false;
3998	}
3999	}
4000
4001	bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4002	const MachineBasicBlock *MBB,
4003	const MachineFunction &MF) const {
4004	// Skipping the check for SP writes in the base implementation. The reason it
4005	// was added was apparently due to compile time concerns.
4006	//
4007	// TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4008	// but is probably avoidable.
4009
4010	// Copied from base implementation.
4011	// Terminators and labels can't be scheduled around.
4012	if (MI.isTerminator() || MI.isPosition())
4013	return true;
4014
4015	// INLINEASM_BR can jump to another block
4016	if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4017	return true;
4018
4019	if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0)
4020	return true;
4021
4022	// Target-independent instructions do not have an implicit-use of EXEC, even
4023	// when they operate on VGPRs. Treating EXEC modifications as scheduling
4024	// boundaries prevents incorrect movements of such instructions.
4025	return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
4026	MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4027	MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4028	MI.getOpcode() == AMDGPU::S_SETPRIO ||
4029	changesVGPRIndexingMode(MI);
4030	}
4031
4032	bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
4033	return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode);
4034	}
4035
4036	bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
4037	// Skip the full operand and register alias search modifiesRegister
4038	// does. There's only a handful of instructions that touch this, it's only an
4039	// implicit def, and doesn't alias any other registers.
4040	return is_contained(MI.getDesc().implicit_defs(), AMDGPU::MODE);
4041	}
4042
4043	bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
4044	unsigned Opcode = MI.getOpcode();
4045
4046	if (MI.mayStore() && isSMRD(MI))
4047	return true; // scalar store or atomic
4048
4049	// This will terminate the function when other lanes may need to continue.
4050	if (MI.isReturn())
4051	return true;
4052
4053	// These instructions cause shader I/O that may cause hardware lockups
4054	// when executed with an empty EXEC mask.
4055	//
4056	// Note: exp with VM = DONE = 0 is automatically skipped by hardware when
4057	// EXEC = 0, but checking for that case here seems not worth it
4058	// given the typical code patterns.
4059	if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4060	isEXP(Opcode) ||
4061	Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
4062	Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
4063	return true;
4064
4065	if (MI.isCall() || MI.isInlineAsm())
4066	return true; // conservative assumption
4067
4068	// A mode change is a scalar operation that influences vector instructions.
4069	if (modifiesModeRegister(MI))
4070	return true;
4071
4072	// These are like SALU instructions in terms of effects, so it's questionable
4073	// whether we should return true for those.
4074	//
4075	// However, executing them with EXEC = 0 causes them to operate on undefined
4076	// data, which we avoid by returning true here.
4077	if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4078	Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4079	Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4080	Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4081	return true;
4082
4083	return false;
4084	}
4085
4086	bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
4087	const MachineInstr &MI) const {
4088	if (MI.isMetaInstruction())
4089	return false;
4090
4091	// This won't read exec if this is an SGPR->SGPR copy.
4092	if (MI.isCopyLike()) {
4093	if (!RI.isSGPRReg(MRI, MI.getOperand(i: 0).getReg()))
4094	return true;
4095
4096	// Make sure this isn't copying exec as a normal operand
4097	return MI.readsRegister(AMDGPU::EXEC, &RI);
4098	}
4099
4100	// Make a conservative assumption about the callee.
4101	if (MI.isCall())
4102	return true;
4103
4104	// Be conservative with any unhandled generic opcodes.
4105	if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
4106	return true;
4107
4108	return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
4109	}
4110
4111	bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
4112	switch (Imm.getBitWidth()) {
4113	case 1: // This likely will be a condition code mask.
4114	return true;
4115
4116	case 32:
4117	return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
4118	HasInv2Pi: ST.hasInv2PiInlineImm());
4119	case 64:
4120	return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
4121	HasInv2Pi: ST.hasInv2PiInlineImm());
4122	case 16:
4123	return ST.has16BitInsts() &&
4124	AMDGPU::isInlinableLiteral16(Literal: Imm.getSExtValue(),
4125	HasInv2Pi: ST.hasInv2PiInlineImm());
4126	default:
4127	llvm_unreachable("invalid bitwidth");
4128	}
4129	}
4130
4131	bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
4132	uint8_t OperandType) const {
4133	assert(!MO.isReg() && "isInlineConstant called on register operand!");
4134	if (!MO.isImm())
4135	return false;
4136
4137	// MachineOperand provides no way to tell the true operand size, since it only
4138	// records a 64-bit value. We need to know the size to determine if a 32-bit
4139	// floating point immediate bit pattern is legal for an integer immediate. It
4140	// would be for any 32-bit integer operand, but would not be for a 64-bit one.
4141
4142	int64_t Imm = MO.getImm();
4143	switch (OperandType) {
4144	case AMDGPU::OPERAND_REG_IMM_INT32:
4145	case AMDGPU::OPERAND_REG_IMM_FP32:
4146	case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
4147	case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4148	case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4149	case AMDGPU::OPERAND_REG_IMM_V2FP32:
4150	case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
4151	case AMDGPU::OPERAND_REG_IMM_V2INT32:
4152	case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
4153	case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4154	case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4155	case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4156	int32_t Trunc = static_cast<int32_t>(Imm);
4157	return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4158	}
4159	case AMDGPU::OPERAND_REG_IMM_INT64:
4160	case AMDGPU::OPERAND_REG_IMM_FP64:
4161	case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4162	case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4163	case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4164	return AMDGPU::isInlinableLiteral64(Literal: MO.getImm(),
4165	HasInv2Pi: ST.hasInv2PiInlineImm());
4166	case AMDGPU::OPERAND_REG_IMM_INT16:
4167	case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4168	case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
4169	// We would expect inline immediates to not be concerned with an integer/fp
4170	// distinction. However, in the case of 16-bit integer operations, the
4171	// "floating point" values appear to not work. It seems read the low 16-bits
4172	// of 32-bit immediates, which happens to always work for the integer
4173	// values.
4174	//
4175	// See llvm bugzilla 46302.
4176	//
4177	// TODO: Theoretically we could use op-sel to use the high bits of the
4178	// 32-bit FP values.
4179	return AMDGPU::isInlinableIntLiteral(Literal: Imm);
4180	case AMDGPU::OPERAND_REG_IMM_V2INT16:
4181	case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4182	case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
4183	return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
4184	case AMDGPU::OPERAND_REG_IMM_V2FP16:
4185	case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4186	case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
4187	return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
4188	case AMDGPU::OPERAND_REG_IMM_FP16:
4189	case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
4190	case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4191	case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
4192	if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4193	// A few special case instructions have 16-bit operands on subtargets
4194	// where 16-bit instructions are not legal.
4195	// TODO: Do the 32-bit immediates work? We shouldn't really need to handle
4196	// constants in these cases
4197	int16_t Trunc = static_cast<int16_t>(Imm);
4198	return ST.has16BitInsts() &&
4199	AMDGPU::isInlinableLiteral16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4200	}
4201
4202	return false;
4203	}
4204	case AMDGPU::OPERAND_KIMM32:
4205	case AMDGPU::OPERAND_KIMM16:
4206	return false;
4207	case AMDGPU::OPERAND_INPUT_MODS:
4208	case MCOI::OPERAND_IMMEDIATE:
4209	// Always embedded in the instruction for free.
4210	return true;
4211	case MCOI::OPERAND_UNKNOWN:
4212	case MCOI::OPERAND_REGISTER:
4213	case MCOI::OPERAND_PCREL:
4214	case MCOI::OPERAND_GENERIC_0:
4215	case MCOI::OPERAND_GENERIC_1:
4216	case MCOI::OPERAND_GENERIC_2:
4217	case MCOI::OPERAND_GENERIC_3:
4218	case MCOI::OPERAND_GENERIC_4:
4219	case MCOI::OPERAND_GENERIC_5:
4220	// Just ignore anything else.
4221	return true;
4222	default:
4223	llvm_unreachable("invalid operand type");
4224	}
4225	}
4226
4227	static bool compareMachineOp(const MachineOperand &Op0,
4228	const MachineOperand &Op1) {
4229	if (Op0.getType() != Op1.getType())
4230	return false;
4231
4232	switch (Op0.getType()) {
4233	case MachineOperand::MO_Register:
4234	return Op0.getReg() == Op1.getReg();
4235	case MachineOperand::MO_Immediate:
4236	return Op0.getImm() == Op1.getImm();
4237	default:
4238	llvm_unreachable("Didn't expect to be comparing these operand types");
4239	}
4240	}
4241
4242	bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
4243	const MachineOperand &MO) const {
4244	const MCInstrDesc &InstDesc = MI.getDesc();
4245	const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
4246
4247	assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
4248
4249	if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
4250	return true;
4251
4252	if (OpInfo.RegClass < 0)
4253	return false;
4254
4255	if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
4256	if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
4257	OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
4258	AMDGPU::OpName::src2))
4259	return false;
4260	return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
4261	}
4262
4263	if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
4264	return false;
4265
4266	if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(Desc: InstDesc, OpNo))
4267	return true;
4268
4269	return ST.hasVOP3Literal();
4270	}
4271
4272	bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
4273	// GFX90A does not have V_MUL_LEGACY_F32_e32.
4274	if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
4275	return false;
4276
4277	int Op32 = AMDGPU::getVOPe32(Opcode);
4278	if (Op32 == -1)
4279	return false;
4280
4281	return pseudoToMCOpcode(Opcode: Op32) != -1;
4282	}
4283
4284	bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
4285	// The src0_modifier operand is present on all instructions
4286	// that have modifiers.
4287
4288	return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers);
4289	}
4290
4291	bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
4292	unsigned OpName) const {
4293	const MachineOperand *Mods = getNamedOperand(MI, OpName);
4294	return Mods && Mods->getImm();
4295	}
4296
4297	bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
4298	return any_of(Range: ModifierOpNames,
4299	P: [&](unsigned Name) { return hasModifiersSet(MI, OpName: Name); });
4300	}
4301
4302	bool SIInstrInfo::canShrink(const MachineInstr &MI,
4303	const MachineRegisterInfo &MRI) const {
4304	const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
4305	// Can't shrink instruction with three operands.
4306	if (Src2) {
4307	switch (MI.getOpcode()) {
4308	default: return false;
4309
4310	case AMDGPU::V_ADDC_U32_e64:
4311	case AMDGPU::V_SUBB_U32_e64:
4312	case AMDGPU::V_SUBBREV_U32_e64: {
4313	const MachineOperand *Src1
4314	= getNamedOperand(MI, AMDGPU::OpName::src1);
4315	if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
4316	return false;
4317	// Additional verification is needed for sdst/src2.
4318	return true;
4319	}
4320	case AMDGPU::V_MAC_F16_e64:
4321	case AMDGPU::V_MAC_F32_e64:
4322	case AMDGPU::V_MAC_LEGACY_F32_e64:
4323	case AMDGPU::V_FMAC_F16_e64:
4324	case AMDGPU::V_FMAC_F16_t16_e64:
4325	case AMDGPU::V_FMAC_F32_e64:
4326	case AMDGPU::V_FMAC_F64_e64:
4327	case AMDGPU::V_FMAC_LEGACY_F32_e64:
4328	if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
4329	hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
4330	return false;
4331	break;
4332
4333	case AMDGPU::V_CNDMASK_B32_e64:
4334	break;
4335	}
4336	}
4337
4338	const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
4339	if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
4340	hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
4341	return false;
4342
4343	// We don't need to check src0, all input types are legal, so just make sure
4344	// src0 isn't using any modifiers.
4345	if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
4346	return false;
4347
4348	// Can it be shrunk to a valid 32 bit opcode?
4349	if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
4350	return false;
4351
4352	// Check output modifiers
4353	return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
4354	!hasModifiersSet(MI, AMDGPU::OpName::clamp);
4355	}
4356
4357	// Set VCC operand with all flags from \p Orig, except for setting it as
4358	// implicit.
4359	static void copyFlagsToImplicitVCC(MachineInstr &MI,
4360	const MachineOperand &Orig) {
4361
4362	for (MachineOperand &Use : MI.implicit_operands()) {
4363	if (Use.isUse() &&
4364	(Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
4365	Use.setIsUndef(Orig.isUndef());
4366	Use.setIsKill(Orig.isKill());
4367	return;
4368	}
4369	}
4370	}
4371
4372	MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
4373	unsigned Op32) const {
4374	MachineBasicBlock *MBB = MI.getParent();
4375	MachineInstrBuilder Inst32 =
4376	BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32))
4377	.setMIFlags(MI.getFlags());
4378
4379	// Add the dst operand if the 32-bit encoding also has an explicit $vdst.
4380	// For VOPC instructions, this is replaced by an implicit def of vcc.
4381	if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::vdst)) {
4382	// dst
4383	Inst32.add(MO: MI.getOperand(i: 0));
4384	} else if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::sdst)) {
4385	// VOPCX instructions won't be writing to an explicit dst, so this should
4386	// not fail for these instructions.
4387	assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
4388	(MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
4389	"Unexpected case");
4390	}
4391
4392	Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
4393
4394	const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
4395	if (Src1)
4396	Inst32.add(MO: *Src1);
4397
4398	const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
4399
4400	if (Src2) {
4401	int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
4402	if (Op32Src2Idx != -1) {
4403	Inst32.add(MO: *Src2);
4404	} else {
4405	// In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
4406	// replaced with an implicit read of vcc or vcc_lo. The implicit read
4407	// of vcc was already added during the initial BuildMI, but we
4408	// 1) may need to change vcc to vcc_lo to preserve the original register
4409	// 2) have to preserve the original flags.
4410	fixImplicitOperands(MI&: *Inst32);
4411	copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
4412	}
4413	}
4414
4415	return Inst32;
4416	}
4417
4418	bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
4419	const MachineOperand &MO,
4420	const MCOperandInfo &OpInfo) const {
4421	// Literal constants use the constant bus.
4422	if (!MO.isReg())
4423	return !isInlineConstant(MO, OpInfo);
4424
4425	if (!MO.isUse())
4426	return false;
4427
4428	if (MO.getReg().isVirtual())
4429	return RI.isSGPRClass(MRI.getRegClass(Reg: MO.getReg()));
4430
4431	// Null is free
4432	if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64)
4433	return false;
4434
4435	// SGPRs use the constant bus
4436	if (MO.isImplicit()) {
4437	return MO.getReg() == AMDGPU::M0 ||
4438	MO.getReg() == AMDGPU::VCC ||
4439	MO.getReg() == AMDGPU::VCC_LO;
4440	} else {
4441	return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
4442	AMDGPU::SReg_64RegClass.contains(MO.getReg());
4443	}
4444	}
4445
4446	static Register findImplicitSGPRRead(const MachineInstr &MI) {
4447	for (const MachineOperand &MO : MI.implicit_operands()) {
4448	// We only care about reads.
4449	if (MO.isDef())
4450	continue;
4451
4452	switch (MO.getReg()) {
4453	case AMDGPU::VCC:
4454	case AMDGPU::VCC_LO:
4455	case AMDGPU::VCC_HI:
4456	case AMDGPU::M0:
4457	case AMDGPU::FLAT_SCR:
4458	return MO.getReg();
4459
4460	default:
4461	break;
4462	}
4463	}
4464
4465	return Register();
4466	}
4467
4468	static bool shouldReadExec(const MachineInstr &MI) {
4469	if (SIInstrInfo::isVALU(MI)) {
4470	switch (MI.getOpcode()) {
4471	case AMDGPU::V_READLANE_B32:
4472	case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
4473	case AMDGPU::V_WRITELANE_B32:
4474	case AMDGPU::SI_SPILL_S32_TO_VGPR:
4475	return false;
4476	}
4477
4478	return true;
4479	}
4480
4481	if (MI.isPreISelOpcode() ||
4482	SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
4483	SIInstrInfo::isSALU(MI) ||
4484	SIInstrInfo::isSMRD(MI))
4485	return false;
4486
4487	return true;
4488	}
4489
4490	static bool isSubRegOf(const SIRegisterInfo &TRI,
4491	const MachineOperand &SuperVec,
4492	const MachineOperand &SubReg) {
4493	if (SubReg.getReg().isPhysical())
4494	return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
4495
4496	return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
4497	SubReg.getReg() == SuperVec.getReg();
4498	}
4499
4500	bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
4501	StringRef &ErrInfo) const {
4502	uint16_t Opcode = MI.getOpcode();
4503	if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
4504	return true;
4505
4506	const MachineFunction *MF = MI.getParent()->getParent();
4507	const MachineRegisterInfo &MRI = MF->getRegInfo();
4508
4509	int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4510	int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4511	int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
4512	int Src3Idx = -1;
4513	if (Src0Idx == -1) {
4514	// VOPD V_DUAL_* instructions use different operand names.
4515	Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);
4516	Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);
4517	Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);
4518	Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);
4519	}
4520
4521	// Make sure the number of operands is correct.
4522	const MCInstrDesc &Desc = get(Opcode);
4523	if (!Desc.isVariadic() &&
4524	Desc.getNumOperands() != MI.getNumExplicitOperands()) {
4525	ErrInfo = "Instruction has wrong number of operands.";
4526	return false;
4527	}
4528
4529	if (MI.isInlineAsm()) {
4530	// Verify register classes for inlineasm constraints.
4531	for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
4532	I != E; ++I) {
4533	const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
4534	if (!RC)
4535	continue;
4536
4537	const MachineOperand &Op = MI.getOperand(i: I);
4538	if (!Op.isReg())
4539	continue;
4540
4541	Register Reg = Op.getReg();
4542	if (!Reg.isVirtual() && !RC->contains(Reg)) {
4543	ErrInfo = "inlineasm operand has incorrect register class.";
4544	return false;
4545	}
4546	}
4547
4548	return true;
4549	}
4550
4551	if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
4552	ErrInfo = "missing memory operand from image instruction.";
4553	return false;
4554	}
4555
4556	// Make sure the register classes are correct.
4557	for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
4558	const MachineOperand &MO = MI.getOperand(i);
4559	if (MO.isFPImm()) {
4560	ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
4561	"all fp values to integers.";
4562	return false;
4563	}
4564
4565	int RegClass = Desc.operands()[i].RegClass;
4566
4567	switch (Desc.operands()[i].OperandType) {
4568	case MCOI::OPERAND_REGISTER:
4569	if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
4570	ErrInfo = "Illegal immediate value for operand.";
4571	return false;
4572	}
4573	break;
4574	case AMDGPU::OPERAND_REG_IMM_INT32:
4575	case AMDGPU::OPERAND_REG_IMM_FP32:
4576	case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
4577	case AMDGPU::OPERAND_REG_IMM_V2FP32:
4578	break;
4579	case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4580	case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4581	case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4582	case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4583	case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4584	case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4585	case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4586	case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4587	case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
4588	case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
4589	case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
4590	if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
4591	ErrInfo = "Illegal immediate value for operand.";
4592	return false;
4593	}
4594	break;
4595	}
4596	case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
4597	if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
4598	ErrInfo = "Expected inline constant for operand.";
4599	return false;
4600	}
4601	break;
4602	case MCOI::OPERAND_IMMEDIATE:
4603	case AMDGPU::OPERAND_KIMM32:
4604	// Check if this operand is an immediate.
4605	// FrameIndex operands will be replaced by immediates, so they are
4606	// allowed.
4607	if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
4608	ErrInfo = "Expected immediate, but got non-immediate";
4609	return false;
4610	}
4611	[[fallthrough]];
4612	default:
4613	continue;
4614	}
4615
4616	if (!MO.isReg())
4617	continue;
4618	Register Reg = MO.getReg();
4619	if (!Reg)
4620	continue;
4621
4622	// FIXME: Ideally we would have separate instruction definitions with the
4623	// aligned register constraint.
4624	// FIXME: We do not verify inline asm operands, but custom inline asm
4625	// verification is broken anyway
4626	if (ST.needsAlignedVGPRs()) {
4627	const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
4628	if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {
4629	const TargetRegisterClass *SubRC =
4630	RI.getSubRegisterClass(RC, MO.getSubReg());
4631	RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg());
4632	if (RC)
4633	RC = SubRC;
4634	}
4635
4636	// Check that this is the aligned version of the class.
4637	if (!RC || !RI.isProperlyAlignedRC(*RC)) {
4638	ErrInfo = "Subtarget requires even aligned vector registers";
4639	return false;
4640	}
4641	}
4642
4643	if (RegClass != -1) {
4644	if (Reg.isVirtual())
4645	continue;
4646
4647	const TargetRegisterClass *RC = RI.getRegClass(RegClass);
4648	if (!RC->contains(Reg)) {
4649	ErrInfo = "Operand has incorrect register class.";
4650	return false;
4651	}
4652	}
4653	}
4654
4655	// Verify SDWA
4656	if (isSDWA(MI)) {
4657	if (!ST.hasSDWA()) {
4658	ErrInfo = "SDWA is not supported on this target";
4659	return false;
4660	}
4661
4662	int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4663
4664	for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
4665	if (OpIdx == -1)
4666	continue;
4667	const MachineOperand &MO = MI.getOperand(i: OpIdx);
4668
4669	if (!ST.hasSDWAScalar()) {
4670	// Only VGPRS on VI
4671	if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
4672	ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
4673	return false;
4674	}
4675	} else {
4676	// No immediates on GFX9
4677	if (!MO.isReg()) {
4678	ErrInfo =
4679	"Only reg allowed as operands in SDWA instructions on GFX9+";
4680	return false;
4681	}
4682	}
4683	}
4684
4685	if (!ST.hasSDWAOmod()) {
4686	// No omod allowed on VI
4687	const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4688	if (OMod != nullptr &&
4689	(!OMod->isImm() || OMod->getImm() != 0)) {
4690	ErrInfo = "OMod not allowed in SDWA instructions on VI";
4691	return false;
4692	}
4693	}
4694
4695	uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
4696	if (isVOPC(Opcode: BasicOpcode)) {
4697	if (!ST.hasSDWASdst() && DstIdx != -1) {
4698	// Only vcc allowed as dst on VI for VOPC
4699	const MachineOperand &Dst = MI.getOperand(i: DstIdx);
4700	if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
4701	ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
4702	return false;
4703	}
4704	} else if (!ST.hasSDWAOutModsVOPC()) {
4705	// No clamp allowed on GFX9 for VOPC
4706	const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
4707	if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
4708	ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
4709	return false;
4710	}
4711
4712	// No omod allowed on GFX9 for VOPC
4713	const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4714	if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
4715	ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
4716	return false;
4717	}
4718	}
4719	}
4720
4721	const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
4722	if (DstUnused && DstUnused->isImm() &&
4723	DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
4724	const MachineOperand &Dst = MI.getOperand(i: DstIdx);
4725	if (!Dst.isReg() || !Dst.isTied()) {
4726	ErrInfo = "Dst register should have tied register";
4727	return false;
4728	}
4729
4730	const MachineOperand &TiedMO =
4731	MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
4732	if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
4733	ErrInfo =
4734	"Dst register should be tied to implicit use of preserved register";
4735	return false;
4736	} else if (TiedMO.getReg().isPhysical() &&
4737	Dst.getReg() != TiedMO.getReg()) {
4738	ErrInfo = "Dst register should use same physical register as preserved";
4739	return false;
4740	}
4741	}
4742	}
4743
4744	// Verify MIMG / VIMAGE / VSAMPLE
4745	if (isImage(Opcode: MI.getOpcode()) && !MI.mayStore()) {
4746	// Ensure that the return type used is large enough for all the options
4747	// being used TFE/LWE require an extra result register.
4748	const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
4749	if (DMask) {
4750	uint64_t DMaskImm = DMask->getImm();
4751	uint32_t RegCount =
4752	isGather4(Opcode: MI.getOpcode()) ? 4 : llvm::popcount(Value: DMaskImm);
4753	const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
4754	const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
4755	const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
4756
4757	// Adjust for packed 16 bit values
4758	if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
4759	RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
4760
4761	// Adjust if using LWE or TFE
4762	if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
4763	RegCount += 1;
4764
4765	const uint32_t DstIdx =
4766	AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
4767	const MachineOperand &Dst = MI.getOperand(i: DstIdx);
4768	if (Dst.isReg()) {
4769	const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
4770	uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
4771	if (RegCount > DstSize) {
4772	ErrInfo = "Image instruction returns too many registers for dst "
4773	"register class";
4774	return false;
4775	}
4776	}
4777	}
4778	}
4779
4780	// Verify VOP*. Ignore multiple sgpr operands on writelane.
4781	if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
4782	unsigned ConstantBusCount = 0;
4783	bool UsesLiteral = false;
4784	const MachineOperand *LiteralVal = nullptr;
4785
4786	int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
4787	if (ImmIdx != -1) {
4788	++ConstantBusCount;
4789	UsesLiteral = true;
4790	LiteralVal = &MI.getOperand(i: ImmIdx);
4791	}
4792
4793	SmallVector<Register, 2> SGPRsUsed;
4794	Register SGPRUsed;
4795
4796	// Only look at the true operands. Only a real operand can use the constant
4797	// bus, and we don't want to check pseudo-operands like the source modifier
4798	// flags.
4799	for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
4800	if (OpIdx == -1)
4801	continue;
4802	const MachineOperand &MO = MI.getOperand(i: OpIdx);
4803	if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
4804	if (MO.isReg()) {
4805	SGPRUsed = MO.getReg();
4806	if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
4807	++ConstantBusCount;
4808	SGPRsUsed.push_back(Elt: SGPRUsed);
4809	}
4810	} else {
4811	if (!UsesLiteral) {
4812	++ConstantBusCount;
4813	UsesLiteral = true;
4814	LiteralVal = &MO;
4815	} else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
4816	assert(isVOP2(MI) || isVOP3(MI));
4817	ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
4818	return false;
4819	}
4820	}
4821	}
4822	}
4823
4824	SGPRUsed = findImplicitSGPRRead(MI);
4825	if (SGPRUsed) {
4826	// Implicit uses may safely overlap true operands
4827	if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
4828	return !RI.regsOverlap(SGPRUsed, SGPR);
4829	})) {
4830	++ConstantBusCount;
4831	SGPRsUsed.push_back(Elt: SGPRUsed);
4832	}
4833	}
4834
4835	// v_writelane_b32 is an exception from constant bus restriction:
4836	// vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
4837	if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
4838	Opcode != AMDGPU::V_WRITELANE_B32) {
4839	ErrInfo = "VOP* instruction violates constant bus restriction";
4840	return false;
4841	}
4842
4843	if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
4844	ErrInfo = "VOP3 instruction uses literal";
4845	return false;
4846	}
4847	}
4848
4849	// Special case for writelane - this can break the multiple constant bus rule,
4850	// but still can't use more than one SGPR register
4851	if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
4852	unsigned SGPRCount = 0;
4853	Register SGPRUsed;
4854
4855	for (int OpIdx : {Src0Idx, Src1Idx}) {
4856	if (OpIdx == -1)
4857	break;
4858
4859	const MachineOperand &MO = MI.getOperand(i: OpIdx);
4860
4861	if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
4862	if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
4863	if (MO.getReg() != SGPRUsed)
4864	++SGPRCount;
4865	SGPRUsed = MO.getReg();
4866	}
4867	}
4868	if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
4869	ErrInfo = "WRITELANE instruction violates constant bus restriction";
4870	return false;
4871	}
4872	}
4873	}
4874
4875	// Verify misc. restrictions on specific instructions.
4876	if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
4877	Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
4878	const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
4879	const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
4880	const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
4881	if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
4882	if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
4883	!compareMachineOp(Op0: Src0, Op1: Src2)) {
4884	ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
4885	return false;
4886	}
4887	}
4888	if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
4889	SISrcMods::ABS) ||
4890	(getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &
4891	SISrcMods::ABS) ||
4892	(getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &
4893	SISrcMods::ABS)) {
4894	ErrInfo = "ABS not allowed in VOP3B instructions";
4895	return false;
4896	}
4897	}
4898
4899	if (isSOP2(MI) || isSOPC(MI)) {
4900	const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
4901	const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
4902
4903	if (!Src0.isReg() && !Src1.isReg() &&
4904	!isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
4905	!isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
4906	!Src0.isIdenticalTo(Other: Src1)) {
4907	ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
4908	return false;
4909	}
4910	}
4911
4912	if (isSOPK(MI)) {
4913	auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
4914	if (Desc.isBranch()) {
4915	if (!Op->isMBB()) {
4916	ErrInfo = "invalid branch target for SOPK instruction";
4917	return false;
4918	}
4919	} else {
4920	uint64_t Imm = Op->getImm();
4921	if (sopkIsZext(Opcode)) {
4922	if (!isUInt<16>(x: Imm)) {
4923	ErrInfo = "invalid immediate for SOPK instruction";
4924	return false;
4925	}
4926	} else {
4927	if (!isInt<16>(x: Imm)) {
4928	ErrInfo = "invalid immediate for SOPK instruction";
4929	return false;
4930	}
4931	}
4932	}
4933	}
4934
4935	if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
4936	Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
4937	Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4938	Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
4939	const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4940	Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
4941
4942	const unsigned StaticNumOps =
4943	Desc.getNumOperands() + Desc.implicit_uses().size();
4944	const unsigned NumImplicitOps = IsDst ? 2 : 1;
4945
4946	// Allow additional implicit operands. This allows a fixup done by the post
4947	// RA scheduler where the main implicit operand is killed and implicit-defs
4948	// are added for sub-registers that remain live after this instruction.
4949	if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
4950	ErrInfo = "missing implicit register operands";
4951	return false;
4952	}
4953
4954	const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4955	if (IsDst) {
4956	if (!Dst->isUse()) {
4957	ErrInfo = "v_movreld_b32 vdst should be a use operand";
4958	return false;
4959	}
4960
4961	unsigned UseOpIdx;
4962	if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
4963	UseOpIdx != StaticNumOps + 1) {
4964	ErrInfo = "movrel implicit operands should be tied";
4965	return false;
4966	}
4967	}
4968
4969	const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
4970	const MachineOperand &ImpUse
4971	= MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
4972	if (!ImpUse.isReg() || !ImpUse.isUse() ||
4973	!isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
4974	ErrInfo = "src0 should be subreg of implicit vector use";
4975	return false;
4976	}
4977	}
4978
4979	// Make sure we aren't losing exec uses in the td files. This mostly requires
4980	// being careful when using let Uses to try to add other use registers.
4981	if (shouldReadExec(MI)) {
4982	if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
4983	ErrInfo = "VALU instruction does not implicitly read exec mask";
4984	return false;
4985	}
4986	}
4987
4988	if (isSMRD(MI)) {
4989	if (MI.mayStore() &&
4990	ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4991	// The register offset form of scalar stores may only use m0 as the
4992	// soffset register.
4993	const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);
4994	if (Soff && Soff->getReg() != AMDGPU::M0) {
4995	ErrInfo = "scalar stores must use m0 as offset register";
4996	return false;
4997	}
4998	}
4999	}
5000
5001	if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
5002	const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
5003	if (Offset->getImm() != 0) {
5004	ErrInfo = "subtarget does not support offsets in flat instructions";
5005	return false;
5006	}
5007	}
5008
5009	if (isDS(MI) && !ST.hasGDS()) {
5010	const MachineOperand *GDSOp = getNamedOperand(MI, AMDGPU::OpName::gds);
5011	if (GDSOp && GDSOp->getImm() != 0) {
5012	ErrInfo = "GDS is not supported on this subtarget";
5013	return false;
5014	}
5015	}
5016
5017	if (isImage(MI)) {
5018	const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
5019	if (DimOp) {
5020	int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
5021	AMDGPU::OpName::vaddr0);
5022	int RSrcOpName =
5023	isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
5024	int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: RSrcOpName);
5025	const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
5026	const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
5027	AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
5028	const AMDGPU::MIMGDimInfo *Dim =
5029	AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
5030
5031	if (!Dim) {
5032	ErrInfo = "dim is out of range";
5033	return false;
5034	}
5035
5036	bool IsA16 = false;
5037	if (ST.hasR128A16()) {
5038	const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);
5039	IsA16 = R128A16->getImm() != 0;
5040	} else if (ST.hasA16()) {
5041	const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);
5042	IsA16 = A16->getImm() != 0;
5043	}
5044
5045	bool IsNSA = RsrcIdx - VAddr0Idx > 1;
5046
5047	unsigned AddrWords =
5048	AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
5049
5050	unsigned VAddrWords;
5051	if (IsNSA) {
5052	VAddrWords = RsrcIdx - VAddr0Idx;
5053	if (ST.hasPartialNSAEncoding() &&
5054	AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
5055	unsigned LastVAddrIdx = RsrcIdx - 1;
5056	VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
5057	}
5058	} else {
5059	VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
5060	if (AddrWords > 12)
5061	AddrWords = 16;
5062	}
5063
5064	if (VAddrWords != AddrWords) {
5065	LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
5066	<< " but got " << VAddrWords << "\n");
5067	ErrInfo = "bad vaddr size";
5068	return false;
5069	}
5070	}
5071	}
5072
5073	const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
5074	if (DppCt) {
5075	using namespace AMDGPU::DPP;
5076
5077	unsigned DC = DppCt->getImm();
5078	if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
5079	DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
5080	(DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
5081	(DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
5082	(DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
5083	(DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
5084	(DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
5085	ErrInfo = "Invalid dpp_ctrl value";
5086	return false;
5087	}
5088	if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
5089	ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
5090	ErrInfo = "Invalid dpp_ctrl value: "
5091	"wavefront shifts are not supported on GFX10+";
5092	return false;
5093	}
5094	if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
5095	ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
5096	ErrInfo = "Invalid dpp_ctrl value: "
5097	"broadcasts are not supported on GFX10+";
5098	return false;
5099	}
5100	if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
5101	ST.getGeneration() < AMDGPUSubtarget::GFX10) {
5102	if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
5103	DC <= DppCtrl::ROW_NEWBCAST_LAST &&
5104	!ST.hasGFX90AInsts()) {
5105	ErrInfo = "Invalid dpp_ctrl value: "
5106	"row_newbroadcast/row_share is not supported before "
5107	"GFX90A/GFX10";
5108	return false;
5109	} else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
5110	ErrInfo = "Invalid dpp_ctrl value: "
5111	"row_share and row_xmask are not supported before GFX10";
5112	return false;
5113	}
5114	}
5115
5116	if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
5117	!AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(Desc)) {
5118	ErrInfo = "Invalid dpp_ctrl value: "
5119	"DP ALU dpp only support row_newbcast";
5120	return false;
5121	}
5122	}
5123
5124	if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
5125	const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
5126	uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0
5127	: AMDGPU::OpName::vdata;
5128	const MachineOperand *Data = getNamedOperand(MI, OpName: DataNameIdx);
5129	const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);
5130	if (Data && !Data->isReg())
5131	Data = nullptr;
5132
5133	if (ST.hasGFX90AInsts()) {
5134	if (Dst && Data &&
5135	(RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) {
5136	ErrInfo = "Invalid register class: "
5137	"vdata and vdst should be both VGPR or AGPR";
5138	return false;
5139	}
5140	if (Data && Data2 &&
5141	(RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) {
5142	ErrInfo = "Invalid register class: "
5143	"both data operands should be VGPR or AGPR";
5144	return false;
5145	}
5146	} else {
5147	if ((Dst && RI.isAGPR(MRI, Dst->getReg())) ||
5148	(Data && RI.isAGPR(MRI, Data->getReg())) ||
5149	(Data2 && RI.isAGPR(MRI, Data2->getReg()))) {
5150	ErrInfo = "Invalid register class: "
5151	"agpr loads and stores not supported on this GPU";
5152	return false;
5153	}
5154	}
5155	}
5156
5157	if (ST.needsAlignedVGPRs()) {
5158	const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool {
5159	const MachineOperand *Op = getNamedOperand(MI, OpName);
5160	if (!Op)
5161	return true;
5162	Register Reg = Op->getReg();
5163	if (Reg.isPhysical())
5164	return !(RI.getHWRegIndex(Reg) & 1);
5165	const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
5166	return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
5167	!(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
5168	};
5169
5170	if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
5171	MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
5172	MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
5173
5174	if (!isAlignedReg(AMDGPU::OpName::data0)) {
5175	ErrInfo = "Subtarget requires even aligned vector registers "
5176	"for DS_GWS instructions";
5177	return false;
5178	}
5179	}
5180
5181	if (isMIMG(MI)) {
5182	if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
5183	ErrInfo = "Subtarget requires even aligned vector registers "
5184	"for vaddr operand of image instructions";
5185	return false;
5186	}
5187	}
5188	}
5189
5190	if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
5191	!ST.hasGFX90AInsts()) {
5192	const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);
5193	if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) {
5194	ErrInfo = "Invalid register class: "
5195	"v_accvgpr_write with an SGPR is not supported on this GPU";
5196	return false;
5197	}
5198	}
5199
5200	if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
5201	const MachineOperand &SrcOp = MI.getOperand(i: 1);
5202	if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
5203	ErrInfo = "pseudo expects only physical SGPRs";
5204	return false;
5205	}
5206	}
5207
5208	return true;
5209	}
5210
5211	// It is more readable to list mapped opcodes on the same line.
5212	// clang-format off
5213
5214	unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
5215	switch (MI.getOpcode()) {
5216	default: return AMDGPU::INSTRUCTION_LIST_END;
5217	case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
5218	case AMDGPU::COPY: return AMDGPU::COPY;
5219	case AMDGPU::PHI: return AMDGPU::PHI;
5220	case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
5221	case AMDGPU::WQM: return AMDGPU::WQM;
5222	case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
5223	case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
5224	case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
5225	case AMDGPU::S_MOV_B32: {
5226	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5227	return MI.getOperand(1).isReg() ||
5228	RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
5229	AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
5230	}
5231	case AMDGPU::S_ADD_I32:
5232	return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
5233	case AMDGPU::S_ADDC_U32:
5234	return AMDGPU::V_ADDC_U32_e32;
5235	case AMDGPU::S_SUB_I32:
5236	return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
5237	// FIXME: These are not consistently handled, and selected when the carry is
5238	// used.
5239	case AMDGPU::S_ADD_U32:
5240	return AMDGPU::V_ADD_CO_U32_e32;
5241	case AMDGPU::S_SUB_U32:
5242	return AMDGPU::V_SUB_CO_U32_e32;
5243	case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
5244	case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
5245	case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
5246	case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
5247	case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
5248	case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
5249	case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
5250	case AMDGPU::S_XNOR_B32:
5251	return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
5252	case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
5253	case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
5254	case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
5255	case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
5256	case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
5257	case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
5258	case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
5259	case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
5260	case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
5261	case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
5262	case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
5263	case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
5264	case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
5265	case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
5266	case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
5267	case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
5268	case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
5269	case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
5270	case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
5271	case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
5272	case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
5273	case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
5274	case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
5275	case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
5276	case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
5277	case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
5278	case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
5279	case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
5280	case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
5281	case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
5282	case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
5283	case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
5284	case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
5285	case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
5286	case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
5287	case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
5288	case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
5289	case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
5290	case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
5291	case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
5292	case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
5293	case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
5294	case AMDGPU::S_CVT_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64;
5295	case AMDGPU::S_CVT_HI_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64;
5296	case AMDGPU::S_CVT_F16_F32: return AMDGPU::V_CVT_F16_F32_t16_e64;
5297	case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
5298	case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
5299	case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
5300	case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
5301	case AMDGPU::S_CEIL_F16:
5302	return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
5303	: AMDGPU::V_CEIL_F16_fake16_e64;
5304	case AMDGPU::S_FLOOR_F16:
5305	return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
5306	: AMDGPU::V_FLOOR_F16_fake16_e64;
5307	case AMDGPU::S_TRUNC_F16:
5308	return AMDGPU::V_TRUNC_F16_fake16_e64;
5309	case AMDGPU::S_RNDNE_F16:
5310	return AMDGPU::V_RNDNE_F16_fake16_e64;
5311	case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
5312	case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
5313	case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
5314	case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
5315	case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
5316	case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
5317	case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
5318	case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64;
5319	case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64;
5320	case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64;
5321	case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64;
5322	case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64;
5323	case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64;
5324	case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64;
5325	case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
5326	case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
5327	case AMDGPU::S_FMAC_F16: return AMDGPU::V_FMAC_F16_t16_e64;
5328	case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
5329	case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
5330	case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
5331	case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
5332	case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
5333	case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
5334	case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
5335	case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
5336	case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
5337	case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
5338	case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
5339	case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
5340	case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
5341	case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
5342	case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
5343	case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
5344	case AMDGPU::S_CMP_LT_F16: return AMDGPU::V_CMP_LT_F16_t16_e64;
5345	case AMDGPU::S_CMP_EQ_F16: return AMDGPU::V_CMP_EQ_F16_t16_e64;
5346	case AMDGPU::S_CMP_LE_F16: return AMDGPU::V_CMP_LE_F16_t16_e64;
5347	case AMDGPU::S_CMP_GT_F16: return AMDGPU::V_CMP_GT_F16_t16_e64;
5348	case AMDGPU::S_CMP_LG_F16: return AMDGPU::V_CMP_LG_F16_t16_e64;
5349	case AMDGPU::S_CMP_GE_F16: return AMDGPU::V_CMP_GE_F16_t16_e64;
5350	case AMDGPU::S_CMP_O_F16: return AMDGPU::V_CMP_O_F16_t16_e64;
5351	case AMDGPU::S_CMP_U_F16: return AMDGPU::V_CMP_U_F16_t16_e64;
5352	case AMDGPU::S_CMP_NGE_F16: return AMDGPU::V_CMP_NGE_F16_t16_e64;
5353	case AMDGPU::S_CMP_NLG_F16: return AMDGPU::V_CMP_NLG_F16_t16_e64;
5354	case AMDGPU::S_CMP_NGT_F16: return AMDGPU::V_CMP_NGT_F16_t16_e64;
5355	case AMDGPU::S_CMP_NLE_F16: return AMDGPU::V_CMP_NLE_F16_t16_e64;
5356	case AMDGPU::S_CMP_NEQ_F16: return AMDGPU::V_CMP_NEQ_F16_t16_e64;
5357	case AMDGPU::S_CMP_NLT_F16: return AMDGPU::V_CMP_NLT_F16_t16_e64;
5358	case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
5359	case AMDGPU::V_S_EXP_F16_e64: return AMDGPU::V_EXP_F16_fake16_e64;
5360	case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
5361	case AMDGPU::V_S_LOG_F16_e64: return AMDGPU::V_LOG_F16_fake16_e64;
5362	case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
5363	case AMDGPU::V_S_RCP_F16_e64: return AMDGPU::V_RCP_F16_fake16_e64;
5364	case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
5365	case AMDGPU::V_S_RSQ_F16_e64: return AMDGPU::V_RSQ_F16_fake16_e64;
5366	case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
5367	case AMDGPU::V_S_SQRT_F16_e64: return AMDGPU::V_SQRT_F16_fake16_e64;
5368	}
5369	llvm_unreachable(
5370	"Unexpected scalar opcode without corresponding vector one!");
5371	}
5372
5373	// clang-format on
5374
5375	void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
5376	MachineBasicBlock &MBB,
5377	MachineBasicBlock::iterator MBBI,
5378	const DebugLoc &DL, Register Reg,
5379	bool IsSCCLive,
5380	SlotIndexes *Indexes) const {
5381	const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
5382	const SIInstrInfo *TII = ST.getInstrInfo();
5383	bool IsWave32 = ST.isWave32();
5384	if (IsSCCLive) {
5385	// Insert two move instructions, one to save the original value of EXEC and
5386	// the other to turn on all bits in EXEC. This is required as we can't use
5387	// the single instruction S_OR_SAVEEXEC that clobbers SCC.
5388	unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5389	MCRegister Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5390	auto StoreExecMI = BuildMI(MBB, MBBI, DL, TII->get(MovOpc), Reg)
5391	.addReg(Exec, RegState::Kill);
5392	auto FlipExecMI = BuildMI(MBB, MBBI, DL, TII->get(MovOpc), Exec).addImm(-1);
5393	if (Indexes) {
5394	Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
5395	Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
5396	}
5397	} else {
5398	const unsigned OrSaveExec =
5399	IsWave32 ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64;
5400	auto SaveExec =
5401	BuildMI(MBB, MBBI, DL, TII->get(OrSaveExec), Reg).addImm(-1);
5402	SaveExec->getOperand(3).setIsDead(); // Mark SCC as dead.
5403	if (Indexes)
5404	Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
5405	}
5406	}
5407
5408	void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
5409	MachineBasicBlock::iterator MBBI,
5410	const DebugLoc &DL, Register Reg,
5411	SlotIndexes *Indexes) const {
5412	unsigned ExecMov = isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5413	MCRegister Exec = isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5414	auto ExecRestoreMI =
5415	BuildMI(MBB, MBBI, DL, get(ExecMov), Exec).addReg(Reg, RegState::Kill);
5416	if (Indexes)
5417	Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
5418	}
5419
5420	static const TargetRegisterClass *
5421	adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI,
5422	const MachineRegisterInfo &MRI,
5423	const MCInstrDesc &TID, unsigned RCID,
5424	bool IsAllocatable) {
5425	if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
5426	(((TID.mayLoad() || TID.mayStore()) &&
5427	!(TID.TSFlags & SIInstrFlags::VGPRSpill)) ||
5428	(TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) {
5429	switch (RCID) {
5430	case AMDGPU::AV_32RegClassID:
5431	RCID = AMDGPU::VGPR_32RegClassID;
5432	break;
5433	case AMDGPU::AV_64RegClassID:
5434	RCID = AMDGPU::VReg_64RegClassID;
5435	break;
5436	case AMDGPU::AV_96RegClassID:
5437	RCID = AMDGPU::VReg_96RegClassID;
5438	break;
5439	case AMDGPU::AV_128RegClassID:
5440	RCID = AMDGPU::VReg_128RegClassID;
5441	break;
5442	case AMDGPU::AV_160RegClassID:
5443	RCID = AMDGPU::VReg_160RegClassID;
5444	break;
5445	case AMDGPU::AV_512RegClassID:
5446	RCID = AMDGPU::VReg_512RegClassID;
5447	break;
5448	default:
5449	break;
5450	}
5451	}
5452
5453	return RI.getProperlyAlignedRC(RI.getRegClass(RCID));
5454	}
5455
5456	const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID,
5457	unsigned OpNum, const TargetRegisterInfo *TRI,
5458	const MachineFunction &MF)
5459	const {
5460	if (OpNum >= TID.getNumOperands())
5461	return nullptr;
5462	auto RegClass = TID.operands()[OpNum].RegClass;
5463	bool IsAllocatable = false;
5464	if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) {
5465	// vdst and vdata should be both VGPR or AGPR, same for the DS instructions
5466	// with two data operands. Request register class constrained to VGPR only
5467	// of both operands present as Machine Copy Propagation can not check this
5468	// constraint and possibly other passes too.
5469	//
5470	// The check is limited to FLAT and DS because atomics in non-flat encoding
5471	// have their vdst and vdata tied to be the same register.
5472	const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
5473	AMDGPU::OpName::vdst);
5474	const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
5475	(TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
5476	: AMDGPU::OpName::vdata);
5477	if (DataIdx != -1) {
5478	IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand(
5479	TID.Opcode, AMDGPU::OpName::data1);
5480	}
5481	}
5482	return adjustAllocatableRegClass(ST, RI, MRI: MF.getRegInfo(), TID, RCID: RegClass,
5483	IsAllocatable);
5484	}
5485
5486	const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
5487	unsigned OpNo) const {
5488	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5489	const MCInstrDesc &Desc = get(MI.getOpcode());
5490	if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
5491	Desc.operands()[OpNo].RegClass == -1) {
5492	Register Reg = MI.getOperand(i: OpNo).getReg();
5493
5494	if (Reg.isVirtual())
5495	return MRI.getRegClass(Reg);
5496	return RI.getPhysRegBaseClass(Reg);
5497	}
5498
5499	unsigned RCID = Desc.operands()[OpNo].RegClass;
5500	return adjustAllocatableRegClass(ST, RI, MRI, TID: Desc, RCID, IsAllocatable: true);
5501	}
5502
5503	void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
5504	MachineBasicBlock::iterator I = MI;
5505	MachineBasicBlock *MBB = MI.getParent();
5506	MachineOperand &MO = MI.getOperand(i: OpIdx);
5507	MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
5508	unsigned RCID = get(MI.getOpcode()).operands()[OpIdx].RegClass;
5509	const TargetRegisterClass *RC = RI.getRegClass(RCID);
5510	unsigned Size = RI.getRegSizeInBits(*RC);
5511	unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
5512	if (MO.isReg())
5513	Opcode = AMDGPU::COPY;
5514	else if (RI.isSGPRClass(RC))
5515	Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
5516
5517	const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
5518	Register Reg = MRI.createVirtualRegister(RegClass: VRC);
5519	DebugLoc DL = MBB->findDebugLoc(MBBI: I);
5520	BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
5521	MO.ChangeToRegister(Reg, isDef: false);
5522	}
5523
5524	unsigned SIInstrInfo::buildExtractSubReg(
5525	MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
5526	const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
5527	unsigned SubIdx, const TargetRegisterClass *SubRC) const {
5528	MachineBasicBlock *MBB = MI->getParent();
5529	DebugLoc DL = MI->getDebugLoc();
5530	Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
5531
5532	if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
5533	BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
5534	.addReg(SuperReg.getReg(), 0, SubIdx);
5535	return SubReg;
5536	}
5537
5538	// Just in case the super register is itself a sub-register, copy it to a new
5539	// value so we don't need to worry about merging its subreg index with the
5540	// SubIdx passed to this function. The register coalescer should be able to
5541	// eliminate this extra copy.
5542	Register NewSuperReg = MRI.createVirtualRegister(RegClass: SuperRC);
5543
5544	BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
5545	.addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
5546
5547	BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
5548	.addReg(NewSuperReg, 0, SubIdx);
5549
5550	return SubReg;
5551	}
5552
5553	MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
5554	MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
5555	const MachineOperand &Op, const TargetRegisterClass *SuperRC,
5556	unsigned SubIdx, const TargetRegisterClass *SubRC) const {
5557	if (Op.isImm()) {
5558	if (SubIdx == AMDGPU::sub0)
5559	return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
5560	if (SubIdx == AMDGPU::sub1)
5561	return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
5562
5563	llvm_unreachable("Unhandled register index for immediate");
5564	}
5565
5566	unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
5567	SubIdx, SubRC);
5568	return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
5569	}
5570
5571	// Change the order of operands from (0, 1, 2) to (0, 2, 1)
5572	void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
5573	assert(Inst.getNumExplicitOperands() == 3);
5574	MachineOperand Op1 = Inst.getOperand(i: 1);
5575	Inst.removeOperand(OpNo: 1);
5576	Inst.addOperand(Op: Op1);
5577	}
5578
5579	bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
5580	const MCOperandInfo &OpInfo,
5581	const MachineOperand &MO) const {
5582	if (!MO.isReg())
5583	return false;
5584
5585	Register Reg = MO.getReg();
5586
5587	const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass);
5588	if (Reg.isPhysical())
5589	return DRC->contains(Reg);
5590
5591	const TargetRegisterClass *RC = MRI.getRegClass(Reg);
5592
5593	if (MO.getSubReg()) {
5594	const MachineFunction *MF = MO.getParent()->getParent()->getParent();
5595	const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF);
5596	if (!SuperRC)
5597	return false;
5598
5599	DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg());
5600	if (!DRC)
5601	return false;
5602	}
5603	return RC->hasSuperClassEq(RC: DRC);
5604	}
5605
5606	bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
5607	const MCOperandInfo &OpInfo,
5608	const MachineOperand &MO) const {
5609	if (MO.isReg())
5610	return isLegalRegOperand(MRI, OpInfo, MO);
5611
5612	// Handle non-register types that are treated like immediates.
5613	assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
5614	return true;
5615	}
5616
5617	bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
5618	const MachineOperand *MO) const {
5619	const MachineFunction &MF = *MI.getParent()->getParent();
5620	const MachineRegisterInfo &MRI = MF.getRegInfo();
5621	const MCInstrDesc &InstDesc = MI.getDesc();
5622	const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
5623	const TargetRegisterClass *DefinedRC =
5624	OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
5625	if (!MO)
5626	MO = &MI.getOperand(i: OpIdx);
5627
5628	int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
5629	int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
5630	if (isVALU(MI) && usesConstantBus(MRI, MO: *MO, OpInfo)) {
5631	if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo) && !LiteralLimit--)
5632	return false;
5633
5634	SmallDenseSet<RegSubRegPair> SGPRsUsed;
5635	if (MO->isReg())
5636	SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
5637
5638	for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
5639	if (i == OpIdx)
5640	continue;
5641	const MachineOperand &Op = MI.getOperand(i);
5642	if (Op.isReg()) {
5643	RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
5644	if (!SGPRsUsed.count(SGPR) &&
5645	// FIXME: This can access off the end of the operands() array.
5646	usesConstantBus(MRI, MO: Op, OpInfo: InstDesc.operands().begin()[i])) {
5647	if (--ConstantBusLimit <= 0)
5648	return false;
5649	SGPRsUsed.insert(SGPR);
5650	}
5651	} else if (AMDGPU::isSISrcOperand(Desc: InstDesc, OpNo: i) &&
5652	!isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
5653	if (!LiteralLimit--)
5654	return false;
5655	if (--ConstantBusLimit <= 0)
5656	return false;
5657	}
5658	}
5659	}
5660
5661	if (MO->isReg()) {
5662	if (!DefinedRC)
5663	return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
5664	if (!isLegalRegOperand(MRI, OpInfo, MO: *MO))
5665	return false;
5666	bool IsAGPR = RI.isAGPR(MRI, MO->getReg());
5667	if (IsAGPR && !ST.hasMAIInsts())
5668	return false;
5669	unsigned Opc = MI.getOpcode();
5670	if (IsAGPR &&
5671	(!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
5672	(MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
5673	return false;
5674	// Atomics should have both vdst and vdata either vgpr or agpr.
5675	const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
5676	const int DataIdx = AMDGPU::getNamedOperandIdx(Opc,
5677	isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
5678	if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
5679	MI.getOperand(i: DataIdx).isReg() &&
5680	RI.isAGPR(MRI, MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
5681	return false;
5682	if ((int)OpIdx == DataIdx) {
5683	if (VDstIdx != -1 &&
5684	RI.isAGPR(MRI, MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
5685	return false;
5686	// DS instructions with 2 src operands also must have tied RC.
5687	const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc,
5688	AMDGPU::OpName::data1);
5689	if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
5690	RI.isAGPR(MRI, MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
5691	return false;
5692	}
5693	if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
5694	(int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) &&
5695	RI.isSGPRReg(MRI, MO->getReg()))
5696	return false;
5697	return true;
5698	}
5699
5700	if (MO->isImm()) {
5701	uint64_t Imm = MO->getImm();
5702	bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64;
5703	bool Is64BitOp = Is64BitFPOp ||
5704	OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
5705	OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
5706	OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
5707	if (Is64BitOp &&
5708	!AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
5709	if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp))
5710	return false;
5711
5712	// FIXME: We can use sign extended 64-bit literals, but only for signed
5713	// operands. At the moment we do not know if an operand is signed.
5714	// Such operand will be encoded as its low 32 bits and then either
5715	// correctly sign extended or incorrectly zero extended by HW.
5716	if (!Is64BitFPOp && (int32_t)Imm < 0)
5717	return false;
5718	}
5719	}
5720
5721	// Handle non-register types that are treated like immediates.
5722	assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
5723
5724	if (!DefinedRC) {
5725	// This operand expects an immediate.
5726	return true;
5727	}
5728
5729	return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
5730	}
5731
5732	void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
5733	MachineInstr &MI) const {
5734	unsigned Opc = MI.getOpcode();
5735	const MCInstrDesc &InstrDesc = get(Opc);
5736
5737	int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
5738	MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5739
5740	int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
5741	MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5742
5743	// If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
5744	// we need to only have one constant bus use before GFX10.
5745	bool HasImplicitSGPR = findImplicitSGPRRead(MI);
5746	if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
5747	RI.isSGPRReg(MRI, Src0.getReg()))
5748	legalizeOpWithMove(MI, OpIdx: Src0Idx);
5749
5750	// Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
5751	// both the value to write (src0) and lane select (src1). Fix up non-SGPR
5752	// src0/src1 with V_READFIRSTLANE.
5753	if (Opc == AMDGPU::V_WRITELANE_B32) {
5754	const DebugLoc &DL = MI.getDebugLoc();
5755	if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
5756	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5757	BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
5758	.add(Src0);
5759	Src0.ChangeToRegister(Reg, isDef: false);
5760	}
5761	if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
5762	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5763	const DebugLoc &DL = MI.getDebugLoc();
5764	BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
5765	.add(Src1);
5766	Src1.ChangeToRegister(Reg, isDef: false);
5767	}
5768	return;
5769	}
5770
5771	// No VOP2 instructions support AGPRs.
5772	if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
5773	legalizeOpWithMove(MI, OpIdx: Src0Idx);
5774
5775	if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
5776	legalizeOpWithMove(MI, OpIdx: Src1Idx);
5777
5778	// Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
5779	if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
5780	int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
5781	if (!RI.isVGPR(MRI, MI.getOperand(i: Src2Idx).getReg()))
5782	legalizeOpWithMove(MI, OpIdx: Src2Idx);
5783	}
5784
5785	// VOP2 src0 instructions support all operand types, so we don't need to check
5786	// their legality. If src1 is already legal, we don't need to do anything.
5787	if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
5788	return;
5789
5790	// Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
5791	// lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
5792	// select is uniform.
5793	if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
5794	RI.isVGPR(MRI, Src1.getReg())) {
5795	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5796	const DebugLoc &DL = MI.getDebugLoc();
5797	BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
5798	.add(Src1);
5799	Src1.ChangeToRegister(Reg, isDef: false);
5800	return;
5801	}
5802
5803	// We do not use commuteInstruction here because it is too aggressive and will
5804	// commute if it is possible. We only want to commute here if it improves
5805	// legality. This can be called a fairly large number of times so don't waste
5806	// compile time pointlessly swapping and checking legality again.
5807	if (HasImplicitSGPR || !MI.isCommutable()) {
5808	legalizeOpWithMove(MI, OpIdx: Src1Idx);
5809	return;
5810	}
5811
5812	// If src0 can be used as src1, commuting will make the operands legal.
5813	// Otherwise we have to give up and insert a move.
5814	//
5815	// TODO: Other immediate-like operand kinds could be commuted if there was a
5816	// MachineOperand::ChangeTo* for them.
5817	if ((!Src1.isImm() && !Src1.isReg()) ||
5818	!isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
5819	legalizeOpWithMove(MI, OpIdx: Src1Idx);
5820	return;
5821	}
5822
5823	int CommutedOpc = commuteOpcode(MI);
5824	if (CommutedOpc == -1) {
5825	legalizeOpWithMove(MI, OpIdx: Src1Idx);
5826	return;
5827	}
5828
5829	MI.setDesc(get(CommutedOpc));
5830
5831	Register Src0Reg = Src0.getReg();
5832	unsigned Src0SubReg = Src0.getSubReg();
5833	bool Src0Kill = Src0.isKill();
5834
5835	if (Src1.isImm())
5836	Src0.ChangeToImmediate(ImmVal: Src1.getImm());
5837	else if (Src1.isReg()) {
5838	Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
5839	Src0.setSubReg(Src1.getSubReg());
5840	} else
5841	llvm_unreachable("Should only have register or immediate operands");
5842
5843	Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
5844	Src1.setSubReg(Src0SubReg);
5845	fixImplicitOperands(MI);
5846	}
5847
5848	// Legalize VOP3 operands. All operand types are supported for any operand
5849	// but only one literal constant and only starting from GFX10.
5850	void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
5851	MachineInstr &MI) const {
5852	unsigned Opc = MI.getOpcode();
5853
5854	int VOP3Idx[3] = {
5855	AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
5856	AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
5857	AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
5858	};
5859
5860	if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
5861	Opc == AMDGPU::V_PERMLANEX16_B32_e64) {
5862	// src1 and src2 must be scalar
5863	MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
5864	MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
5865	const DebugLoc &DL = MI.getDebugLoc();
5866	if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Reg: Src1.getReg()))) {
5867	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5868	BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
5869	.add(Src1);
5870	Src1.ChangeToRegister(Reg, isDef: false);
5871	}
5872	if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Reg: Src2.getReg()))) {
5873	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5874	BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
5875	.add(Src2);
5876	Src2.ChangeToRegister(Reg, isDef: false);
5877	}
5878	}
5879
5880	// Find the one SGPR operand we are allowed to use.
5881	int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
5882	int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
5883	SmallDenseSet<unsigned> SGPRsUsed;
5884	Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
5885	if (SGPRReg) {
5886	SGPRsUsed.insert(V: SGPRReg);
5887	--ConstantBusLimit;
5888	}
5889
5890	for (int Idx : VOP3Idx) {
5891	if (Idx == -1)
5892	break;
5893	MachineOperand &MO = MI.getOperand(i: Idx);
5894
5895	if (!MO.isReg()) {
5896	if (isInlineConstant(MO, get(Opc).operands()[Idx]))
5897	continue;
5898
5899	if (LiteralLimit > 0 && ConstantBusLimit > 0) {
5900	--LiteralLimit;
5901	--ConstantBusLimit;
5902	continue;
5903	}
5904
5905	--LiteralLimit;
5906	--ConstantBusLimit;
5907	legalizeOpWithMove(MI, OpIdx: Idx);
5908	continue;
5909	}
5910
5911	if (RI.hasAGPRs(RI.getRegClassForReg(MRI, MO.getReg())) &&
5912	!isOperandLegal(MI, OpIdx: Idx, MO: &MO)) {
5913	legalizeOpWithMove(MI, OpIdx: Idx);
5914	continue;
5915	}
5916
5917	if (!RI.isSGPRClass(RI.getRegClassForReg(MRI, MO.getReg())))
5918	continue; // VGPRs are legal
5919
5920	// We can use one SGPR in each VOP3 instruction prior to GFX10
5921	// and two starting from GFX10.
5922	if (SGPRsUsed.count(V: MO.getReg()))
5923	continue;
5924	if (ConstantBusLimit > 0) {
5925	SGPRsUsed.insert(V: MO.getReg());
5926	--ConstantBusLimit;
5927	continue;
5928	}
5929
5930	// If we make it this far, then the operand is not legal and we must
5931	// legalize it.
5932	legalizeOpWithMove(MI, OpIdx: Idx);
5933	}
5934
5935	// Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
5936	if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
5937	!RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg()))
5938	legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
5939	}
5940
5941	Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
5942	MachineRegisterInfo &MRI) const {
5943	const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
5944	const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
5945	Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
5946	unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
5947
5948	if (RI.hasAGPRs(VRC)) {
5949	VRC = RI.getEquivalentVGPRClass(VRC);
5950	Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
5951	BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
5952	get(TargetOpcode::COPY), NewSrcReg)
5953	.addReg(SrcReg);
5954	SrcReg = NewSrcReg;
5955	}
5956
5957	if (SubRegs == 1) {
5958	BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
5959	get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
5960	.addReg(SrcReg);
5961	return DstReg;
5962	}
5963
5964	SmallVector<Register, 8> SRegs;
5965	for (unsigned i = 0; i < SubRegs; ++i) {
5966	Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5967	BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
5968	get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
5969	.addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
5970	SRegs.push_back(Elt: SGPR);
5971	}
5972
5973	MachineInstrBuilder MIB =
5974	BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
5975	get(AMDGPU::REG_SEQUENCE), DstReg);
5976	for (unsigned i = 0; i < SubRegs; ++i) {
5977	MIB.addReg(RegNo: SRegs[i]);
5978	MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
5979	}
5980	return DstReg;
5981	}
5982
5983	void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
5984	MachineInstr &MI) const {
5985
5986	// If the pointer is store in VGPRs, then we need to move them to
5987	// SGPRs using v_readfirstlane. This is safe because we only select
5988	// loads with uniform pointers to SMRD instruction so we know the
5989	// pointer value is uniform.
5990	MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
5991	if (SBase && !RI.isSGPRClass(MRI.getRegClass(Reg: SBase->getReg()))) {
5992	Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
5993	SBase->setReg(SGPR);
5994	}
5995	MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soffset);
5996	if (SOff && !RI.isSGPRClass(MRI.getRegClass(Reg: SOff->getReg()))) {
5997	Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
5998	SOff->setReg(SGPR);
5999	}
6000	}
6001
6002	bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
6003	unsigned Opc = Inst.getOpcode();
6004	int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
6005	if (OldSAddrIdx < 0)
6006	return false;
6007
6008	assert(isSegmentSpecificFLAT(Inst));
6009
6010	int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
6011	if (NewOpc < 0)
6012	NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
6013	if (NewOpc < 0)
6014	return false;
6015
6016	MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
6017	MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
6018	if (RI.isSGPRReg(MRI, SAddr.getReg()))
6019	return false;
6020
6021	int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
6022	if (NewVAddrIdx < 0)
6023	return false;
6024
6025	int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
6026
6027	// Check vaddr, it shall be zero or absent.
6028	MachineInstr *VAddrDef = nullptr;
6029	if (OldVAddrIdx >= 0) {
6030	MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
6031	VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
6032	if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 ||
6033	!VAddrDef->getOperand(1).isImm() ||
6034	VAddrDef->getOperand(1).getImm() != 0)
6035	return false;
6036	}
6037
6038	const MCInstrDesc &NewDesc = get(NewOpc);
6039	Inst.setDesc(NewDesc);
6040
6041	// Callers expect iterator to be valid after this call, so modify the
6042	// instruction in place.
6043	if (OldVAddrIdx == NewVAddrIdx) {
6044	MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
6045	// Clear use list from the old vaddr holding a zero register.
6046	MRI.removeRegOperandFromUseList(MO: &NewVAddr);
6047	MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
6048	Inst.removeOperand(OpNo: OldSAddrIdx);
6049	// Update the use list with the pointer we have just moved from vaddr to
6050	// saddr position. Otherwise new vaddr will be missing from the use list.
6051	MRI.removeRegOperandFromUseList(MO: &NewVAddr);
6052	MRI.addRegOperandToUseList(MO: &NewVAddr);
6053	} else {
6054	assert(OldSAddrIdx == NewVAddrIdx);
6055
6056	if (OldVAddrIdx >= 0) {
6057	int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc,
6058	AMDGPU::OpName::vdst_in);
6059
6060	// removeOperand doesn't try to fixup tied operand indexes at it goes, so
6061	// it asserts. Untie the operands for now and retie them afterwards.
6062	if (NewVDstIn != -1) {
6063	int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in);
6064	Inst.untieRegOperand(OpIdx: OldVDstIn);
6065	}
6066
6067	Inst.removeOperand(OpNo: OldVAddrIdx);
6068
6069	if (NewVDstIn != -1) {
6070	int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);
6071	Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
6072	}
6073	}
6074	}
6075
6076	if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
6077	VAddrDef->eraseFromParent();
6078
6079	return true;
6080	}
6081
6082	// FIXME: Remove this when SelectionDAG is obsoleted.
6083	void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
6084	MachineInstr &MI) const {
6085	if (!isSegmentSpecificFLAT(MI))
6086	return;
6087
6088	// Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
6089	// thinks they are uniform, so a readfirstlane should be valid.
6090	MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr);
6091	if (!SAddr || RI.isSGPRClass(MRI.getRegClass(Reg: SAddr->getReg())))
6092	return;
6093
6094	if (moveFlatAddrToVGPR(Inst&: MI))
6095	return;
6096
6097	Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI);
6098	SAddr->setReg(ToSGPR);
6099	}
6100
6101	void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
6102	MachineBasicBlock::iterator I,
6103	const TargetRegisterClass *DstRC,
6104	MachineOperand &Op,
6105	MachineRegisterInfo &MRI,
6106	const DebugLoc &DL) const {
6107	Register OpReg = Op.getReg();
6108	unsigned OpSubReg = Op.getSubReg();
6109
6110	const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
6111	RI.getRegClassForReg(MRI, OpReg), OpSubReg);
6112
6113	// Check if operand is already the correct register class.
6114	if (DstRC == OpRC)
6115	return;
6116
6117	Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
6118	auto Copy = BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
6119
6120	Op.setReg(DstReg);
6121	Op.setSubReg(0);
6122
6123	MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
6124	if (!Def)
6125	return;
6126
6127	// Try to eliminate the copy if it is copying an immediate value.
6128	if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
6129	foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
6130
6131	bool ImpDef = Def->isImplicitDef();
6132	while (!ImpDef && Def && Def->isCopy()) {
6133	if (Def->getOperand(i: 1).getReg().isPhysical())
6134	break;
6135	Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
6136	ImpDef = Def && Def->isImplicitDef();
6137	}
6138	if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
6139	!ImpDef)
6140	Copy.addReg(AMDGPU::EXEC, RegState::Implicit);
6141	}
6142
6143	// Emit the actual waterfall loop, executing the wrapped instruction for each
6144	// unique value of \p ScalarOps across all lanes. In the best case we execute 1
6145	// iteration, in the worst case we execute 64 (once per lane).
6146	static void emitLoadScalarOpsFromVGPRLoop(
6147	const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB,
6148	MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
6149	ArrayRef<MachineOperand *> ScalarOps) {
6150	MachineFunction &MF = *OrigBB.getParent();
6151	const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6152	const SIRegisterInfo *TRI = ST.getRegisterInfo();
6153	unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6154	unsigned SaveExecOpc =
6155	ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
6156	unsigned XorTermOpc =
6157	ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
6158	unsigned AndOpc =
6159	ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6160	const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
6161
6162	MachineBasicBlock::iterator I = LoopBB.begin();
6163
6164	SmallVector<Register, 8> ReadlanePieces;
6165	Register CondReg;
6166
6167	for (MachineOperand *ScalarOp : ScalarOps) {
6168	unsigned RegSize = TRI->getRegSizeInBits(ScalarOp->getReg(), MRI);
6169	unsigned NumSubRegs = RegSize / 32;
6170	Register VScalarOp = ScalarOp->getReg();
6171
6172	if (NumSubRegs == 1) {
6173	Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6174
6175	BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurReg)
6176	.addReg(VScalarOp);
6177
6178	Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
6179
6180	BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U32_e64), NewCondReg)
6181	.addReg(CurReg)
6182	.addReg(VScalarOp);
6183
6184	// Combine the comparison results with AND.
6185	if (!CondReg) // First.
6186	CondReg = NewCondReg;
6187	else { // If not the first, we create an AND.
6188	Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
6189	BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg)
6190	.addReg(CondReg)
6191	.addReg(NewCondReg);
6192	CondReg = AndReg;
6193	}
6194
6195	// Update ScalarOp operand to use the SGPR ScalarOp.
6196	ScalarOp->setReg(CurReg);
6197	ScalarOp->setIsKill();
6198	} else {
6199	unsigned VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
6200	assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
6201	"Unhandled register size");
6202
6203	for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
6204	Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6205	Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6206
6207	// Read the next variant <- also loop target.
6208	BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
6209	.addReg(VScalarOp, VScalarOpUndef, TRI->getSubRegFromChannel(Idx));
6210
6211	// Read the next variant <- also loop target.
6212	BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
6213	.addReg(VScalarOp, VScalarOpUndef,
6214	TRI->getSubRegFromChannel(Idx + 1));
6215
6216	ReadlanePieces.push_back(Elt: CurRegLo);
6217	ReadlanePieces.push_back(Elt: CurRegHi);
6218
6219	// Comparison is to be done as 64-bit.
6220	Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
6221	BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
6222	.addReg(CurRegLo)
6223	.addImm(AMDGPU::sub0)
6224	.addReg(CurRegHi)
6225	.addImm(AMDGPU::sub1);
6226
6227	Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
6228	auto Cmp = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64),
6229	NewCondReg)
6230	.addReg(CurReg);
6231	if (NumSubRegs <= 2)
6232	Cmp.addReg(VScalarOp);
6233	else
6234	Cmp.addReg(VScalarOp, VScalarOpUndef,
6235	TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2));
6236
6237	// Combine the comparison results with AND.
6238	if (!CondReg) // First.
6239	CondReg = NewCondReg;
6240	else { // If not the first, we create an AND.
6241	Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
6242	BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg)
6243	.addReg(CondReg)
6244	.addReg(NewCondReg);
6245	CondReg = AndReg;
6246	}
6247	} // End for loop.
6248
6249	auto SScalarOpRC =
6250	TRI->getEquivalentSGPRClass(MRI.getRegClass(Reg: VScalarOp));
6251	Register SScalarOp = MRI.createVirtualRegister(SScalarOpRC);
6252
6253	// Build scalar ScalarOp.
6254	auto Merge =
6255	BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SScalarOp);
6256	unsigned Channel = 0;
6257	for (Register Piece : ReadlanePieces) {
6258	Merge.addReg(Piece).addImm(TRI->getSubRegFromChannel(Channel: Channel++));
6259	}
6260
6261	// Update ScalarOp operand to use the SGPR ScalarOp.
6262	ScalarOp->setReg(SScalarOp);
6263	ScalarOp->setIsKill();
6264	}
6265	}
6266
6267	Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
6268	MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
6269
6270	// Update EXEC to matching lanes, saving original to SaveExec.
6271	BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
6272	.addReg(CondReg, RegState::Kill);
6273
6274	// The original instruction is here; we insert the terminators after it.
6275	I = BodyBB.end();
6276
6277	// Update EXEC, switch all done bits to 0 and all todo bits to 1.
6278	BuildMI(BodyBB, I, DL, TII.get(XorTermOpc), Exec)
6279	.addReg(Exec)
6280	.addReg(SaveExec);
6281
6282	BuildMI(BodyBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB);
6283	}
6284
6285	// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
6286	// with SGPRs by iterating over all unique values across all lanes.
6287	// Returns the loop basic block that now contains \p MI.
6288	static MachineBasicBlock *
6289	loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
6290	ArrayRef<MachineOperand *> ScalarOps,
6291	MachineDominatorTree *MDT,
6292	MachineBasicBlock::iterator Begin = nullptr,
6293	MachineBasicBlock::iterator End = nullptr) {
6294	MachineBasicBlock &MBB = *MI.getParent();
6295	MachineFunction &MF = *MBB.getParent();
6296	const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6297	const SIRegisterInfo *TRI = ST.getRegisterInfo();
6298	MachineRegisterInfo &MRI = MF.getRegInfo();
6299	if (!Begin.isValid())
6300	Begin = &MI;
6301	if (!End.isValid()) {
6302	End = &MI;
6303	++End;
6304	}
6305	const DebugLoc &DL = MI.getDebugLoc();
6306	unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6307	unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6308	const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
6309
6310	// Save SCC. Waterfall Loop may overwrite SCC.
6311	Register SaveSCCReg;
6312	bool SCCNotDead = (MBB.computeRegisterLiveness(TRI, AMDGPU::SCC, MI, 30) !=
6313	MachineBasicBlock::LQR_Dead);
6314	if (SCCNotDead) {
6315	SaveSCCReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6316	BuildMI(MBB, Begin, DL, TII.get(AMDGPU::S_CSELECT_B32), SaveSCCReg)
6317	.addImm(1)
6318	.addImm(0);
6319	}
6320
6321	Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
6322
6323	// Save the EXEC mask
6324	BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
6325
6326	// Killed uses in the instruction we are waterfalling around will be
6327	// incorrect due to the added control-flow.
6328	MachineBasicBlock::iterator AfterMI = MI;
6329	++AfterMI;
6330	for (auto I = Begin; I != AfterMI; I++) {
6331	for (auto &MO : I->all_uses())
6332	MRI.clearKillFlags(Reg: MO.getReg());
6333	}
6334
6335	// To insert the loop we need to split the block. Move everything after this
6336	// point to a new block, and insert a new empty block between the two.
6337	MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
6338	MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
6339	MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
6340	MachineFunction::iterator MBBI(MBB);
6341	++MBBI;
6342
6343	MF.insert(MBBI, MBB: LoopBB);
6344	MF.insert(MBBI, MBB: BodyBB);
6345	MF.insert(MBBI, MBB: RemainderBB);
6346
6347	LoopBB->addSuccessor(Succ: BodyBB);
6348	BodyBB->addSuccessor(Succ: LoopBB);
6349	BodyBB->addSuccessor(Succ: RemainderBB);
6350
6351	// Move Begin to MI to the BodyBB, and the remainder of the block to
6352	// RemainderBB.
6353	RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
6354	RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
6355	BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
6356
6357	MBB.addSuccessor(Succ: LoopBB);
6358
6359	// Update dominators. We know that MBB immediately dominates LoopBB, that
6360	// LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
6361	// RemainderBB. RemainderBB immediately dominates all of the successors
6362	// transferred to it from MBB that MBB used to properly dominate.
6363	if (MDT) {
6364	MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
6365	MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
6366	MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
6367	for (auto &Succ : RemainderBB->successors()) {
6368	if (MDT->properlyDominates(A: &MBB, B: Succ)) {
6369	MDT->changeImmediateDominator(N: Succ, NewIDom: RemainderBB);
6370	}
6371	}
6372	}
6373
6374	emitLoadScalarOpsFromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps);
6375
6376	MachineBasicBlock::iterator First = RemainderBB->begin();
6377	// Restore SCC
6378	if (SCCNotDead) {
6379	BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_CMP_LG_U32))
6380	.addReg(SaveSCCReg, RegState::Kill)
6381	.addImm(0);
6382	}
6383
6384	// Restore the EXEC mask
6385	BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
6386	return BodyBB;
6387	}
6388
6389	// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
6390	static std::tuple<unsigned, unsigned>
6391	extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
6392	MachineBasicBlock &MBB = *MI.getParent();
6393	MachineFunction &MF = *MBB.getParent();
6394	MachineRegisterInfo &MRI = MF.getRegInfo();
6395
6396	// Extract the ptr from the resource descriptor.
6397	unsigned RsrcPtr =
6398	TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
6399	AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
6400
6401	// Create an empty resource descriptor
6402	Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6403	Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6404	Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
6405	Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
6406	uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
6407
6408	// Zero64 = 0
6409	BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
6410	.addImm(0);
6411
6412	// SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
6413	BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
6414	.addImm(RsrcDataFormat & 0xFFFFFFFF);
6415
6416	// SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
6417	BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
6418	.addImm(RsrcDataFormat >> 32);
6419
6420	// NewSRsrc = {Zero64, SRsrcFormat}
6421	BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
6422	.addReg(Zero64)
6423	.addImm(AMDGPU::sub0_sub1)
6424	.addReg(SRsrcFormatLo)
6425	.addImm(AMDGPU::sub2)
6426	.addReg(SRsrcFormatHi)
6427	.addImm(AMDGPU::sub3);
6428
6429	return std::tuple(RsrcPtr, NewSRsrc);
6430	}
6431
6432	MachineBasicBlock *
6433	SIInstrInfo::legalizeOperands(MachineInstr &MI,
6434	MachineDominatorTree *MDT) const {
6435	MachineFunction &MF = *MI.getParent()->getParent();
6436	MachineRegisterInfo &MRI = MF.getRegInfo();
6437	MachineBasicBlock *CreatedBB = nullptr;
6438
6439	// Legalize VOP2
6440	if (isVOP2(MI) || isVOPC(MI)) {
6441	legalizeOperandsVOP2(MRI, MI);
6442	return CreatedBB;
6443	}
6444
6445	// Legalize VOP3
6446	if (isVOP3(MI)) {
6447	legalizeOperandsVOP3(MRI, MI);
6448	return CreatedBB;
6449	}
6450
6451	// Legalize SMRD
6452	if (isSMRD(MI)) {
6453	legalizeOperandsSMRD(MRI, MI);
6454	return CreatedBB;
6455	}
6456
6457	// Legalize FLAT
6458	if (isFLAT(MI)) {
6459	legalizeOperandsFLAT(MRI, MI);
6460	return CreatedBB;
6461	}
6462
6463	// Legalize REG_SEQUENCE and PHI
6464	// The register class of the operands much be the same type as the register
6465	// class of the output.
6466	if (MI.getOpcode() == AMDGPU::PHI) {
6467	const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
6468	for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
6469	if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual())
6470	continue;
6471	const TargetRegisterClass *OpRC =
6472	MRI.getRegClass(Reg: MI.getOperand(i).getReg());
6473	if (RI.hasVectorRegisters(OpRC)) {
6474	VRC = OpRC;
6475	} else {
6476	SRC = OpRC;
6477	}
6478	}
6479
6480	// If any of the operands are VGPR registers, then they all most be
6481	// otherwise we will create illegal VGPR->SGPR copies when legalizing
6482	// them.
6483	if (VRC || !RI.isSGPRClass(getOpRegClass(MI, OpNo: 0))) {
6484	if (!VRC) {
6485	assert(SRC);
6486	if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) {
6487	VRC = &AMDGPU::VReg_1RegClass;
6488	} else
6489	VRC = RI.isAGPRClass(getOpRegClass(MI, OpNo: 0))
6490	? RI.getEquivalentAGPRClass(SRC)
6491	: RI.getEquivalentVGPRClass(SRC);
6492	} else {
6493	VRC = RI.isAGPRClass(getOpRegClass(MI, OpNo: 0))
6494	? RI.getEquivalentAGPRClass(VRC)
6495	: RI.getEquivalentVGPRClass(VRC);
6496	}
6497	RC = VRC;
6498	} else {
6499	RC = SRC;
6500	}
6501
6502	// Update all the operands so they have the same type.
6503	for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6504	MachineOperand &Op = MI.getOperand(i: I);
6505	if (!Op.isReg() || !Op.getReg().isVirtual())
6506	continue;
6507
6508	// MI is a PHI instruction.
6509	MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
6510	MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
6511
6512	// Avoid creating no-op copies with the same src and dst reg class. These
6513	// confuse some of the machine passes.
6514	legalizeGenericOperand(InsertMBB&: *InsertBB, I: Insert, DstRC: RC, Op, MRI, DL: MI.getDebugLoc());
6515	}
6516	}
6517
6518	// REG_SEQUENCE doesn't really require operand legalization, but if one has a
6519	// VGPR dest type and SGPR sources, insert copies so all operands are
6520	// VGPRs. This seems to help operand folding / the register coalescer.
6521	if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
6522	MachineBasicBlock *MBB = MI.getParent();
6523	const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
6524	if (RI.hasVGPRs(DstRC)) {
6525	// Update all the operands so they are VGPR register classes. These may
6526	// not be the same register class because REG_SEQUENCE supports mixing
6527	// subregister index types e.g. sub0_sub1 + sub2 + sub3
6528	for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6529	MachineOperand &Op = MI.getOperand(i: I);
6530	if (!Op.isReg() || !Op.getReg().isVirtual())
6531	continue;
6532
6533	const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
6534	const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
6535	if (VRC == OpRC)
6536	continue;
6537
6538	legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
6539	Op.setIsKill();
6540	}
6541	}
6542
6543	return CreatedBB;
6544	}
6545
6546	// Legalize INSERT_SUBREG
6547	// src0 must have the same register class as dst
6548	if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
6549	Register Dst = MI.getOperand(i: 0).getReg();
6550	Register Src0 = MI.getOperand(i: 1).getReg();
6551	const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
6552	const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
6553	if (DstRC != Src0RC) {
6554	MachineBasicBlock *MBB = MI.getParent();
6555	MachineOperand &Op = MI.getOperand(i: 1);
6556	legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
6557	}
6558	return CreatedBB;
6559	}
6560
6561	// Legalize SI_INIT_M0
6562	if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
6563	MachineOperand &Src = MI.getOperand(i: 0);
6564	if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Reg: Src.getReg())))
6565	Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
6566	return CreatedBB;
6567	}
6568
6569	// Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
6570	if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
6571	MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
6572	MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
6573	MI.getOpcode() == AMDGPU::S_WQM_B32 ||
6574	MI.getOpcode() == AMDGPU::S_WQM_B64) {
6575	MachineOperand &Src = MI.getOperand(i: 1);
6576	if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Reg: Src.getReg())))
6577	Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
6578	return CreatedBB;
6579	}
6580
6581	// Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
6582	//
6583	// Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
6584	// scratch memory access. In both cases, the legalization never involves
6585	// conversion to the addr64 form.
6586	if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
6587	(isMUBUF(MI) || isMTBUF(MI)))) {
6588	int RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI)) ? AMDGPU::OpName::rsrc
6589	: AMDGPU::OpName::srsrc;
6590	MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
6591	if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(Reg: SRsrc->getReg())))
6592	CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
6593
6594	int SampOpName = isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
6595	MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
6596	if (SSamp && !RI.isSGPRClass(MRI.getRegClass(Reg: SSamp->getReg())))
6597	CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {SSamp}, MDT);
6598
6599	return CreatedBB;
6600	}
6601
6602	// Legalize SI_CALL
6603	if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
6604	MachineOperand *Dest = &MI.getOperand(i: 0);
6605	if (!RI.isSGPRClass(MRI.getRegClass(Reg: Dest->getReg()))) {
6606	// Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
6607	// following copies, we also need to move copies from and to physical
6608	// registers into the loop block.
6609	unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
6610	unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
6611
6612	// Also move the copies to physical registers into the loop block
6613	MachineBasicBlock &MBB = *MI.getParent();
6614	MachineBasicBlock::iterator Start(&MI);
6615	while (Start->getOpcode() != FrameSetupOpcode)
6616	--Start;
6617	MachineBasicBlock::iterator End(&MI);
6618	while (End->getOpcode() != FrameDestroyOpcode)
6619	++End;
6620	// Also include following copies of the return value
6621	++End;
6622	while (End != MBB.end() && End->isCopy() && End->getOperand(i: 1).isReg() &&
6623	MI.definesRegister(Reg: End->getOperand(i: 1).getReg()))
6624	++End;
6625	CreatedBB =
6626	loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Dest}, MDT, Begin: Start, End);
6627	}
6628	}
6629
6630	// Legalize s_sleep_var.
6631	if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
6632	const DebugLoc &DL = MI.getDebugLoc();
6633	Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6634	int Src0Idx =
6635	AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
6636	MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6637	BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
6638	.add(Src0);
6639	Src0.ChangeToRegister(Reg, isDef: false);
6640	return nullptr;
6641	}
6642
6643	// Legalize MUBUF instructions.
6644	bool isSoffsetLegal = true;
6645	int SoffsetIdx =
6646	AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::soffset);
6647	if (SoffsetIdx != -1) {
6648	MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
6649	if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
6650	!RI.isSGPRClass(MRI.getRegClass(Reg: Soffset->getReg()))) {
6651	isSoffsetLegal = false;
6652	}
6653	}
6654
6655	bool isRsrcLegal = true;
6656	int RsrcIdx =
6657	AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
6658	if (RsrcIdx != -1) {
6659	MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
6660	if (Rsrc->isReg() && !RI.isSGPRClass(MRI.getRegClass(Reg: Rsrc->getReg()))) {
6661	isRsrcLegal = false;
6662	}
6663	}
6664
6665	// The operands are legal.
6666	if (isRsrcLegal && isSoffsetLegal)
6667	return CreatedBB;
6668
6669	if (!isRsrcLegal) {
6670	// Legalize a VGPR Rsrc
6671	//
6672	// If the instruction is _ADDR64, we can avoid a waterfall by extracting
6673	// the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
6674	// a zero-value SRsrc.
6675	//
6676	// If the instruction is _OFFSET (both idxen and offen disabled), and we
6677	// support ADDR64 instructions, we can convert to ADDR64 and do the same as
6678	// above.
6679	//
6680	// Otherwise we are on non-ADDR64 hardware, and/or we have
6681	// idxen/offen/bothen and we fall back to a waterfall loop.
6682
6683	MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
6684	MachineBasicBlock &MBB = *MI.getParent();
6685
6686	MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
6687	if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
6688	// This is already an ADDR64 instruction so we need to add the pointer
6689	// extracted from the resource descriptor to the current value of VAddr.
6690	Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6691	Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6692	Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
6693
6694	const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
6695	Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
6696	Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
6697
6698	unsigned RsrcPtr, NewSRsrc;
6699	std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
6700
6701	// NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
6702	const DebugLoc &DL = MI.getDebugLoc();
6703	BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo)
6704	.addDef(CondReg0)
6705	.addReg(RsrcPtr, 0, AMDGPU::sub0)
6706	.addReg(VAddr->getReg(), 0, AMDGPU::sub0)
6707	.addImm(0);
6708
6709	// NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
6710	BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
6711	.addDef(CondReg1, RegState::Dead)
6712	.addReg(RsrcPtr, 0, AMDGPU::sub1)
6713	.addReg(VAddr->getReg(), 0, AMDGPU::sub1)
6714	.addReg(CondReg0, RegState::Kill)
6715	.addImm(0);
6716
6717	// NewVaddr = {NewVaddrHi, NewVaddrLo}
6718	BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
6719	.addReg(NewVAddrLo)
6720	.addImm(AMDGPU::sub0)
6721	.addReg(NewVAddrHi)
6722	.addImm(AMDGPU::sub1);
6723
6724	VAddr->setReg(NewVAddr);
6725	Rsrc->setReg(NewSRsrc);
6726	} else if (!VAddr && ST.hasAddr64()) {
6727	// This instructions is the _OFFSET variant, so we need to convert it to
6728	// ADDR64.
6729	assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
6730	"FIXME: Need to emit flat atomics here");
6731
6732	unsigned RsrcPtr, NewSRsrc;
6733	std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
6734
6735	Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
6736	MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
6737	MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
6738	MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
6739	unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
6740
6741	// Atomics with return have an additional tied operand and are
6742	// missing some of the special bits.
6743	MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
6744	MachineInstr *Addr64;
6745
6746	if (!VDataIn) {
6747	// Regular buffer load / store.
6748	MachineInstrBuilder MIB =
6749	BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
6750	.add(*VData)
6751	.addReg(NewVAddr)
6752	.addReg(NewSRsrc)
6753	.add(*SOffset)
6754	.add(*Offset);
6755
6756	if (const MachineOperand *CPol =
6757	getNamedOperand(MI, AMDGPU::OpName::cpol)) {
6758	MIB.addImm(Val: CPol->getImm());
6759	}
6760
6761	if (const MachineOperand *TFE =
6762	getNamedOperand(MI, AMDGPU::OpName::tfe)) {
6763	MIB.addImm(Val: TFE->getImm());
6764	}
6765
6766	MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
6767
6768	MIB.cloneMemRefs(OtherMI: MI);
6769	Addr64 = MIB;
6770	} else {
6771	// Atomics with return.
6772	Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
6773	.add(*VData)
6774	.add(*VDataIn)
6775	.addReg(NewVAddr)
6776	.addReg(NewSRsrc)
6777	.add(*SOffset)
6778	.add(*Offset)
6779	.addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol))
6780	.cloneMemRefs(MI);
6781	}
6782
6783	MI.removeFromParent();
6784
6785	// NewVaddr = {NewVaddrHi, NewVaddrLo}
6786	BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
6787	NewVAddr)
6788	.addReg(RsrcPtr, 0, AMDGPU::sub0)
6789	.addImm(AMDGPU::sub0)
6790	.addReg(RsrcPtr, 0, AMDGPU::sub1)
6791	.addImm(AMDGPU::sub1);
6792	} else {
6793	// Legalize a VGPR Rsrc and soffset together.
6794	if (!isSoffsetLegal) {
6795	MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
6796	CreatedBB =
6797	loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
6798	return CreatedBB;
6799	}
6800	CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
6801	return CreatedBB;
6802	}
6803	}
6804
6805	// Legalize a VGPR soffset.
6806	if (!isSoffsetLegal) {
6807	MachineOperand *Soffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
6808	CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Soffset}, MDT);
6809	return CreatedBB;
6810	}
6811	return CreatedBB;
6812	}
6813
6814	void SIInstrWorklist::insert(MachineInstr *MI) {
6815	InstrList.insert(MI);
6816	// Add MBUF instructiosn to deferred list.
6817	int RsrcIdx =
6818	AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc);
6819	if (RsrcIdx != -1) {
6820	DeferredList.insert(MI);
6821	}
6822	}
6823
6824	bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
6825	return DeferredList.contains(MI);
6826	}
6827
6828	void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
6829	MachineDominatorTree *MDT) const {
6830
6831	while (!Worklist.empty()) {
6832	MachineInstr &Inst = *Worklist.top();
6833	Worklist.erase_top();
6834	// Skip MachineInstr in the deferred list.
6835	if (Worklist.isDeferred(MI: &Inst))
6836	continue;
6837	moveToVALUImpl(Worklist, MDT, Inst);
6838	}
6839
6840	// Deferred list of instructions will be processed once
6841	// all the MachineInstr in the worklist are done.
6842	for (MachineInstr *Inst : Worklist.getDeferredList()) {
6843	moveToVALUImpl(Worklist, MDT, *Inst);
6844	assert(Worklist.empty() &&
6845	"Deferred MachineInstr are not supposed to re-populate worklist");
6846	}
6847	}
6848
6849	void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
6850	MachineDominatorTree *MDT,
6851	MachineInstr &Inst) const {
6852
6853	MachineBasicBlock *MBB = Inst.getParent();
6854	if (!MBB)
6855	return;
6856	MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6857	unsigned Opcode = Inst.getOpcode();
6858	unsigned NewOpcode = getVALUOp(MI: Inst);
6859	// Handle some special cases
6860	switch (Opcode) {
6861	default:
6862	break;
6863	case AMDGPU::S_ADD_U64_PSEUDO:
6864	NewOpcode = AMDGPU::V_ADD_U64_PSEUDO;
6865	break;
6866	case AMDGPU::S_SUB_U64_PSEUDO:
6867	NewOpcode = AMDGPU::V_SUB_U64_PSEUDO;
6868	break;
6869	case AMDGPU::S_ADD_I32:
6870	case AMDGPU::S_SUB_I32: {
6871	// FIXME: The u32 versions currently selected use the carry.
6872	bool Changed;
6873	MachineBasicBlock *CreatedBBTmp = nullptr;
6874	std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
6875	if (Changed)
6876	return;
6877
6878	// Default handling
6879	break;
6880	}
6881
6882	case AMDGPU::S_MUL_U64:
6883	// Split s_mul_u64 in 32-bit vector multiplications.
6884	splitScalarSMulU64(Worklist, Inst, MDT);
6885	Inst.eraseFromParent();
6886	return;
6887
6888	case AMDGPU::S_MUL_U64_U32_PSEUDO:
6889	case AMDGPU::S_MUL_I64_I32_PSEUDO:
6890	// This is a special case of s_mul_u64 where all the operands are either
6891	// zero extended or sign extended.
6892	splitScalarSMulPseudo(Worklist, Inst, MDT);
6893	Inst.eraseFromParent();
6894	return;
6895
6896	case AMDGPU::S_AND_B64:
6897	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
6898	Inst.eraseFromParent();
6899	return;
6900
6901	case AMDGPU::S_OR_B64:
6902	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
6903	Inst.eraseFromParent();
6904	return;
6905
6906	case AMDGPU::S_XOR_B64:
6907	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
6908	Inst.eraseFromParent();
6909	return;
6910
6911	case AMDGPU::S_NAND_B64:
6912	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
6913	Inst.eraseFromParent();
6914	return;
6915
6916	case AMDGPU::S_NOR_B64:
6917	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
6918	Inst.eraseFromParent();
6919	return;
6920
6921	case AMDGPU::S_XNOR_B64:
6922	if (ST.hasDLInsts())
6923	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
6924	else
6925	splitScalar64BitXnor(Worklist, Inst, MDT);
6926	Inst.eraseFromParent();
6927	return;
6928
6929	case AMDGPU::S_ANDN2_B64:
6930	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
6931	Inst.eraseFromParent();
6932	return;
6933
6934	case AMDGPU::S_ORN2_B64:
6935	splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
6936	Inst.eraseFromParent();
6937	return;
6938
6939	case AMDGPU::S_BREV_B64:
6940	splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true);
6941	Inst.eraseFromParent();
6942	return;
6943
6944	case AMDGPU::S_NOT_B64:
6945	splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
6946	Inst.eraseFromParent();
6947	return;
6948
6949	case AMDGPU::S_BCNT1_I32_B64:
6950	splitScalar64BitBCNT(Worklist, Inst);
6951	Inst.eraseFromParent();
6952	return;
6953
6954	case AMDGPU::S_BFE_I64:
6955	splitScalar64BitBFE(Worklist, Inst);
6956	Inst.eraseFromParent();
6957	return;
6958
6959	case AMDGPU::S_FLBIT_I32_B64:
6960	splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBH_U32_e32);
6961	Inst.eraseFromParent();
6962	return;
6963	case AMDGPU::S_FF1_I32_B64:
6964	splitScalar64BitCountOp(Worklist, Inst, AMDGPU::V_FFBL_B32_e32);
6965	Inst.eraseFromParent();
6966	return;
6967
6968	case AMDGPU::S_LSHL_B32:
6969	if (ST.hasOnlyRevVALUShifts()) {
6970	NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
6971	swapOperands(Inst);
6972	}
6973	break;
6974	case AMDGPU::S_ASHR_I32:
6975	if (ST.hasOnlyRevVALUShifts()) {
6976	NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
6977	swapOperands(Inst);
6978	}
6979	break;
6980	case AMDGPU::S_LSHR_B32:
6981	if (ST.hasOnlyRevVALUShifts()) {
6982	NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
6983	swapOperands(Inst);
6984	}
6985	break;
6986	case AMDGPU::S_LSHL_B64:
6987	if (ST.hasOnlyRevVALUShifts()) {
6988	NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
6989	? AMDGPU::V_LSHLREV_B64_pseudo_e64
6990	: AMDGPU::V_LSHLREV_B64_e64;
6991	swapOperands(Inst);
6992	}
6993	break;
6994	case AMDGPU::S_ASHR_I64:
6995	if (ST.hasOnlyRevVALUShifts()) {
6996	NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
6997	swapOperands(Inst);
6998	}
6999	break;
7000	case AMDGPU::S_LSHR_B64:
7001	if (ST.hasOnlyRevVALUShifts()) {
7002	NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
7003	swapOperands(Inst);
7004	}
7005	break;
7006
7007	case AMDGPU::S_ABS_I32:
7008	lowerScalarAbs(Worklist, Inst);
7009	Inst.eraseFromParent();
7010	return;
7011
7012	case AMDGPU::S_CBRANCH_SCC0:
7013	case AMDGPU::S_CBRANCH_SCC1: {
7014	// Clear unused bits of vcc
7015	Register CondReg = Inst.getOperand(i: 1).getReg();
7016	bool IsSCC = CondReg == AMDGPU::SCC;
7017	Register VCC = RI.getVCC();
7018	Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
7019	unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
7020	BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(Opc), VCC)
7021	.addReg(EXEC)
7022	.addReg(IsSCC ? VCC : CondReg);
7023	Inst.removeOperand(OpNo: 1);
7024	} break;
7025
7026	case AMDGPU::S_BFE_U64:
7027	case AMDGPU::S_BFM_B64:
7028	llvm_unreachable("Moving this op to VALU not implemented");
7029
7030	case AMDGPU::S_PACK_LL_B32_B16:
7031	case AMDGPU::S_PACK_LH_B32_B16:
7032	case AMDGPU::S_PACK_HL_B32_B16:
7033	case AMDGPU::S_PACK_HH_B32_B16:
7034	movePackToVALU(Worklist, MRI, Inst);
7035	Inst.eraseFromParent();
7036	return;
7037
7038	case AMDGPU::S_XNOR_B32:
7039	lowerScalarXnor(Worklist, Inst);
7040	Inst.eraseFromParent();
7041	return;
7042
7043	case AMDGPU::S_NAND_B32:
7044	splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
7045	Inst.eraseFromParent();
7046	return;
7047
7048	case AMDGPU::S_NOR_B32:
7049	splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
7050	Inst.eraseFromParent();
7051	return;
7052
7053	case AMDGPU::S_ANDN2_B32:
7054	splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
7055	Inst.eraseFromParent();
7056	return;
7057
7058	case AMDGPU::S_ORN2_B32:
7059	splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
7060	Inst.eraseFromParent();
7061	return;
7062
7063	// TODO: remove as soon as everything is ready
7064	// to replace VGPR to SGPR copy with V_READFIRSTLANEs.
7065	// S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
7066	// can only be selected from the uniform SDNode.
7067	case AMDGPU::S_ADD_CO_PSEUDO:
7068	case AMDGPU::S_SUB_CO_PSEUDO: {
7069	unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
7070	? AMDGPU::V_ADDC_U32_e64
7071	: AMDGPU::V_SUBB_U32_e64;
7072	const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
7073
7074	Register CarryInReg = Inst.getOperand(i: 4).getReg();
7075	if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
7076	Register NewCarryReg = MRI.createVirtualRegister(CarryRC);
7077	BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg)
7078	.addReg(CarryInReg);
7079	}
7080
7081	Register CarryOutReg = Inst.getOperand(i: 1).getReg();
7082
7083	Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(
7084	MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
7085	MachineInstr *CarryOp =
7086	BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg)
7087	.addReg(CarryOutReg, RegState::Define)
7088	.add(Inst.getOperand(i: 2))
7089	.add(Inst.getOperand(i: 3))
7090	.addReg(CarryInReg)
7091	.addImm(0);
7092	legalizeOperands(MI&: *CarryOp);
7093	MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
7094	addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
7095	Inst.eraseFromParent();
7096	}
7097	return;
7098	case AMDGPU::S_UADDO_PSEUDO:
7099	case AMDGPU::S_USUBO_PSEUDO: {
7100	const DebugLoc &DL = Inst.getDebugLoc();
7101	MachineOperand &Dest0 = Inst.getOperand(i: 0);
7102	MachineOperand &Dest1 = Inst.getOperand(i: 1);
7103	MachineOperand &Src0 = Inst.getOperand(i: 2);
7104	MachineOperand &Src1 = Inst.getOperand(i: 3);
7105
7106	unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7107	? AMDGPU::V_ADD_CO_U32_e64
7108	: AMDGPU::V_SUB_CO_U32_e64;
7109	const TargetRegisterClass *NewRC =
7110	RI.getEquivalentVGPRClass(MRI.getRegClass(Reg: Dest0.getReg()));
7111	Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
7112	MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg)
7113	.addReg(Dest1.getReg(), RegState::Define)
7114	.add(Src0)
7115	.add(Src1)
7116	.addImm(0); // clamp bit
7117
7118	legalizeOperands(MI&: *NewInstr, MDT);
7119	MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
7120	addUsersToMoveToVALUWorklist(Reg: NewInstr->getOperand(i: 0).getReg(), MRI,
7121	Worklist);
7122	Inst.eraseFromParent();
7123	}
7124	return;
7125
7126	case AMDGPU::S_CSELECT_B32:
7127	case AMDGPU::S_CSELECT_B64:
7128	lowerSelect(Worklist, Inst, MDT);
7129	Inst.eraseFromParent();
7130	return;
7131	case AMDGPU::S_CMP_EQ_I32:
7132	case AMDGPU::S_CMP_LG_I32:
7133	case AMDGPU::S_CMP_GT_I32:
7134	case AMDGPU::S_CMP_GE_I32:
7135	case AMDGPU::S_CMP_LT_I32:
7136	case AMDGPU::S_CMP_LE_I32:
7137	case AMDGPU::S_CMP_EQ_U32:
7138	case AMDGPU::S_CMP_LG_U32:
7139	case AMDGPU::S_CMP_GT_U32:
7140	case AMDGPU::S_CMP_GE_U32:
7141	case AMDGPU::S_CMP_LT_U32:
7142	case AMDGPU::S_CMP_LE_U32:
7143	case AMDGPU::S_CMP_EQ_U64:
7144	case AMDGPU::S_CMP_LG_U64:
7145	case AMDGPU::S_CMP_LT_F32:
7146	case AMDGPU::S_CMP_EQ_F32:
7147	case AMDGPU::S_CMP_LE_F32:
7148	case AMDGPU::S_CMP_GT_F32:
7149	case AMDGPU::S_CMP_LG_F32:
7150	case AMDGPU::S_CMP_GE_F32:
7151	case AMDGPU::S_CMP_O_F32:
7152	case AMDGPU::S_CMP_U_F32:
7153	case AMDGPU::S_CMP_NGE_F32:
7154	case AMDGPU::S_CMP_NLG_F32:
7155	case AMDGPU::S_CMP_NGT_F32:
7156	case AMDGPU::S_CMP_NLE_F32:
7157	case AMDGPU::S_CMP_NEQ_F32:
7158	case AMDGPU::S_CMP_NLT_F32:
7159	case AMDGPU::S_CMP_LT_F16:
7160	case AMDGPU::S_CMP_EQ_F16:
7161	case AMDGPU::S_CMP_LE_F16:
7162	case AMDGPU::S_CMP_GT_F16:
7163	case AMDGPU::S_CMP_LG_F16:
7164	case AMDGPU::S_CMP_GE_F16:
7165	case AMDGPU::S_CMP_O_F16:
7166	case AMDGPU::S_CMP_U_F16:
7167	case AMDGPU::S_CMP_NGE_F16:
7168	case AMDGPU::S_CMP_NLG_F16:
7169	case AMDGPU::S_CMP_NGT_F16:
7170	case AMDGPU::S_CMP_NLE_F16:
7171	case AMDGPU::S_CMP_NEQ_F16:
7172	case AMDGPU::S_CMP_NLT_F16: {
7173	Register CondReg = MRI.createVirtualRegister(RI.getWaveMaskRegClass());
7174	auto NewInstr =
7175	BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode), CondReg)
7176	.setMIFlags(Inst.getFlags());
7177	if (AMDGPU::getNamedOperandIdx(NewOpcode,
7178	AMDGPU::OpName::src0_modifiers) >= 0) {
7179	NewInstr
7180	.addImm(0) // src0_modifiers
7181	.add(Inst.getOperand(i: 0)) // src0
7182	.addImm(0) // src1_modifiers
7183	.add(Inst.getOperand(i: 1)) // src1
7184	.addImm(0); // clamp
7185	} else {
7186	NewInstr
7187	.add(Inst.getOperand(i: 0))
7188	.add(Inst.getOperand(i: 1));
7189	}
7190	legalizeOperands(MI&: *NewInstr, MDT);
7191	int SCCIdx = Inst.findRegisterDefOperandIdx(AMDGPU::SCC);
7192	MachineOperand SCCOp = Inst.getOperand(i: SCCIdx);
7193	addSCCDefUsersToVALUWorklist(Op&: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
7194	Inst.eraseFromParent();
7195	return;
7196	}
7197	case AMDGPU::S_CVT_HI_F32_F16: {
7198	const DebugLoc &DL = Inst.getDebugLoc();
7199	Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7200	Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7201	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
7202	.addImm(16)
7203	.add(Inst.getOperand(1));
7204	BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
7205	.addImm(0) // src0_modifiers
7206	.addReg(TmpReg)
7207	.addImm(0) // clamp
7208	.addImm(0); // omod
7209
7210	MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
7211	addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
7212	Inst.eraseFromParent();
7213	return;
7214	}
7215	case AMDGPU::S_MINIMUM_F32:
7216	case AMDGPU::S_MAXIMUM_F32:
7217	case AMDGPU::S_MINIMUM_F16:
7218	case AMDGPU::S_MAXIMUM_F16: {
7219	const DebugLoc &DL = Inst.getDebugLoc();
7220	Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7221	MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
7222	.addImm(0) // src0_modifiers
7223	.add(Inst.getOperand(i: 1))
7224	.addImm(0) // src1_modifiers
7225	.add(Inst.getOperand(i: 2))
7226	.addImm(0) // clamp
7227	.addImm(0); // omod
7228	MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
7229
7230	legalizeOperands(MI&: *NewInstr, MDT);
7231	addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
7232	Inst.eraseFromParent();
7233	return;
7234	}
7235	}
7236
7237	if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
7238	// We cannot move this instruction to the VALU, so we should try to
7239	// legalize its operands instead.
7240	legalizeOperands(MI&: Inst, MDT);
7241	return;
7242	}
7243	// Handle converting generic instructions like COPY-to-SGPR into
7244	// COPY-to-VGPR.
7245	if (NewOpcode == Opcode) {
7246	Register DstReg = Inst.getOperand(i: 0).getReg();
7247	const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
7248
7249	// If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
7250	// hope for the best.
7251	if (Inst.isCopy() && DstReg.isPhysical() &&
7252	RI.isVGPR(MRI, Inst.getOperand(i: 1).getReg())) {
7253	// TODO: Only works for 32 bit registers.
7254	BuildMI(*Inst.getParent(), &Inst, Inst.getDebugLoc(),
7255	get(AMDGPU::V_READFIRSTLANE_B32), Inst.getOperand(0).getReg())
7256	.add(Inst.getOperand(1));
7257	Inst.eraseFromParent();
7258	return;
7259	}
7260
7261	if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual() &&
7262	NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(i: 1).getReg())) {
7263	// Instead of creating a copy where src and dst are the same register
7264	// class, we just replace all uses of dst with src. These kinds of
7265	// copies interfere with the heuristics MachineSink uses to decide
7266	// whether or not to split a critical edge. Since the pass assumes
7267	// that copies will end up as machine instructions and not be
7268	// eliminated.
7269	addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
7270	MRI.replaceRegWith(FromReg: DstReg, ToReg: Inst.getOperand(i: 1).getReg());
7271	MRI.clearKillFlags(Reg: Inst.getOperand(i: 1).getReg());
7272	Inst.getOperand(i: 0).setReg(DstReg);
7273	// Make sure we don't leave around a dead VGPR->SGPR copy. Normally
7274	// these are deleted later, but at -O0 it would leave a suspicious
7275	// looking illegal copy of an undef register.
7276	for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
7277	Inst.removeOperand(OpNo: I);
7278	Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
7279	return;
7280	}
7281	Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
7282	MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
7283	legalizeOperands(MI&: Inst, MDT);
7284	addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
7285	return;
7286	}
7287
7288	// Use the new VALU Opcode.
7289	auto NewInstr = BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(NewOpcode))
7290	.setMIFlags(Inst.getFlags());
7291	if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
7292	// Intersperse VOP3 modifiers among the SALU operands.
7293	NewInstr->addOperand(Inst.getOperand(i: 0));
7294	if (AMDGPU::getNamedOperandIdx(NewOpcode,
7295	AMDGPU::OpName::src0_modifiers) >= 0)
7296	NewInstr.addImm(0);
7297	if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::src0)) {
7298	MachineOperand Src = Inst.getOperand(i: 1);
7299	if (AMDGPU::isTrue16Inst(Opc: NewOpcode) && ST.useRealTrue16Insts() &&
7300	Src.isReg() && RI.isVGPR(MRI, Src.getReg()))
7301	NewInstr.addReg(Src.getReg(), 0, AMDGPU::lo16);
7302	else
7303	NewInstr->addOperand(Src);
7304	}
7305
7306	if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
7307	// We are converting these to a BFE, so we need to add the missing
7308	// operands for the size and offset.
7309	unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
7310	NewInstr.addImm(0);
7311	NewInstr.addImm(Size);
7312	} else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
7313	// The VALU version adds the second operand to the result, so insert an
7314	// extra 0 operand.
7315	NewInstr.addImm(0);
7316	} else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
7317	const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
7318	// If we need to move this to VGPRs, we need to unpack the second
7319	// operand back into the 2 separate ones for bit offset and width.
7320	assert(OffsetWidthOp.isImm() &&
7321	"Scalar BFE is only implemented for constant width and offset");
7322	uint32_t Imm = OffsetWidthOp.getImm();
7323
7324	uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
7325	uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
7326	NewInstr.addImm(Offset);
7327	NewInstr.addImm(BitWidth);
7328	} else {
7329	if (AMDGPU::getNamedOperandIdx(NewOpcode,
7330	AMDGPU::OpName::src1_modifiers) >= 0)
7331	NewInstr.addImm(0);
7332	if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src1) >= 0)
7333	NewInstr->addOperand(Inst.getOperand(i: 2));
7334	if (AMDGPU::getNamedOperandIdx(NewOpcode,
7335	AMDGPU::OpName::src2_modifiers) >= 0)
7336	NewInstr.addImm(0);
7337	if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::src2) >= 0)
7338	NewInstr->addOperand(Inst.getOperand(i: 3));
7339	if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::clamp) >= 0)
7340	NewInstr.addImm(0);
7341	if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::omod) >= 0)
7342	NewInstr.addImm(0);
7343	if (AMDGPU::getNamedOperandIdx(NewOpcode, AMDGPU::OpName::op_sel) >= 0)
7344	NewInstr.addImm(0);
7345	}
7346	} else {
7347	// Just copy the SALU operands.
7348	for (const MachineOperand &Op : Inst.explicit_operands())
7349	NewInstr->addOperand(Op);
7350	}
7351
7352	// Remove any references to SCC. Vector instructions can't read from it, and
7353	// We're just about to add the implicit use / defs of VCC, and we don't want
7354	// both.
7355	for (MachineOperand &Op : Inst.implicit_operands()) {
7356	if (Op.getReg() == AMDGPU::SCC) {
7357	// Only propagate through live-def of SCC.
7358	if (Op.isDef() && !Op.isDead())
7359	addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
7360	if (Op.isUse())
7361	addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
7362	}
7363	}
7364	Inst.eraseFromParent();
7365	Register NewDstReg;
7366	if (NewInstr->getOperand(0).isReg() && NewInstr->getOperand(0).isDef()) {
7367	Register DstReg = NewInstr->getOperand(0).getReg();
7368	assert(DstReg.isVirtual());
7369	// Update the destination register class.
7370	const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
7371	assert(NewDstRC);
7372	NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
7373	MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
7374	}
7375	fixImplicitOperands(MI&: *NewInstr);
7376	// Legalize the operands
7377	legalizeOperands(MI&: *NewInstr, MDT);
7378	if (NewDstReg)
7379	addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
7380	}
7381
7382	// Add/sub require special handling to deal with carry outs.
7383	std::pair<bool, MachineBasicBlock *>
7384	SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
7385	MachineDominatorTree *MDT) const {
7386	if (ST.hasAddNoCarry()) {
7387	// Assume there is no user of scc since we don't select this in that case.
7388	// Since scc isn't used, it doesn't really matter if the i32 or u32 variant
7389	// is used.
7390
7391	MachineBasicBlock &MBB = *Inst.getParent();
7392	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7393
7394	Register OldDstReg = Inst.getOperand(i: 0).getReg();
7395	Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7396
7397	unsigned Opc = Inst.getOpcode();
7398	assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
7399
7400	unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
7401	AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
7402
7403	assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
7404	Inst.removeOperand(OpNo: 3);
7405
7406	Inst.setDesc(get(NewOpc));
7407	Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
7408	Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
7409	MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
7410	MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
7411
7412	addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
7413	return std::pair(true, NewBB);
7414	}
7415
7416	return std::pair(false, nullptr);
7417	}
7418
7419	void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
7420	MachineDominatorTree *MDT) const {
7421
7422	MachineBasicBlock &MBB = *Inst.getParent();
7423	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7424	MachineBasicBlock::iterator MII = Inst;
7425	DebugLoc DL = Inst.getDebugLoc();
7426
7427	MachineOperand &Dest = Inst.getOperand(i: 0);
7428	MachineOperand &Src0 = Inst.getOperand(i: 1);
7429	MachineOperand &Src1 = Inst.getOperand(i: 2);
7430	MachineOperand &Cond = Inst.getOperand(i: 3);
7431
7432	Register CondReg = Cond.getReg();
7433	bool IsSCC = (CondReg == AMDGPU::SCC);
7434
7435	// If this is a trivial select where the condition is effectively not SCC
7436	// (CondReg is a source of copy to SCC), then the select is semantically
7437	// equivalent to copying CondReg. Hence, there is no need to create
7438	// V_CNDMASK, we can just use that and bail out.
7439	if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
7440	(Src1.getImm() == 0)) {
7441	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: CondReg);
7442	return;
7443	}
7444
7445	Register NewCondReg = CondReg;
7446	if (IsSCC) {
7447	const TargetRegisterClass *TC =
7448	RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
7449	NewCondReg = MRI.createVirtualRegister(RegClass: TC);
7450
7451	// Now look for the closest SCC def if it is a copy
7452	// replacing the CondReg with the COPY source register
7453	bool CopyFound = false;
7454	for (MachineInstr &CandI :
7455	make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
7456	y: Inst.getParent()->rend())) {
7457	if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) !=
7458	-1) {
7459	if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) {
7460	BuildMI(MBB, MII, DL, get(AMDGPU::COPY), NewCondReg)
7461	.addReg(CandI.getOperand(1).getReg());
7462	CopyFound = true;
7463	}
7464	break;
7465	}
7466	}
7467	if (!CopyFound) {
7468	// SCC def is not a copy
7469	// Insert a trivial select instead of creating a copy, because a copy from
7470	// SCC would semantically mean just copying a single bit, but we may need
7471	// the result to be a vector condition mask that needs preserving.
7472	unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64
7473	: AMDGPU::S_CSELECT_B32;
7474	auto NewSelect =
7475	BuildMI(MBB, MII, DL, get(Opcode), NewCondReg).addImm(-1).addImm(0);
7476	NewSelect->getOperand(3).setIsUndef(Cond.isUndef());
7477	}
7478	}
7479
7480	Register NewDestReg = MRI.createVirtualRegister(
7481	RI.getEquivalentVGPRClass(MRI.getRegClass(Reg: Dest.getReg())));
7482	MachineInstr *NewInst;
7483	if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
7484	NewInst = BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), NewDestReg)
7485	.addImm(0)
7486	.add(Src1) // False
7487	.addImm(0)
7488	.add(Src0) // True
7489	.addReg(NewCondReg);
7490	} else {
7491	NewInst =
7492	BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B64_PSEUDO), NewDestReg)
7493	.add(Src1) // False
7494	.add(Src0) // True
7495	.addReg(NewCondReg);
7496	}
7497	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
7498	legalizeOperands(MI&: *NewInst, MDT);
7499	addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
7500	}
7501
7502	void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
7503	MachineInstr &Inst) const {
7504	MachineBasicBlock &MBB = *Inst.getParent();
7505	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7506	MachineBasicBlock::iterator MII = Inst;
7507	DebugLoc DL = Inst.getDebugLoc();
7508
7509	MachineOperand &Dest = Inst.getOperand(i: 0);
7510	MachineOperand &Src = Inst.getOperand(i: 1);
7511	Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7512	Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7513
7514	unsigned SubOp = ST.hasAddNoCarry() ?
7515	AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32;
7516
7517	BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
7518	.addImm(0)
7519	.addReg(Src.getReg());
7520
7521	BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
7522	.addReg(Src.getReg())
7523	.addReg(TmpReg);
7524
7525	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
7526	addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
7527	}
7528
7529	void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
7530	MachineInstr &Inst) const {
7531	MachineBasicBlock &MBB = *Inst.getParent();
7532	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7533	MachineBasicBlock::iterator MII = Inst;
7534	const DebugLoc &DL = Inst.getDebugLoc();
7535
7536	MachineOperand &Dest = Inst.getOperand(i: 0);
7537	MachineOperand &Src0 = Inst.getOperand(i: 1);
7538	MachineOperand &Src1 = Inst.getOperand(i: 2);
7539
7540	if (ST.hasDLInsts()) {
7541	Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7542	legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
7543	legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
7544
7545	BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
7546	.add(Src0)
7547	.add(Src1);
7548
7549	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7550	addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7551	} else {
7552	// Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
7553	// invert either source and then perform the XOR. If either source is a
7554	// scalar register, then we can leave the inversion on the scalar unit to
7555	// achieve a better distribution of scalar and vector instructions.
7556	bool Src0IsSGPR = Src0.isReg() &&
7557	RI.isSGPRClass(MRI.getRegClass(Reg: Src0.getReg()));
7558	bool Src1IsSGPR = Src1.isReg() &&
7559	RI.isSGPRClass(MRI.getRegClass(Reg: Src1.getReg()));
7560	MachineInstr *Xor;
7561	Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7562	Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7563
7564	// Build a pair of scalar instructions and add them to the work list.
7565	// The next iteration over the work list will lower these to the vector
7566	// unit as necessary.
7567	if (Src0IsSGPR) {
7568	BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
7569	Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
7570	.addReg(Temp)
7571	.add(Src1);
7572	} else if (Src1IsSGPR) {
7573	BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
7574	Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
7575	.add(Src0)
7576	.addReg(Temp);
7577	} else {
7578	Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
7579	.add(Src0)
7580	.add(Src1);
7581	MachineInstr *Not =
7582	BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
7583	Worklist.insert(MI: Not);
7584	}
7585
7586	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7587
7588	Worklist.insert(MI: Xor);
7589
7590	addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7591	}
7592	}
7593
7594	void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
7595	MachineInstr &Inst,
7596	unsigned Opcode) const {
7597	MachineBasicBlock &MBB = *Inst.getParent();
7598	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7599	MachineBasicBlock::iterator MII = Inst;
7600	const DebugLoc &DL = Inst.getDebugLoc();
7601
7602	MachineOperand &Dest = Inst.getOperand(i: 0);
7603	MachineOperand &Src0 = Inst.getOperand(i: 1);
7604	MachineOperand &Src1 = Inst.getOperand(i: 2);
7605
7606	Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7607	Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7608
7609	MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
7610	.add(Src0)
7611	.add(Src1);
7612
7613	MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
7614	.addReg(Interm);
7615
7616	Worklist.insert(MI: &Op);
7617	Worklist.insert(MI: &Not);
7618
7619	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7620	addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7621	}
7622
7623	void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
7624	MachineInstr &Inst,
7625	unsigned Opcode) const {
7626	MachineBasicBlock &MBB = *Inst.getParent();
7627	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7628	MachineBasicBlock::iterator MII = Inst;
7629	const DebugLoc &DL = Inst.getDebugLoc();
7630
7631	MachineOperand &Dest = Inst.getOperand(i: 0);
7632	MachineOperand &Src0 = Inst.getOperand(i: 1);
7633	MachineOperand &Src1 = Inst.getOperand(i: 2);
7634
7635	Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7636	Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7637
7638	MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
7639	.add(Src1);
7640
7641	MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
7642	.add(Src0)
7643	.addReg(Interm);
7644
7645	Worklist.insert(MI: &Not);
7646	Worklist.insert(MI: &Op);
7647
7648	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7649	addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7650	}
7651
7652	void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
7653	MachineInstr &Inst, unsigned Opcode,
7654	bool Swap) const {
7655	MachineBasicBlock &MBB = *Inst.getParent();
7656	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7657
7658	MachineOperand &Dest = Inst.getOperand(i: 0);
7659	MachineOperand &Src0 = Inst.getOperand(i: 1);
7660	DebugLoc DL = Inst.getDebugLoc();
7661
7662	MachineBasicBlock::iterator MII = Inst;
7663
7664	const MCInstrDesc &InstDesc = get(Opcode);
7665	const TargetRegisterClass *Src0RC = Src0.isReg() ?
7666	MRI.getRegClass(Src0.getReg()) :
7667	&AMDGPU::SGPR_32RegClass;
7668
7669	const TargetRegisterClass *Src0SubRC =
7670	RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7671
7672	MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
7673	AMDGPU::sub0, Src0SubRC);
7674
7675	const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
7676	const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
7677	const TargetRegisterClass *NewDestSubRC =
7678	RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
7679
7680	Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
7681	MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
7682
7683	MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
7684	AMDGPU::sub1, Src0SubRC);
7685
7686	Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
7687	MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
7688
7689	if (Swap)
7690	std::swap(a&: DestSub0, b&: DestSub1);
7691
7692	Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
7693	BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
7694	.addReg(DestSub0)
7695	.addImm(AMDGPU::sub0)
7696	.addReg(DestSub1)
7697	.addImm(AMDGPU::sub1);
7698
7699	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7700
7701	Worklist.insert(MI: &LoHalf);
7702	Worklist.insert(MI: &HiHalf);
7703
7704	// We don't need to legalizeOperands here because for a single operand, src0
7705	// will support any kind of input.
7706
7707	// Move all users of this moved value.
7708	addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
7709	}
7710
7711	// There is not a vector equivalent of s_mul_u64. For this reason, we need to
7712	// split the s_mul_u64 in 32-bit vector multiplications.
7713	void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
7714	MachineInstr &Inst,
7715	MachineDominatorTree *MDT) const {
7716	MachineBasicBlock &MBB = *Inst.getParent();
7717	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7718
7719	Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
7720	Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7721	Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7722
7723	MachineOperand &Dest = Inst.getOperand(i: 0);
7724	MachineOperand &Src0 = Inst.getOperand(i: 1);
7725	MachineOperand &Src1 = Inst.getOperand(i: 2);
7726	const DebugLoc &DL = Inst.getDebugLoc();
7727	MachineBasicBlock::iterator MII = Inst;
7728
7729	const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
7730	const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
7731	const TargetRegisterClass *Src0SubRC =
7732	RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7733	if (RI.isSGPRClass(Src0SubRC))
7734	Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);
7735	const TargetRegisterClass *Src1SubRC =
7736	RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
7737	if (RI.isSGPRClass(Src1SubRC))
7738	Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);
7739
7740	// First, we extract the low 32-bit and high 32-bit values from each of the
7741	// operands.
7742	MachineOperand Op0L =
7743	buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
7744	MachineOperand Op1L =
7745	buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
7746	MachineOperand Op0H =
7747	buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
7748	MachineOperand Op1H =
7749	buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
7750
7751	// The multilication is done as follows:
7752	//
7753	// Op1H Op1L
7754	// * Op0H Op0L
7755	// --------------------
7756	// Op1H*Op0L Op1L*Op0L
7757	// + Op1H*Op0H Op1L*Op0H
7758	// -----------------------------------------
7759	// (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
7760	//
7761	// We drop Op1H*Op0H because the result of the multiplication is a 64-bit
7762	// value and that would overflow.
7763	// The low 32-bit value is Op1L*Op0L.
7764	// The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
7765
7766	Register Op1L_Op0H_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7767	MachineInstr *Op1L_Op0H =
7768	BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1L_Op0H_Reg)
7769	.add(Op1L)
7770	.add(Op0H);
7771
7772	Register Op1H_Op0L_Reg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7773	MachineInstr *Op1H_Op0L =
7774	BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), Op1H_Op0L_Reg)
7775	.add(Op1H)
7776	.add(Op0L);
7777
7778	Register CarryReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7779	MachineInstr *Carry =
7780	BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_HI_U32_e64), CarryReg)
7781	.add(Op1L)
7782	.add(Op0L);
7783
7784	MachineInstr *LoHalf =
7785	BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)
7786	.add(Op1L)
7787	.add(Op0L);
7788
7789	Register AddReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7790	MachineInstr *Add = BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), AddReg)
7791	.addReg(Op1L_Op0H_Reg)
7792	.addReg(Op1H_Op0L_Reg);
7793
7794	MachineInstr *HiHalf =
7795	BuildMI(MBB, MII, DL, get(AMDGPU::V_ADD_U32_e32), DestSub1)
7796	.addReg(AddReg)
7797	.addReg(CarryReg);
7798
7799	BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
7800	.addReg(DestSub0)
7801	.addImm(AMDGPU::sub0)
7802	.addReg(DestSub1)
7803	.addImm(AMDGPU::sub1);
7804
7805	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7806
7807	// Try to legalize the operands in case we need to swap the order to keep it
7808	// valid.
7809	legalizeOperands(MI&: *Op1L_Op0H, MDT);
7810	legalizeOperands(MI&: *Op1H_Op0L, MDT);
7811	legalizeOperands(MI&: *Carry, MDT);
7812	legalizeOperands(MI&: *LoHalf, MDT);
7813	legalizeOperands(MI&: *Add, MDT);
7814	legalizeOperands(MI&: *HiHalf, MDT);
7815
7816	// Move all users of this moved value.
7817	addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
7818	}
7819
7820	// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
7821	// multiplications.
7822	void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
7823	MachineInstr &Inst,
7824	MachineDominatorTree *MDT) const {
7825	MachineBasicBlock &MBB = *Inst.getParent();
7826	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7827
7828	Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
7829	Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7830	Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7831
7832	MachineOperand &Dest = Inst.getOperand(i: 0);
7833	MachineOperand &Src0 = Inst.getOperand(i: 1);
7834	MachineOperand &Src1 = Inst.getOperand(i: 2);
7835	const DebugLoc &DL = Inst.getDebugLoc();
7836	MachineBasicBlock::iterator MII = Inst;
7837
7838	const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
7839	const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
7840	const TargetRegisterClass *Src0SubRC =
7841	RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7842	if (RI.isSGPRClass(Src0SubRC))
7843	Src0SubRC = RI.getEquivalentVGPRClass(Src0SubRC);
7844	const TargetRegisterClass *Src1SubRC =
7845	RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
7846	if (RI.isSGPRClass(Src1SubRC))
7847	Src1SubRC = RI.getEquivalentVGPRClass(Src1SubRC);
7848
7849	// First, we extract the low 32-bit and high 32-bit values from each of the
7850	// operands.
7851	MachineOperand Op0L =
7852	buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
7853	MachineOperand Op1L =
7854	buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
7855
7856	unsigned Opc = Inst.getOpcode();
7857	unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
7858	? AMDGPU::V_MUL_HI_U32_e64
7859	: AMDGPU::V_MUL_HI_I32_e64;
7860	MachineInstr *HiHalf =
7861	BuildMI(MBB, MII, DL, get(NewOpc), DestSub1).add(Op1L).add(Op0L);
7862
7863	MachineInstr *LoHalf =
7864	BuildMI(MBB, MII, DL, get(AMDGPU::V_MUL_LO_U32_e64), DestSub0)
7865	.add(Op1L)
7866	.add(Op0L);
7867
7868	BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
7869	.addReg(DestSub0)
7870	.addImm(AMDGPU::sub0)
7871	.addReg(DestSub1)
7872	.addImm(AMDGPU::sub1);
7873
7874	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7875
7876	// Try to legalize the operands in case we need to swap the order to keep it
7877	// valid.
7878	legalizeOperands(MI&: *HiHalf, MDT);
7879	legalizeOperands(MI&: *LoHalf, MDT);
7880
7881	// Move all users of this moved value.
7882	addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
7883	}
7884
7885	void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
7886	MachineInstr &Inst, unsigned Opcode,
7887	MachineDominatorTree *MDT) const {
7888	MachineBasicBlock &MBB = *Inst.getParent();
7889	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7890
7891	MachineOperand &Dest = Inst.getOperand(i: 0);
7892	MachineOperand &Src0 = Inst.getOperand(i: 1);
7893	MachineOperand &Src1 = Inst.getOperand(i: 2);
7894	DebugLoc DL = Inst.getDebugLoc();
7895
7896	MachineBasicBlock::iterator MII = Inst;
7897
7898	const MCInstrDesc &InstDesc = get(Opcode);
7899	const TargetRegisterClass *Src0RC = Src0.isReg() ?
7900	MRI.getRegClass(Src0.getReg()) :
7901	&AMDGPU::SGPR_32RegClass;
7902
7903	const TargetRegisterClass *Src0SubRC =
7904	RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7905	const TargetRegisterClass *Src1RC = Src1.isReg() ?
7906	MRI.getRegClass(Src1.getReg()) :
7907	&AMDGPU::SGPR_32RegClass;
7908
7909	const TargetRegisterClass *Src1SubRC =
7910	RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
7911
7912	MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
7913	AMDGPU::sub0, Src0SubRC);
7914	MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
7915	AMDGPU::sub0, Src1SubRC);
7916	MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
7917	AMDGPU::sub1, Src0SubRC);
7918	MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
7919	AMDGPU::sub1, Src1SubRC);
7920
7921	const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
7922	const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
7923	const TargetRegisterClass *NewDestSubRC =
7924	RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
7925
7926	Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
7927	MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
7928	.add(MO: SrcReg0Sub0)
7929	.add(MO: SrcReg1Sub0);
7930
7931	Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
7932	MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
7933	.add(MO: SrcReg0Sub1)
7934	.add(MO: SrcReg1Sub1);
7935
7936	Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
7937	BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
7938	.addReg(DestSub0)
7939	.addImm(AMDGPU::sub0)
7940	.addReg(DestSub1)
7941	.addImm(AMDGPU::sub1);
7942
7943	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7944
7945	Worklist.insert(MI: &LoHalf);
7946	Worklist.insert(MI: &HiHalf);
7947
7948	// Move all users of this moved value.
7949	addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
7950	}
7951
7952	void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
7953	MachineInstr &Inst,
7954	MachineDominatorTree *MDT) const {
7955	MachineBasicBlock &MBB = *Inst.getParent();
7956	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7957
7958	MachineOperand &Dest = Inst.getOperand(i: 0);
7959	MachineOperand &Src0 = Inst.getOperand(i: 1);
7960	MachineOperand &Src1 = Inst.getOperand(i: 2);
7961	const DebugLoc &DL = Inst.getDebugLoc();
7962
7963	MachineBasicBlock::iterator MII = Inst;
7964
7965	const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
7966
7967	Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
7968
7969	MachineOperand* Op0;
7970	MachineOperand* Op1;
7971
7972	if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {
7973	Op0 = &Src0;
7974	Op1 = &Src1;
7975	} else {
7976	Op0 = &Src1;
7977	Op1 = &Src0;
7978	}
7979
7980	BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
7981	.add(*Op0);
7982
7983	Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
7984
7985	MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
7986	.addReg(Interm)
7987	.add(*Op1);
7988
7989	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7990
7991	Worklist.insert(MI: &Xor);
7992	}
7993
7994	void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
7995	MachineInstr &Inst) const {
7996	MachineBasicBlock &MBB = *Inst.getParent();
7997	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7998
7999	MachineBasicBlock::iterator MII = Inst;
8000	const DebugLoc &DL = Inst.getDebugLoc();
8001
8002	MachineOperand &Dest = Inst.getOperand(i: 0);
8003	MachineOperand &Src = Inst.getOperand(i: 1);
8004
8005	const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
8006	const TargetRegisterClass *SrcRC = Src.isReg() ?
8007	MRI.getRegClass(Src.getReg()) :
8008	&AMDGPU::SGPR_32RegClass;
8009
8010	Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8011	Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8012
8013	const TargetRegisterClass *SrcSubRC =
8014	RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
8015
8016	MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
8017	AMDGPU::sub0, SrcSubRC);
8018	MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
8019	AMDGPU::sub1, SrcSubRC);
8020
8021	BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
8022
8023	BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
8024
8025	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8026
8027	// We don't need to legalize operands here. src0 for either instruction can be
8028	// an SGPR, and the second input is unused or determined here.
8029	addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8030	}
8031
8032	void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
8033	MachineInstr &Inst) const {
8034	MachineBasicBlock &MBB = *Inst.getParent();
8035	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8036	MachineBasicBlock::iterator MII = Inst;
8037	const DebugLoc &DL = Inst.getDebugLoc();
8038
8039	MachineOperand &Dest = Inst.getOperand(i: 0);
8040	uint32_t Imm = Inst.getOperand(i: 2).getImm();
8041	uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
8042	uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
8043
8044	(void) Offset;
8045
8046	// Only sext_inreg cases handled.
8047	assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
8048	Offset == 0 && "Not implemented");
8049
8050	if (BitWidth < 32) {
8051	Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8052	Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8053	Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8054
8055	BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo)
8056	.addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
8057	.addImm(0)
8058	.addImm(BitWidth);
8059
8060	BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
8061	.addImm(31)
8062	.addReg(MidRegLo);
8063
8064	BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
8065	.addReg(MidRegLo)
8066	.addImm(AMDGPU::sub0)
8067	.addReg(MidRegHi)
8068	.addImm(AMDGPU::sub1);
8069
8070	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8071	addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8072	return;
8073	}
8074
8075	MachineOperand &Src = Inst.getOperand(i: 1);
8076	Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8077	Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
8078
8079	BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
8080	.addImm(31)
8081	.addReg(Src.getReg(), 0, AMDGPU::sub0);
8082
8083	BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
8084	.addReg(Src.getReg(), 0, AMDGPU::sub0)
8085	.addImm(AMDGPU::sub0)
8086	.addReg(TmpReg)
8087	.addImm(AMDGPU::sub1);
8088
8089	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8090	addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8091	}
8092
8093	void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
8094	MachineInstr &Inst, unsigned Opcode,
8095	MachineDominatorTree *MDT) const {
8096	// (S_FLBIT_I32_B64 hi:lo) ->
8097	// -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32))
8098	// (S_FF1_I32_B64 hi:lo) ->
8099	// ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
8100
8101	MachineBasicBlock &MBB = *Inst.getParent();
8102	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8103	MachineBasicBlock::iterator MII = Inst;
8104	const DebugLoc &DL = Inst.getDebugLoc();
8105
8106	MachineOperand &Dest = Inst.getOperand(i: 0);
8107	MachineOperand &Src = Inst.getOperand(i: 1);
8108
8109	const MCInstrDesc &InstDesc = get(Opcode);
8110
8111	bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
8112	unsigned OpcodeAdd =
8113	ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
8114
8115	const TargetRegisterClass *SrcRC =
8116	Src.isReg() ? MRI.getRegClass(Src.getReg()) : &AMDGPU::SGPR_32RegClass;
8117	const TargetRegisterClass *SrcSubRC =
8118	RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
8119
8120	MachineOperand SrcRegSub0 =
8121	buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub0, SrcSubRC);
8122	MachineOperand SrcRegSub1 =
8123	buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, AMDGPU::sub1, SrcSubRC);
8124
8125	Register MidReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8126	Register MidReg2 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8127	Register MidReg3 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8128	Register MidReg4 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8129
8130	BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
8131
8132	BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
8133
8134	BuildMI(MBB, MII, DL, get(OpcodeAdd), MidReg3)
8135	.addReg(IsCtlz ? MidReg1 : MidReg2)
8136	.addImm(32)
8137	.addImm(1); // enable clamp
8138
8139	BuildMI(MBB, MII, DL, get(AMDGPU::V_MIN_U32_e64), MidReg4)
8140	.addReg(MidReg3)
8141	.addReg(IsCtlz ? MidReg2 : MidReg1);
8142
8143	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
8144
8145	addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
8146	}
8147
8148	void SIInstrInfo::addUsersToMoveToVALUWorklist(
8149	Register DstReg, MachineRegisterInfo &MRI,
8150	SIInstrWorklist &Worklist) const {
8151	for (MachineRegisterInfo::use_iterator I = MRI.use_begin(RegNo: DstReg),
8152	E = MRI.use_end(); I != E;) {
8153	MachineInstr &UseMI = *I->getParent();
8154
8155	unsigned OpNo = 0;
8156
8157	switch (UseMI.getOpcode()) {
8158	case AMDGPU::COPY:
8159	case AMDGPU::WQM:
8160	case AMDGPU::SOFT_WQM:
8161	case AMDGPU::STRICT_WWM:
8162	case AMDGPU::STRICT_WQM:
8163	case AMDGPU::REG_SEQUENCE:
8164	case AMDGPU::PHI:
8165	case AMDGPU::INSERT_SUBREG:
8166	break;
8167	default:
8168	OpNo = I.getOperandNo();
8169	break;
8170	}
8171
8172	if (!RI.hasVectorRegisters(getOpRegClass(MI: UseMI, OpNo))) {
8173	Worklist.insert(MI: &UseMI);
8174
8175	do {
8176	++I;
8177	} while (I != E && I->getParent() == &UseMI);
8178	} else {
8179	++I;
8180	}
8181	}
8182	}
8183
8184	void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
8185	MachineRegisterInfo &MRI,
8186	MachineInstr &Inst) const {
8187	Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8188	MachineBasicBlock *MBB = Inst.getParent();
8189	MachineOperand &Src0 = Inst.getOperand(i: 1);
8190	MachineOperand &Src1 = Inst.getOperand(i: 2);
8191	const DebugLoc &DL = Inst.getDebugLoc();
8192
8193	switch (Inst.getOpcode()) {
8194	case AMDGPU::S_PACK_LL_B32_B16: {
8195	Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8196	Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8197
8198	// FIXME: Can do a lot better if we know the high bits of src0 or src1 are
8199	// 0.
8200	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
8201	.addImm(0xffff);
8202
8203	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
8204	.addReg(ImmReg, RegState::Kill)
8205	.add(Src0);
8206
8207	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)
8208	.add(Src1)
8209	.addImm(16)
8210	.addReg(TmpReg, RegState::Kill);
8211	break;
8212	}
8213	case AMDGPU::S_PACK_LH_B32_B16: {
8214	Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8215	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
8216	.addImm(0xffff);
8217	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg)
8218	.addReg(ImmReg, RegState::Kill)
8219	.add(Src0)
8220	.add(Src1);
8221	break;
8222	}
8223	case AMDGPU::S_PACK_HL_B32_B16: {
8224	Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8225	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
8226	.addImm(16)
8227	.add(Src0);
8228	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg)
8229	.add(Src1)
8230	.addImm(16)
8231	.addReg(TmpReg, RegState::Kill);
8232	break;
8233	}
8234	case AMDGPU::S_PACK_HH_B32_B16: {
8235	Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8236	Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
8237	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
8238	.addImm(16)
8239	.add(Src0);
8240	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
8241	.addImm(0xffff0000);
8242	BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg)
8243	.add(Src1)
8244	.addReg(ImmReg, RegState::Kill)
8245	.addReg(TmpReg, RegState::Kill);
8246	break;
8247	}
8248	default:
8249	llvm_unreachable("unhandled s_pack_* instruction");
8250	}
8251
8252	MachineOperand &Dest = Inst.getOperand(i: 0);
8253	MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8254	addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
8255	}
8256
8257	void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
8258	MachineInstr &SCCDefInst,
8259	SIInstrWorklist &Worklist,
8260	Register NewCond) const {
8261
8262	// Ensure that def inst defines SCC, which is still live.
8263	assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
8264	!Op.isDead() && Op.getParent() == &SCCDefInst);
8265	SmallVector<MachineInstr *, 4> CopyToDelete;
8266	// This assumes that all the users of SCC are in the same block
8267	// as the SCC def.
8268	for (MachineInstr &MI : // Skip the def inst itself.
8269	make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
8270	y: SCCDefInst.getParent()->end())) {
8271	// Check if SCC is used first.
8272	int SCCIdx = MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI);
8273	if (SCCIdx != -1) {
8274	if (MI.isCopy()) {
8275	MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8276	Register DestReg = MI.getOperand(i: 0).getReg();
8277
8278	MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
8279	CopyToDelete.push_back(Elt: &MI);
8280	} else {
8281
8282	if (NewCond.isValid())
8283	MI.getOperand(i: SCCIdx).setReg(NewCond);
8284
8285	Worklist.insert(MI: &MI);
8286	}
8287	}
8288	// Exit if we find another SCC def.
8289	if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
8290	break;
8291	}
8292	for (auto &Copy : CopyToDelete)
8293	Copy->eraseFromParent();
8294	}
8295
8296	// Instructions that use SCC may be converted to VALU instructions. When that
8297	// happens, the SCC register is changed to VCC_LO. The instruction that defines
8298	// SCC must be changed to an instruction that defines VCC. This function makes
8299	// sure that the instruction that defines SCC is added to the moveToVALU
8300	// worklist.
8301	void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
8302	SIInstrWorklist &Worklist) const {
8303	// Look for a preceding instruction that either defines VCC or SCC. If VCC
8304	// then there is nothing to do because the defining instruction has been
8305	// converted to a VALU already. If SCC then that instruction needs to be
8306	// converted to a VALU.
8307	for (MachineInstr &MI :
8308	make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
8309	y: SCCUseInst->getParent()->rend())) {
8310	if (MI.modifiesRegister(AMDGPU::VCC, &RI))
8311	break;
8312	if (MI.definesRegister(AMDGPU::SCC, &RI)) {
8313	Worklist.insert(MI: &MI);
8314	break;
8315	}
8316	}
8317	}
8318
8319	const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
8320	const MachineInstr &Inst) const {
8321	const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
8322
8323	switch (Inst.getOpcode()) {
8324	// For target instructions, getOpRegClass just returns the virtual register
8325	// class associated with the operand, so we need to find an equivalent VGPR
8326	// register class in order to move the instruction to the VALU.
8327	case AMDGPU::COPY:
8328	case AMDGPU::PHI:
8329	case AMDGPU::REG_SEQUENCE:
8330	case AMDGPU::INSERT_SUBREG:
8331	case AMDGPU::WQM:
8332	case AMDGPU::SOFT_WQM:
8333	case AMDGPU::STRICT_WWM:
8334	case AMDGPU::STRICT_WQM: {
8335	const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
8336	if (RI.isAGPRClass(SrcRC)) {
8337	if (RI.isAGPRClass(NewDstRC))
8338	return nullptr;
8339
8340	switch (Inst.getOpcode()) {
8341	case AMDGPU::PHI:
8342	case AMDGPU::REG_SEQUENCE:
8343	case AMDGPU::INSERT_SUBREG:
8344	NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
8345	break;
8346	default:
8347	NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
8348	}
8349
8350	if (!NewDstRC)
8351	return nullptr;
8352	} else {
8353	if (RI.isVGPRClass(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
8354	return nullptr;
8355
8356	NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
8357	if (!NewDstRC)
8358	return nullptr;
8359	}
8360
8361	return NewDstRC;
8362	}
8363	default:
8364	return NewDstRC;
8365	}
8366	}
8367
8368	// Find the one SGPR operand we are allowed to use.
8369	Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
8370	int OpIndices[3]) const {
8371	const MCInstrDesc &Desc = MI.getDesc();
8372
8373	// Find the one SGPR operand we are allowed to use.
8374	//
8375	// First we need to consider the instruction's operand requirements before
8376	// legalizing. Some operands are required to be SGPRs, such as implicit uses
8377	// of VCC, but we are still bound by the constant bus requirement to only use
8378	// one.
8379	//
8380	// If the operand's class is an SGPR, we can never move it.
8381
8382	Register SGPRReg = findImplicitSGPRRead(MI);
8383	if (SGPRReg)
8384	return SGPRReg;
8385
8386	Register UsedSGPRs[3] = {Register()};
8387	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8388
8389	for (unsigned i = 0; i < 3; ++i) {
8390	int Idx = OpIndices[i];
8391	if (Idx == -1)
8392	break;
8393
8394	const MachineOperand &MO = MI.getOperand(i: Idx);
8395	if (!MO.isReg())
8396	continue;
8397
8398	// Is this operand statically required to be an SGPR based on the operand
8399	// constraints?
8400	const TargetRegisterClass *OpRC =
8401	RI.getRegClass(Desc.operands()[Idx].RegClass);
8402	bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
8403	if (IsRequiredSGPR)
8404	return MO.getReg();
8405
8406	// If this could be a VGPR or an SGPR, Check the dynamic register class.
8407	Register Reg = MO.getReg();
8408	const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
8409	if (RI.isSGPRClass(RegRC))
8410	UsedSGPRs[i] = Reg;
8411	}
8412
8413	// We don't have a required SGPR operand, so we have a bit more freedom in
8414	// selecting operands to move.
8415
8416	// Try to select the most used SGPR. If an SGPR is equal to one of the
8417	// others, we choose that.
8418	//
8419	// e.g.
8420	// V_FMA_F32 v0, s0, s0, s0 -> No moves
8421	// V_FMA_F32 v0, s0, s1, s0 -> Move s1
8422
8423	// TODO: If some of the operands are 64-bit SGPRs and some 32, we should
8424	// prefer those.
8425
8426	if (UsedSGPRs[0]) {
8427	if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
8428	SGPRReg = UsedSGPRs[0];
8429	}
8430
8431	if (!SGPRReg && UsedSGPRs[1]) {
8432	if (UsedSGPRs[1] == UsedSGPRs[2])
8433	SGPRReg = UsedSGPRs[1];
8434	}
8435
8436	return SGPRReg;
8437	}
8438
8439	MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
8440	unsigned OperandName) const {
8441	int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: OperandName);
8442	if (Idx == -1)
8443	return nullptr;
8444
8445	return &MI.getOperand(i: Idx);
8446	}
8447
8448	uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
8449	if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
8450	int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
8451	? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
8452	: (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
8453	return (Format << 44) |
8454	(1ULL << 56) | // RESOURCE_LEVEL = 1
8455	(3ULL << 60); // OOB_SELECT = 3
8456	}
8457
8458	uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
8459	if (ST.isAmdHsaOS()) {
8460	// Set ATC = 1. GFX9 doesn't have this bit.
8461	if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
8462	RsrcDataFormat |= (1ULL << 56);
8463
8464	// Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
8465	// BTW, it disables TC L2 and therefore decreases performance.
8466	if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
8467	RsrcDataFormat |= (2ULL << 59);
8468	}
8469
8470	return RsrcDataFormat;
8471	}
8472
8473	uint64_t SIInstrInfo::getScratchRsrcWords23() const {
8474	uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
8475	AMDGPU::RSRC_TID_ENABLE |
8476	0xffffffff; // Size;
8477
8478	// GFX9 doesn't have ELEMENT_SIZE.
8479	if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
8480	uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
8481	Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
8482	}
8483
8484	// IndexStride = 64 / 32.
8485	uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2;
8486	Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
8487
8488	// If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
8489	// Clear them unless we want a huge stride.
8490	if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
8491	ST.getGeneration() <= AMDGPUSubtarget::GFX9)
8492	Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
8493
8494	return Rsrc23;
8495	}
8496
8497	bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
8498	unsigned Opc = MI.getOpcode();
8499
8500	return isSMRD(Opcode: Opc);
8501	}
8502
8503	bool SIInstrInfo::isHighLatencyDef(int Opc) const {
8504	return get(Opc).mayLoad() &&
8505	(isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
8506	}
8507
8508	unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
8509	int &FrameIndex) const {
8510	const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
8511	if (!Addr || !Addr->isFI())
8512	return Register();
8513
8514	assert(!MI.memoperands_empty() &&
8515	(*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
8516
8517	FrameIndex = Addr->getIndex();
8518	return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
8519	}
8520
8521	unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
8522	int &FrameIndex) const {
8523	const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
8524	assert(Addr && Addr->isFI());
8525	FrameIndex = Addr->getIndex();
8526	return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
8527	}
8528
8529	Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
8530	int &FrameIndex) const {
8531	if (!MI.mayLoad())
8532	return Register();
8533
8534	if (isMUBUF(MI) || isVGPRSpill(MI))
8535	return isStackAccess(MI, FrameIndex);
8536
8537	if (isSGPRSpill(MI))
8538	return isSGPRStackAccess(MI, FrameIndex);
8539
8540	return Register();
8541	}
8542
8543	Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
8544	int &FrameIndex) const {
8545	if (!MI.mayStore())
8546	return Register();
8547
8548	if (isMUBUF(MI) || isVGPRSpill(MI))
8549	return isStackAccess(MI, FrameIndex);
8550
8551	if (isSGPRSpill(MI))
8552	return isSGPRStackAccess(MI, FrameIndex);
8553
8554	return Register();
8555	}
8556
8557	unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const {
8558	unsigned Size = 0;
8559	MachineBasicBlock::const_instr_iterator I = MI.getIterator();
8560	MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
8561	while (++I != E && I->isInsideBundle()) {
8562	assert(!I->isBundle() && "No nested bundle!");
8563	Size += getInstSizeInBytes(MI: *I);
8564	}
8565
8566	return Size;
8567	}
8568
8569	unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
8570	unsigned Opc = MI.getOpcode();
8571	const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
8572	unsigned DescSize = Desc.getSize();
8573
8574	// If we have a definitive size, we can use it. Otherwise we need to inspect
8575	// the operands to know the size.
8576	if (isFixedSize(MI)) {
8577	unsigned Size = DescSize;
8578
8579	// If we hit the buggy offset, an extra nop will be inserted in MC so
8580	// estimate the worst case.
8581	if (MI.isBranch() && ST.hasOffset3fBug())
8582	Size += 4;
8583
8584	return Size;
8585	}
8586
8587	// Instructions may have a 32-bit literal encoded after them. Check
8588	// operands that could ever be literals.
8589	if (isVALU(MI) || isSALU(MI)) {
8590	if (isDPP(MI))
8591	return DescSize;
8592	bool HasLiteral = false;
8593	for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
8594	const MachineOperand &Op = MI.getOperand(i: I);
8595	const MCOperandInfo &OpInfo = Desc.operands()[I];
8596	if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
8597	HasLiteral = true;
8598	break;
8599	}
8600	}
8601	return HasLiteral ? DescSize + 4 : DescSize;
8602	}
8603
8604	// Check whether we have extra NSA words.
8605	if (isMIMG(MI)) {
8606	int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
8607	if (VAddr0Idx < 0)
8608	return 8;
8609
8610	int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
8611	return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
8612	}
8613
8614	switch (Opc) {
8615	case TargetOpcode::BUNDLE:
8616	return getInstBundleSize(MI);
8617	case TargetOpcode::INLINEASM:
8618	case TargetOpcode::INLINEASM_BR: {
8619	const MachineFunction *MF = MI.getParent()->getParent();
8620	const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
8621	return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST);
8622	}
8623	default:
8624	if (MI.isMetaInstruction())
8625	return 0;
8626	return DescSize;
8627	}
8628	}
8629
8630	bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
8631	if (!isFLAT(MI))
8632	return false;
8633
8634	if (MI.memoperands_empty())
8635	return true;
8636
8637	for (const MachineMemOperand *MMO : MI.memoperands()) {
8638	if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
8639	return true;
8640	}
8641	return false;
8642	}
8643
8644	bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const {
8645	return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
8646	}
8647
8648	void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
8649	MachineBasicBlock *IfEnd) const {
8650	MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator();
8651	assert(TI != IfEntry->end());
8652
8653	MachineInstr *Branch = &(*TI);
8654	MachineFunction *MF = IfEntry->getParent();
8655	MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
8656
8657	if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
8658	Register DstReg = MRI.createVirtualRegister(RI.getBoolRC());
8659	MachineInstr *SIIF =
8660	BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
8661	.add(Branch->getOperand(0))
8662	.add(Branch->getOperand(1));
8663	MachineInstr *SIEND =
8664	BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
8665	.addReg(DstReg);
8666
8667	IfEntry->erase(I: TI);
8668	IfEntry->insert(I: IfEntry->end(), MI: SIIF);
8669	IfEnd->insert(I: IfEnd->getFirstNonPHI(), MI: SIEND);
8670	}
8671	}
8672
8673	void SIInstrInfo::convertNonUniformLoopRegion(
8674	MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
8675	MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator();
8676	// We expect 2 terminators, one conditional and one unconditional.
8677	assert(TI != LoopEnd->end());
8678
8679	MachineInstr *Branch = &(*TI);
8680	MachineFunction *MF = LoopEnd->getParent();
8681	MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
8682
8683	if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
8684
8685	Register DstReg = MRI.createVirtualRegister(RI.getBoolRC());
8686	Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC());
8687	MachineInstrBuilder HeaderPHIBuilder =
8688	BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg);
8689	for (MachineBasicBlock *PMBB : LoopEntry->predecessors()) {
8690	if (PMBB == LoopEnd) {
8691	HeaderPHIBuilder.addReg(RegNo: BackEdgeReg);
8692	} else {
8693	Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC());
8694	materializeImmediate(MBB&: *PMBB, MI: PMBB->getFirstTerminator(), DL: DebugLoc(),
8695	DestReg: ZeroReg, Value: 0);
8696	HeaderPHIBuilder.addReg(RegNo: ZeroReg);
8697	}
8698	HeaderPHIBuilder.addMBB(MBB: PMBB);
8699	}
8700	MachineInstr *HeaderPhi = HeaderPHIBuilder;
8701	MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(),
8702	get(AMDGPU::SI_IF_BREAK), BackEdgeReg)
8703	.addReg(DstReg)
8704	.add(Branch->getOperand(0));
8705	MachineInstr *SILOOP =
8706	BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP))
8707	.addReg(BackEdgeReg)
8708	.addMBB(LoopEntry);
8709
8710	LoopEntry->insert(I: LoopEntry->begin(), MI: HeaderPhi);
8711	LoopEnd->erase(I: TI);
8712	LoopEnd->insert(I: LoopEnd->end(), MI: SIIFBREAK);
8713	LoopEnd->insert(I: LoopEnd->end(), MI: SILOOP);
8714	}
8715	}
8716
8717	ArrayRef<std::pair<int, const char *>>
8718	SIInstrInfo::getSerializableTargetIndices() const {
8719	static const std::pair<int, const char *> TargetIndices[] = {
8720	{AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
8721	{AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
8722	{AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
8723	{AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
8724	{AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
8725	return ArrayRef(TargetIndices);
8726	}
8727
8728	/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
8729	/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
8730	ScheduleHazardRecognizer *
8731	SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
8732	const ScheduleDAG *DAG) const {
8733	return new GCNHazardRecognizer(DAG->MF);
8734	}
8735
8736	/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
8737	/// pass.
8738	ScheduleHazardRecognizer *
8739	SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
8740	return new GCNHazardRecognizer(MF);
8741	}
8742
8743	// Called during:
8744	// - pre-RA scheduling and post-RA scheduling
8745	ScheduleHazardRecognizer *
8746	SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
8747	const ScheduleDAGMI *DAG) const {
8748	// Borrowed from Arm Target
8749	// We would like to restrict this hazard recognizer to only
8750	// post-RA scheduling; we can tell that we're post-RA because we don't
8751	// track VRegLiveness.
8752	if (!DAG->hasVRegLiveness())
8753	return new GCNHazardRecognizer(DAG->MF);
8754	return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
8755	}
8756
8757	std::pair<unsigned, unsigned>
8758	SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
8759	return std::pair(TF & MO_MASK, TF & ~MO_MASK);
8760	}
8761
8762	ArrayRef<std::pair<unsigned, const char *>>
8763	SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
8764	static const std::pair<unsigned, const char *> TargetFlags[] = {
8765	{ MO_GOTPCREL, "amdgpu-gotprel" },
8766	{ MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" },
8767	{ MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" },
8768	{ MO_REL32_LO, "amdgpu-rel32-lo" },
8769	{ MO_REL32_HI, "amdgpu-rel32-hi" },
8770	{ MO_ABS32_LO, "amdgpu-abs32-lo" },
8771	{ MO_ABS32_HI, "amdgpu-abs32-hi" },
8772	};
8773
8774	return ArrayRef(TargetFlags);
8775	}
8776
8777	ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
8778	SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
8779	static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8780	{
8781	{MONoClobber, "amdgpu-noclobber"},
8782	{MOLastUse, "amdgpu-last-use"},
8783	};
8784
8785	return ArrayRef(TargetFlags);
8786	}
8787
8788	unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
8789	const MachineFunction &MF) const {
8790	const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8791	assert(SrcReg.isVirtual());
8792	if (MFI->checkFlag(SrcReg, AMDGPU::VirtRegFlag::WWM_REG))
8793	return AMDGPU::WWM_COPY;
8794
8795	return AMDGPU::COPY;
8796	}
8797
8798	bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
8799	Register Reg) const {
8800	// We need to handle instructions which may be inserted during register
8801	// allocation to handle the prolog. The initial prolog instruction may have
8802	// been separated from the start of the block by spills and copies inserted
8803	// needed by the prolog. However, the insertions for scalar registers can
8804	// always be placed at the BB top as they are independent of the exec mask
8805	// value.
8806	bool IsNullOrVectorRegister = true;
8807	if (Reg) {
8808	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8809	IsNullOrVectorRegister = !RI.isSGPRClass(RI.getRegClassForReg(MRI, Reg));
8810	}
8811
8812	uint16_t Opc = MI.getOpcode();
8813	// FIXME: Copies inserted in the block prolog for live-range split should also
8814	// be included.
8815	return IsNullOrVectorRegister &&
8816	(isSpillOpcode(Opc) || (!MI.isTerminator() && Opc != AMDGPU::COPY &&
8817	MI.modifiesRegister(AMDGPU::EXEC, &RI)));
8818	}
8819
8820	MachineInstrBuilder
8821	SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
8822	MachineBasicBlock::iterator I,
8823	const DebugLoc &DL,
8824	Register DestReg) const {
8825	if (ST.hasAddNoCarry())
8826	return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
8827
8828	MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8829	Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC());
8830	MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
8831
8832	return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)
8833	.addReg(UnusedCarry, RegState::Define | RegState::Dead);
8834	}
8835
8836	MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
8837	MachineBasicBlock::iterator I,
8838	const DebugLoc &DL,
8839	Register DestReg,
8840	RegScavenger &RS) const {
8841	if (ST.hasAddNoCarry())
8842	return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg);
8843
8844	// If available, prefer to use vcc.
8845	Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC)
8846	? Register(RI.getVCC())
8847	: RS.scavengeRegisterBackwards(
8848	*RI.getBoolRC(), I, /* RestoreAfter */ false,
8849	0, /* AllowSpill */ false);
8850
8851	// TODO: Users need to deal with this.
8852	if (!UnusedCarry.isValid())
8853	return MachineInstrBuilder();
8854
8855	return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg)
8856	.addReg(UnusedCarry, RegState::Define | RegState::Dead);
8857	}
8858
8859	bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
8860	switch (Opcode) {
8861	case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
8862	case AMDGPU::SI_KILL_I1_TERMINATOR:
8863	return true;
8864	default:
8865	return false;
8866	}
8867	}
8868
8869	const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
8870	switch (Opcode) {
8871	case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
8872	return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
8873	case AMDGPU::SI_KILL_I1_PSEUDO:
8874	return get(AMDGPU::SI_KILL_I1_TERMINATOR);
8875	default:
8876	llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
8877	}
8878	}
8879
8880	bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
8881	return Imm <= getMaxMUBUFImmOffset(ST);
8882	}
8883
8884	unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
8885	// GFX12 field is non-negative 24-bit signed byte offset.
8886	const unsigned OffsetBits =
8887	ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
8888	return (1 << OffsetBits) - 1;
8889	}
8890
8891	void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
8892	if (!ST.isWave32())
8893	return;
8894
8895	if (MI.isInlineAsm())
8896	return;
8897
8898	for (auto &Op : MI.implicit_operands()) {
8899	if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
8900	Op.setReg(AMDGPU::VCC_LO);
8901	}
8902	}
8903
8904	bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
8905	if (!isSMRD(MI))
8906	return false;
8907
8908	// Check that it is using a buffer resource.
8909	int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
8910	if (Idx == -1) // e.g. s_memtime
8911	return false;
8912
8913	const auto RCID = MI.getDesc().operands()[Idx].RegClass;
8914	return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass);
8915	}
8916
8917	// Given Imm, split it into the values to put into the SOffset and ImmOffset
8918	// fields in an MUBUF instruction. Return false if it is not possible (due to a
8919	// hardware bug needing a workaround).
8920	//
8921	// The required alignment ensures that individual address components remain
8922	// aligned if they are aligned to begin with. It also ensures that additional
8923	// offsets within the given alignment can be added to the resulting ImmOffset.
8924	bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
8925	uint32_t &ImmOffset, Align Alignment) const {
8926	const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
8927	const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
8928	uint32_t Overflow = 0;
8929
8930	if (Imm > MaxImm) {
8931	if (Imm <= MaxImm + 64) {
8932	// Use an SOffset inline constant for 4..64
8933	Overflow = Imm - MaxImm;
8934	Imm = MaxImm;
8935	} else {
8936	// Try to keep the same value in SOffset for adjacent loads, so that
8937	// the corresponding register contents can be re-used.
8938	//
8939	// Load values with all low-bits (except for alignment bits) set into
8940	// SOffset, so that a larger range of values can be covered using
8941	// s_movk_i32.
8942	//
8943	// Atomic operations fail to work correctly when individual address
8944	// components are unaligned, even if their sum is aligned.
8945	uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
8946	uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
8947	Imm = Low;
8948	Overflow = High - Alignment.value();
8949	}
8950	}
8951
8952	if (Overflow > 0) {
8953	// There is a hardware bug in SI and CI which prevents address clamping in
8954	// MUBUF instructions from working correctly with SOffsets. The immediate
8955	// offset is unaffected.
8956	if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
8957	return false;
8958
8959	// It is not possible to set immediate in SOffset field on some targets.
8960	if (ST.hasRestrictedSOffset())
8961	return false;
8962	}
8963
8964	ImmOffset = Imm;
8965	SOffset = Overflow;
8966	return true;
8967	}
8968
8969	// Depending on the used address space and instructions, some immediate offsets
8970	// are allowed and some are not.
8971	// Pre-GFX12, flat instruction offsets can only be non-negative, global and
8972	// scratch instruction offsets can also be negative. On GFX12, offsets can be
8973	// negative for all variants.
8974	//
8975	// There are several bugs related to these offsets:
8976	// On gfx10.1, flat instructions that go into the global address space cannot
8977	// use an offset.
8978	//
8979	// For scratch instructions, the address can be either an SGPR or a VGPR.
8980	// The following offsets can be used, depending on the architecture (x means
8981	// cannot be used):
8982	// +----------------------------+------+------+
8983	// | Address-Mode | SGPR | VGPR |
8984	// +----------------------------+------+------+
8985	// | gfx9 | | |
8986	// | negative, 4-aligned offset | x | ok |
8987	// | negative, unaligned offset | x | ok |
8988	// +----------------------------+------+------+
8989	// | gfx10 | | |
8990	// | negative, 4-aligned offset | ok | ok |
8991	// | negative, unaligned offset | ok | x |
8992	// +----------------------------+------+------+
8993	// | gfx10.3 | | |
8994	// | negative, 4-aligned offset | ok | ok |
8995	// | negative, unaligned offset | ok | ok |
8996	// +----------------------------+------+------+
8997	//
8998	// This function ignores the addressing mode, so if an offset cannot be used in
8999	// one addressing mode, it is considered illegal.
9000	bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
9001	uint64_t FlatVariant) const {
9002	// TODO: Should 0 be special cased?
9003	if (!ST.hasFlatInstOffsets())
9004	return false;
9005
9006	if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT &&
9007	(AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
9008	AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))
9009	return false;
9010
9011	if (ST.hasNegativeUnalignedScratchOffsetBug() &&
9012	FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 &&
9013	(Offset % 4) != 0) {
9014	return false;
9015	}
9016
9017	bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
9018	unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
9019	return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
9020	}
9021
9022	// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
9023	std::pair<int64_t, int64_t>
9024	SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
9025	uint64_t FlatVariant) const {
9026	int64_t RemainderOffset = COffsetVal;
9027	int64_t ImmField = 0;
9028
9029	bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
9030	const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
9031
9032	if (AllowNegative) {
9033	// Use signed division by a power of two to truncate towards 0.
9034	int64_t D = 1LL << NumBits;
9035	RemainderOffset = (COffsetVal / D) * D;
9036	ImmField = COffsetVal - RemainderOffset;
9037
9038	if (ST.hasNegativeUnalignedScratchOffsetBug() &&
9039	FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 &&
9040	(ImmField % 4) != 0) {
9041	// Make ImmField a multiple of 4
9042	RemainderOffset += ImmField % 4;
9043	ImmField -= ImmField % 4;
9044	}
9045	} else if (COffsetVal >= 0) {
9046	ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
9047	RemainderOffset = COffsetVal - ImmField;
9048	}
9049
9050	assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
9051	assert(RemainderOffset + ImmField == COffsetVal);
9052	return {ImmField, RemainderOffset};
9053	}
9054
9055	bool SIInstrInfo::allowNegativeFlatOffset(uint64_t FlatVariant) const {
9056	if (ST.hasNegativeScratchOffsetBug() &&
9057	FlatVariant == SIInstrFlags::FlatScratch)
9058	return false;
9059
9060	return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(ST);
9061	}
9062
9063	static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
9064	switch (ST.getGeneration()) {
9065	default:
9066	break;
9067	case AMDGPUSubtarget::SOUTHERN_ISLANDS:
9068	case AMDGPUSubtarget::SEA_ISLANDS:
9069	return SIEncodingFamily::SI;
9070	case AMDGPUSubtarget::VOLCANIC_ISLANDS:
9071	case AMDGPUSubtarget::GFX9:
9072	return SIEncodingFamily::VI;
9073	case AMDGPUSubtarget::GFX10:
9074	return SIEncodingFamily::GFX10;
9075	case AMDGPUSubtarget::GFX11:
9076	return SIEncodingFamily::GFX11;
9077	case AMDGPUSubtarget::GFX12:
9078	return SIEncodingFamily::GFX12;
9079	}
9080	llvm_unreachable("Unknown subtarget generation!");
9081	}
9082
9083	bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
9084	switch(MCOp) {
9085	// These opcodes use indirect register addressing so
9086	// they need special handling by codegen (currently missing).
9087	// Therefore it is too risky to allow these opcodes
9088	// to be selected by dpp combiner or sdwa peepholer.
9089	case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
9090	case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
9091	case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
9092	case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
9093	case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
9094	case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
9095	case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
9096	case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
9097	return true;
9098	default:
9099	return false;
9100	}
9101	}
9102
9103	int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
9104	Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode);
9105
9106	unsigned Gen = subtargetEncodingFamily(ST);
9107
9108	if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
9109	ST.getGeneration() == AMDGPUSubtarget::GFX9)
9110	Gen = SIEncodingFamily::GFX9;
9111
9112	// Adjust the encoding family to GFX80 for D16 buffer instructions when the
9113	// subtarget has UnpackedD16VMem feature.
9114	// TODO: remove this when we discard GFX80 encoding.
9115	if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))
9116	Gen = SIEncodingFamily::GFX80;
9117
9118	if (get(Opcode).TSFlags & SIInstrFlags::SDWA) {
9119	switch (ST.getGeneration()) {
9120	default:
9121	Gen = SIEncodingFamily::SDWA;
9122	break;
9123	case AMDGPUSubtarget::GFX9:
9124	Gen = SIEncodingFamily::SDWA9;
9125	break;
9126	case AMDGPUSubtarget::GFX10:
9127	Gen = SIEncodingFamily::SDWA10;
9128	break;
9129	}
9130	}
9131
9132	if (isMAI(Opcode)) {
9133	int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
9134	if (MFMAOp != -1)
9135	Opcode = MFMAOp;
9136	}
9137
9138	int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
9139
9140	// -1 means that Opcode is already a native instruction.
9141	if (MCOp == -1)
9142	return Opcode;
9143
9144	if (ST.hasGFX90AInsts()) {
9145	uint16_t NMCOp = (uint16_t)-1;
9146	if (ST.hasGFX940Insts())
9147	NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
9148	if (NMCOp == (uint16_t)-1)
9149	NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
9150	if (NMCOp == (uint16_t)-1)
9151	NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
9152	if (NMCOp != (uint16_t)-1)
9153	MCOp = NMCOp;
9154	}
9155
9156	// (uint16_t)-1 means that Opcode is a pseudo instruction that has
9157	// no encoding in the given subtarget generation.
9158	if (MCOp == (uint16_t)-1)
9159	return -1;
9160
9161	if (isAsmOnlyOpcode(MCOp))
9162	return -1;
9163
9164	return MCOp;
9165	}
9166
9167	static
9168	TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
9169	assert(RegOpnd.isReg());
9170	return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
9171	getRegSubRegPair(O: RegOpnd);
9172	}
9173
9174	TargetInstrInfo::RegSubRegPair
9175	llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
9176	assert(MI.isRegSequence());
9177	for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
9178	if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
9179	auto &RegOp = MI.getOperand(i: 1 + 2 * I);
9180	return getRegOrUndef(RegOpnd: RegOp);
9181	}
9182	return TargetInstrInfo::RegSubRegPair();
9183	}
9184
9185	// Try to find the definition of reg:subreg in subreg-manipulation pseudos
9186	// Following a subreg of reg:subreg isn't supported
9187	static bool followSubRegDef(MachineInstr &MI,
9188	TargetInstrInfo::RegSubRegPair &RSR) {
9189	if (!RSR.SubReg)
9190	return false;
9191	switch (MI.getOpcode()) {
9192	default: break;
9193	case AMDGPU::REG_SEQUENCE:
9194	RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
9195	return true;
9196	// EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
9197	case AMDGPU::INSERT_SUBREG:
9198	if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
9199	// inserted the subreg we're looking for
9200	RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
9201	else { // the subreg in the rest of the reg
9202	auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
9203	if (R1.SubReg) // subreg of subreg isn't supported
9204	return false;
9205	RSR.Reg = R1.Reg;
9206	}
9207	return true;
9208	}
9209	return false;
9210	}
9211
9212	MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
9213	MachineRegisterInfo &MRI) {
9214	assert(MRI.isSSA());
9215	if (!P.Reg.isVirtual())
9216	return nullptr;
9217
9218	auto RSR = P;
9219	auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
9220	while (auto *MI = DefInst) {
9221	DefInst = nullptr;
9222	switch (MI->getOpcode()) {
9223	case AMDGPU::COPY:
9224	case AMDGPU::V_MOV_B32_e32: {
9225	auto &Op1 = MI->getOperand(i: 1);
9226	if (Op1.isReg() && Op1.getReg().isVirtual()) {
9227	if (Op1.isUndef())
9228	return nullptr;
9229	RSR = getRegSubRegPair(O: Op1);
9230	DefInst = MRI.getVRegDef(Reg: RSR.Reg);
9231	}
9232	break;
9233	}
9234	default:
9235	if (followSubRegDef(MI&: *MI, RSR)) {
9236	if (!RSR.Reg)
9237	return nullptr;
9238	DefInst = MRI.getVRegDef(Reg: RSR.Reg);
9239	}
9240	}
9241	if (!DefInst)
9242	return MI;
9243	}
9244	return nullptr;
9245	}
9246
9247	bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
9248	Register VReg,
9249	const MachineInstr &DefMI,
9250	const MachineInstr &UseMI) {
9251	assert(MRI.isSSA() && "Must be run on SSA");
9252
9253	auto *TRI = MRI.getTargetRegisterInfo();
9254	auto *DefBB = DefMI.getParent();
9255
9256	// Don't bother searching between blocks, although it is possible this block
9257	// doesn't modify exec.
9258	if (UseMI.getParent() != DefBB)
9259	return true;
9260
9261	const int MaxInstScan = 20;
9262	int NumInst = 0;
9263
9264	// Stop scan at the use.
9265	auto E = UseMI.getIterator();
9266	for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
9267	if (I->isDebugInstr())
9268	continue;
9269
9270	if (++NumInst > MaxInstScan)
9271	return true;
9272
9273	if (I->modifiesRegister(AMDGPU::EXEC, TRI))
9274	return true;
9275	}
9276
9277	return false;
9278	}
9279
9280	bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
9281	Register VReg,
9282	const MachineInstr &DefMI) {
9283	assert(MRI.isSSA() && "Must be run on SSA");
9284
9285	auto *TRI = MRI.getTargetRegisterInfo();
9286	auto *DefBB = DefMI.getParent();
9287
9288	const int MaxUseScan = 10;
9289	int NumUse = 0;
9290
9291	for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
9292	auto &UseInst = *Use.getParent();
9293	// Don't bother searching between blocks, although it is possible this block
9294	// doesn't modify exec.
9295	if (UseInst.getParent() != DefBB || UseInst.isPHI())
9296	return true;
9297
9298	if (++NumUse > MaxUseScan)
9299	return true;
9300	}
9301
9302	if (NumUse == 0)
9303	return false;
9304
9305	const int MaxInstScan = 20;
9306	int NumInst = 0;
9307
9308	// Stop scan when we have seen all the uses.
9309	for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
9310	assert(I != DefBB->end());
9311
9312	if (I->isDebugInstr())
9313	continue;
9314
9315	if (++NumInst > MaxInstScan)
9316	return true;
9317
9318	for (const MachineOperand &Op : I->operands()) {
9319	// We don't check reg masks here as they're used only on calls:
9320	// 1. EXEC is only considered const within one BB
9321	// 2. Call should be a terminator instruction if present in a BB
9322
9323	if (!Op.isReg())
9324	continue;
9325
9326	Register Reg = Op.getReg();
9327	if (Op.isUse()) {
9328	if (Reg == VReg && --NumUse == 0)
9329	return false;
9330	} else if (TRI->regsOverlap(Reg, AMDGPU::EXEC))
9331	return true;
9332	}
9333	}
9334	}
9335
9336	MachineInstr *SIInstrInfo::createPHIDestinationCopy(
9337	MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
9338	const DebugLoc &DL, Register Src, Register Dst) const {
9339	auto Cur = MBB.begin();
9340	if (Cur != MBB.end())
9341	do {
9342	if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst))
9343	return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src);
9344	++Cur;
9345	} while (Cur != MBB.end() && Cur != LastPHIIt);
9346
9347	return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src,
9348	Dst);
9349	}
9350
9351	MachineInstr *SIInstrInfo::createPHISourceCopy(
9352	MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
9353	const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
9354	if (InsPt != MBB.end() &&
9355	(InsPt->getOpcode() == AMDGPU::SI_IF ||
9356	InsPt->getOpcode() == AMDGPU::SI_ELSE ||
9357	InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
9358	InsPt->definesRegister(Src)) {
9359	InsPt++;
9360	return BuildMI(MBB, InsPt, DL,
9361	get(ST.isWave32() ? AMDGPU::S_MOV_B32_term
9362	: AMDGPU::S_MOV_B64_term),
9363	Dst)
9364	.addReg(Src, 0, SrcSubReg)
9365	.addReg(AMDGPU::EXEC, RegState::Implicit);
9366	}
9367	return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
9368	Dst);
9369	}
9370
9371	bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
9372
9373	MachineInstr *SIInstrInfo::foldMemoryOperandImpl(
9374	MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
9375	MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
9376	VirtRegMap *VRM) const {
9377	// This is a bit of a hack (copied from AArch64). Consider this instruction:
9378	//
9379	// %0:sreg_32 = COPY $m0
9380	//
9381	// We explicitly chose SReg_32 for the virtual register so such a copy might
9382	// be eliminated by RegisterCoalescer. However, that may not be possible, and
9383	// %0 may even spill. We can't spill $m0 normally (it would require copying to
9384	// a numbered SGPR anyway), and since it is in the SReg_32 register class,
9385	// TargetInstrInfo::foldMemoryOperand() is going to try.
9386	// A similar issue also exists with spilling and reloading $exec registers.
9387	//
9388	// To prevent that, constrain the %0 register class here.
9389	if (isFullCopyInstr(MI)) {
9390	Register DstReg = MI.getOperand(i: 0).getReg();
9391	Register SrcReg = MI.getOperand(i: 1).getReg();
9392	if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
9393	(DstReg.isVirtual() != SrcReg.isVirtual())) {
9394	MachineRegisterInfo &MRI = MF.getRegInfo();
9395	Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
9396	const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
9397	if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) {
9398	MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
9399	return nullptr;
9400	} else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) {
9401	MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass);
9402	return nullptr;
9403	}
9404	}
9405	}
9406
9407	return nullptr;
9408	}
9409
9410	unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
9411	const MachineInstr &MI,
9412	unsigned *PredCost) const {
9413	if (MI.isBundle()) {
9414	MachineBasicBlock::const_instr_iterator I(MI.getIterator());
9415	MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
9416	unsigned Lat = 0, Count = 0;
9417	for (++I; I != E && I->isBundledWithPred(); ++I) {
9418	++Count;
9419	Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
9420	}
9421	return Lat + Count - 1;
9422	}
9423
9424	return SchedModel.computeInstrLatency(MI: &MI);
9425	}
9426
9427	InstructionUniformity
9428	SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {
9429	unsigned opcode = MI.getOpcode();
9430	if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
9431	auto IID = GI->getIntrinsicID();
9432	if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
9433	return InstructionUniformity::NeverUniform;
9434	if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
9435	return InstructionUniformity::AlwaysUniform;
9436
9437	switch (IID) {
9438	case Intrinsic::amdgcn_if:
9439	case Intrinsic::amdgcn_else:
9440	// FIXME: Uniform if second result
9441	break;
9442	}
9443
9444	return InstructionUniformity::Default;
9445	}
9446
9447	// Loads from the private and flat address spaces are divergent, because
9448	// threads can execute the load instruction with the same inputs and get
9449	// different results.
9450	//
9451	// All other loads are not divergent, because if threads issue loads with the
9452	// same arguments, they will always get the same result.
9453	if (opcode == AMDGPU::G_LOAD) {
9454	if (MI.memoperands_empty())
9455	return InstructionUniformity::NeverUniform; // conservative assumption
9456
9457	if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
9458	return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
9459	mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
9460	})) {
9461	// At least one MMO in a non-global address space.
9462	return InstructionUniformity::NeverUniform;
9463	}
9464	return InstructionUniformity::Default;
9465	}
9466
9467	if (SIInstrInfo::isGenericAtomicRMWOpcode(opcode) ||
9468	opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
9469	opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
9470	AMDGPU::isGenericAtomic(opcode)) {
9471	return InstructionUniformity::NeverUniform;
9472	}
9473	return InstructionUniformity::Default;
9474	}
9475
9476	InstructionUniformity
9477	SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const {
9478
9479	if (isNeverUniform(MI))
9480	return InstructionUniformity::NeverUniform;
9481
9482	unsigned opcode = MI.getOpcode();
9483	if (opcode == AMDGPU::V_READLANE_B32 ||
9484	opcode == AMDGPU::V_READFIRSTLANE_B32 ||
9485	opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
9486	return InstructionUniformity::AlwaysUniform;
9487
9488	if (isCopyInstr(MI)) {
9489	const MachineOperand &srcOp = MI.getOperand(i: 1);
9490	if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
9491	const TargetRegisterClass *regClass =
9492	RI.getPhysRegBaseClass(srcOp.getReg());
9493	return RI.isSGPRClass(regClass) ? InstructionUniformity::AlwaysUniform
9494	: InstructionUniformity::NeverUniform;
9495	}
9496	return InstructionUniformity::Default;
9497	}
9498
9499	// GMIR handling
9500	if (MI.isPreISelOpcode())
9501	return SIInstrInfo::getGenericInstructionUniformity(MI);
9502
9503	// Atomics are divergent because they are executed sequentially: when an
9504	// atomic operation refers to the same address in each thread, then each
9505	// thread after the first sees the value written by the previous thread as
9506	// original value.
9507
9508	if (isAtomic(MI))
9509	return InstructionUniformity::NeverUniform;
9510
9511	// Loads from the private and flat address spaces are divergent, because
9512	// threads can execute the load instruction with the same inputs and get
9513	// different results.
9514	if (isFLAT(MI) && MI.mayLoad()) {
9515	if (MI.memoperands_empty())
9516	return InstructionUniformity::NeverUniform; // conservative assumption
9517
9518	if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
9519	return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
9520	mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
9521	})) {
9522	// At least one MMO in a non-global address space.
9523	return InstructionUniformity::NeverUniform;
9524	}
9525
9526	return InstructionUniformity::Default;
9527	}
9528
9529	const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
9530	const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
9531
9532	// FIXME: It's conceptually broken to report this for an instruction, and not
9533	// a specific def operand. For inline asm in particular, there could be mixed
9534	// uniform and divergent results.
9535	for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
9536	const MachineOperand &SrcOp = MI.getOperand(i: I);
9537	if (!SrcOp.isReg())
9538	continue;
9539
9540	Register Reg = SrcOp.getReg();
9541	if (!Reg || !SrcOp.readsReg())
9542	continue;
9543
9544	// If RegBank is null, this is unassigned or an unallocatable special
9545	// register, which are all scalars.
9546	const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, RI);
9547	if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
9548	return InstructionUniformity::NeverUniform;
9549	}
9550
9551	// TODO: Uniformity check condtions above can be rearranged for more
9552	// redability
9553
9554	// TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
9555	// currently turned into no-op COPYs by SelectionDAG ISel and are
9556	// therefore no longer recognizable.
9557
9558	return InstructionUniformity::Default;
9559	}
9560
9561	unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
9562	switch (MF.getFunction().getCallingConv()) {
9563	case CallingConv::AMDGPU_PS:
9564	return 1;
9565	case CallingConv::AMDGPU_VS:
9566	return 2;
9567	case CallingConv::AMDGPU_GS:
9568	return 3;
9569	case CallingConv::AMDGPU_HS:
9570	case CallingConv::AMDGPU_LS:
9571	case CallingConv::AMDGPU_ES:
9572	report_fatal_error(reason: "ds_ordered_count unsupported for this calling conv");
9573	case CallingConv::AMDGPU_CS:
9574	case CallingConv::AMDGPU_KERNEL:
9575	case CallingConv::C:
9576	case CallingConv::Fast:
9577	default:
9578	// Assume other calling conventions are various compute callable functions
9579	return 0;
9580	}
9581	}
9582
9583	bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
9584	Register &SrcReg2, int64_t &CmpMask,
9585	int64_t &CmpValue) const {
9586	if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
9587	return false;
9588
9589	switch (MI.getOpcode()) {
9590	default:
9591	break;
9592	case AMDGPU::S_CMP_EQ_U32:
9593	case AMDGPU::S_CMP_EQ_I32:
9594	case AMDGPU::S_CMP_LG_U32:
9595	case AMDGPU::S_CMP_LG_I32:
9596	case AMDGPU::S_CMP_LT_U32:
9597	case AMDGPU::S_CMP_LT_I32:
9598	case AMDGPU::S_CMP_GT_U32:
9599	case AMDGPU::S_CMP_GT_I32:
9600	case AMDGPU::S_CMP_LE_U32:
9601	case AMDGPU::S_CMP_LE_I32:
9602	case AMDGPU::S_CMP_GE_U32:
9603	case AMDGPU::S_CMP_GE_I32:
9604	case AMDGPU::S_CMP_EQ_U64:
9605	case AMDGPU::S_CMP_LG_U64:
9606	SrcReg = MI.getOperand(i: 0).getReg();
9607	if (MI.getOperand(i: 1).isReg()) {
9608	if (MI.getOperand(i: 1).getSubReg())
9609	return false;
9610	SrcReg2 = MI.getOperand(i: 1).getReg();
9611	CmpValue = 0;
9612	} else if (MI.getOperand(i: 1).isImm()) {
9613	SrcReg2 = Register();
9614	CmpValue = MI.getOperand(i: 1).getImm();
9615	} else {
9616	return false;
9617	}
9618	CmpMask = ~0;
9619	return true;
9620	case AMDGPU::S_CMPK_EQ_U32:
9621	case AMDGPU::S_CMPK_EQ_I32:
9622	case AMDGPU::S_CMPK_LG_U32:
9623	case AMDGPU::S_CMPK_LG_I32:
9624	case AMDGPU::S_CMPK_LT_U32:
9625	case AMDGPU::S_CMPK_LT_I32:
9626	case AMDGPU::S_CMPK_GT_U32:
9627	case AMDGPU::S_CMPK_GT_I32:
9628	case AMDGPU::S_CMPK_LE_U32:
9629	case AMDGPU::S_CMPK_LE_I32:
9630	case AMDGPU::S_CMPK_GE_U32:
9631	case AMDGPU::S_CMPK_GE_I32:
9632	SrcReg = MI.getOperand(i: 0).getReg();
9633	SrcReg2 = Register();
9634	CmpValue = MI.getOperand(i: 1).getImm();
9635	CmpMask = ~0;
9636	return true;
9637	}
9638
9639	return false;
9640	}
9641
9642	bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
9643	Register SrcReg2, int64_t CmpMask,
9644	int64_t CmpValue,
9645	const MachineRegisterInfo *MRI) const {
9646	if (!SrcReg || SrcReg.isPhysical())
9647	return false;
9648
9649	if (SrcReg2 && !getFoldableImm(Reg: SrcReg2, MRI: *MRI, Imm&: CmpValue))
9650	return false;
9651
9652	const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
9653	this](int64_t ExpectedValue, unsigned SrcSize,
9654	bool IsReversible, bool IsSigned) -> bool {
9655	// s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9656	// s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9657	// s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9658	// s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9659	// s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
9660	// s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9661	// s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9662	// s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9663	// s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9664	// s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
9665	//
9666	// Signed ge/gt are not used for the sign bit.
9667	//
9668	// If result of the AND is unused except in the compare:
9669	// s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
9670	//
9671	// s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
9672	// s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
9673	// s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
9674	// s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
9675	// s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
9676	// s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
9677
9678	MachineInstr *Def = MRI->getUniqueVRegDef(Reg: SrcReg);
9679	if (!Def || Def->getParent() != CmpInstr.getParent())
9680	return false;
9681
9682	if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
9683	Def->getOpcode() != AMDGPU::S_AND_B64)
9684	return false;
9685
9686	int64_t Mask;
9687	const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool {
9688	if (MO->isImm())
9689	Mask = MO->getImm();
9690	else if (!getFoldableImm(MO, Imm&: Mask))
9691	return false;
9692	Mask &= maxUIntN(N: SrcSize);
9693	return isPowerOf2_64(Value: Mask);
9694	};
9695
9696	MachineOperand *SrcOp = &Def->getOperand(i: 1);
9697	if (isMask(SrcOp))
9698	SrcOp = &Def->getOperand(i: 2);
9699	else if (isMask(&Def->getOperand(i: 2)))
9700	SrcOp = &Def->getOperand(i: 1);
9701	else
9702	return false;
9703
9704	unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
9705	if (IsSigned && BitNo == SrcSize - 1)
9706	return false;
9707
9708	ExpectedValue <<= BitNo;
9709
9710	bool IsReversedCC = false;
9711	if (CmpValue != ExpectedValue) {
9712	if (!IsReversible)
9713	return false;
9714	IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
9715	if (!IsReversedCC)
9716	return false;
9717	}
9718
9719	Register DefReg = Def->getOperand(i: 0).getReg();
9720	if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
9721	return false;
9722
9723	for (auto I = std::next(x: Def->getIterator()), E = CmpInstr.getIterator();
9724	I != E; ++I) {
9725	if (I->modifiesRegister(AMDGPU::SCC, &RI) ||
9726	I->killsRegister(AMDGPU::SCC, &RI))
9727	return false;
9728	}
9729
9730	MachineOperand *SccDef = Def->findRegisterDefOperand(AMDGPU::SCC);
9731	SccDef->setIsDead(false);
9732	CmpInstr.eraseFromParent();
9733
9734	if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
9735	assert(!IsReversedCC);
9736	return true;
9737	}
9738
9739	// Replace AND with unused result with a S_BITCMP.
9740	MachineBasicBlock *MBB = Def->getParent();
9741
9742	unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
9743	: AMDGPU::S_BITCMP1_B32
9744	: IsReversedCC ? AMDGPU::S_BITCMP0_B64
9745	: AMDGPU::S_BITCMP1_B64;
9746
9747	BuildMI(*MBB, Def, Def->getDebugLoc(), get(NewOpc))
9748	.add(*SrcOp)
9749	.addImm(BitNo);
9750	Def->eraseFromParent();
9751
9752	return true;
9753	};
9754
9755	switch (CmpInstr.getOpcode()) {
9756	default:
9757	break;
9758	case AMDGPU::S_CMP_EQ_U32:
9759	case AMDGPU::S_CMP_EQ_I32:
9760	case AMDGPU::S_CMPK_EQ_U32:
9761	case AMDGPU::S_CMPK_EQ_I32:
9762	return optimizeCmpAnd(1, 32, true, false);
9763	case AMDGPU::S_CMP_GE_U32:
9764	case AMDGPU::S_CMPK_GE_U32:
9765	return optimizeCmpAnd(1, 32, false, false);
9766	case AMDGPU::S_CMP_GE_I32:
9767	case AMDGPU::S_CMPK_GE_I32:
9768	return optimizeCmpAnd(1, 32, false, true);
9769	case AMDGPU::S_CMP_EQ_U64:
9770	return optimizeCmpAnd(1, 64, true, false);
9771	case AMDGPU::S_CMP_LG_U32:
9772	case AMDGPU::S_CMP_LG_I32:
9773	case AMDGPU::S_CMPK_LG_U32:
9774	case AMDGPU::S_CMPK_LG_I32:
9775	return optimizeCmpAnd(0, 32, true, false);
9776	case AMDGPU::S_CMP_GT_U32:
9777	case AMDGPU::S_CMPK_GT_U32:
9778	return optimizeCmpAnd(0, 32, false, false);
9779	case AMDGPU::S_CMP_GT_I32:
9780	case AMDGPU::S_CMPK_GT_I32:
9781	return optimizeCmpAnd(0, 32, false, true);
9782	case AMDGPU::S_CMP_LG_U64:
9783	return optimizeCmpAnd(0, 64, true, false);
9784	}
9785
9786	return false;
9787	}
9788
9789	void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
9790	unsigned OpName) const {
9791	if (!ST.needsAlignedVGPRs())
9792	return;
9793
9794	int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: OpName);
9795	if (OpNo < 0)
9796	return;
9797	MachineOperand &Op = MI.getOperand(i: OpNo);
9798	if (getOpSize(MI, OpNo) > 4)
9799	return;
9800
9801	// Add implicit aligned super-reg to force alignment on the data operand.
9802	const DebugLoc &DL = MI.getDebugLoc();
9803	MachineBasicBlock *BB = MI.getParent();
9804	MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9805	Register DataReg = Op.getReg();
9806	bool IsAGPR = RI.isAGPR(MRI, DataReg);
9807	Register Undef = MRI.createVirtualRegister(
9808	IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
9809	BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
9810	Register NewVR =
9811	MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
9812	: &AMDGPU::VReg_64_Align2RegClass);
9813	BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR)
9814	.addReg(DataReg, 0, Op.getSubReg())
9815	.addImm(AMDGPU::sub0)
9816	.addReg(Undef)
9817	.addImm(AMDGPU::sub1);
9818	Op.setReg(NewVR);
9819	Op.setSubReg(AMDGPU::sub0);
9820	MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
9821	}
9822