1	//===- lib/CodeGen/MachineInstr.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Methods common to all machine instructions.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "llvm/CodeGen/MachineInstr.h"
14	#include "llvm/ADT/ArrayRef.h"
15	#include "llvm/ADT/Hashing.h"
16	#include "llvm/ADT/STLExtras.h"
17	#include "llvm/ADT/SmallBitVector.h"
18	#include "llvm/ADT/SmallVector.h"
19	#include "llvm/Analysis/AliasAnalysis.h"
20	#include "llvm/Analysis/MemoryLocation.h"
21	#include "llvm/CodeGen/MachineBasicBlock.h"
22	#include "llvm/CodeGen/MachineFrameInfo.h"
23	#include "llvm/CodeGen/MachineFunction.h"
24	#include "llvm/CodeGen/MachineInstrBuilder.h"
25	#include "llvm/CodeGen/MachineInstrBundle.h"
26	#include "llvm/CodeGen/MachineMemOperand.h"
27	#include "llvm/CodeGen/MachineModuleInfo.h"
28	#include "llvm/CodeGen/MachineOperand.h"
29	#include "llvm/CodeGen/MachineRegisterInfo.h"
30	#include "llvm/CodeGen/PseudoSourceValue.h"
31	#include "llvm/CodeGen/Register.h"
32	#include "llvm/CodeGen/StackMaps.h"
33	#include "llvm/CodeGen/TargetInstrInfo.h"
34	#include "llvm/CodeGen/TargetRegisterInfo.h"
35	#include "llvm/CodeGen/TargetSubtargetInfo.h"
36	#include "llvm/CodeGenTypes/LowLevelType.h"
37	#include "llvm/IR/Constants.h"
38	#include "llvm/IR/DebugInfoMetadata.h"
39	#include "llvm/IR/DebugLoc.h"
40	#include "llvm/IR/Function.h"
41	#include "llvm/IR/InlineAsm.h"
42	#include "llvm/IR/Instructions.h"
43	#include "llvm/IR/LLVMContext.h"
44	#include "llvm/IR/Metadata.h"
45	#include "llvm/IR/Module.h"
46	#include "llvm/IR/ModuleSlotTracker.h"
47	#include "llvm/IR/Operator.h"
48	#include "llvm/MC/MCInstrDesc.h"
49	#include "llvm/MC/MCRegisterInfo.h"
50	#include "llvm/Support/Casting.h"
51	#include "llvm/Support/Compiler.h"
52	#include "llvm/Support/Debug.h"
53	#include "llvm/Support/ErrorHandling.h"
54	#include "llvm/Support/FormattedStream.h"
55	#include "llvm/Support/raw_ostream.h"
56	#include "llvm/Target/TargetMachine.h"
57	#include <algorithm>
58	#include <cassert>
59	#include <cstdint>
60	#include <cstring>
61	#include <utility>
62
63	using namespace llvm;
64
65	static const MachineFunction *getMFIfAvailable(const MachineInstr &MI) {
66	if (const MachineBasicBlock *MBB = MI.getParent())
67	if (const MachineFunction *MF = MBB->getParent())
68	return MF;
69	return nullptr;
70	}
71
72	// Try to crawl up to the machine function and get TRI and IntrinsicInfo from
73	// it.
74	static void tryToGetTargetInfo(const MachineInstr &MI,
75	const TargetRegisterInfo *&TRI,
76	const MachineRegisterInfo *&MRI,
77	const TargetIntrinsicInfo *&IntrinsicInfo,
78	const TargetInstrInfo *&TII) {
79
80	if (const MachineFunction *MF = getMFIfAvailable(MI)) {
81	TRI = MF->getSubtarget().getRegisterInfo();
82	MRI = &MF->getRegInfo();
83	IntrinsicInfo = MF->getTarget().getIntrinsicInfo();
84	TII = MF->getSubtarget().getInstrInfo();
85	}
86	}
87
88	void MachineInstr::addImplicitDefUseOperands(MachineFunction &MF) {
89	for (MCPhysReg ImpDef : MCID->implicit_defs())
90	addOperand(MF, Op: MachineOperand::CreateReg(Reg: ImpDef, isDef: true, isImp: true));
91	for (MCPhysReg ImpUse : MCID->implicit_uses())
92	addOperand(MF, Op: MachineOperand::CreateReg(Reg: ImpUse, isDef: false, isImp: true));
93	}
94
95	/// MachineInstr ctor - This constructor creates a MachineInstr and adds the
96	/// implicit operands. It reserves space for the number of operands specified by
97	/// the MCInstrDesc.
98	MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &TID,
99	DebugLoc DL, bool NoImp)
100	: MCID(&TID), NumOperands(0), Flags(0), AsmPrinterFlags(0),
101	DbgLoc(std::move(DL)), DebugInstrNum(0) {
102	assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor");
103
104	// Reserve space for the expected number of operands.
105	if (unsigned NumOps = MCID->getNumOperands() + MCID->implicit_defs().size() +
106	MCID->implicit_uses().size()) {
107	CapOperands = OperandCapacity::get(N: NumOps);
108	Operands = MF.allocateOperandArray(Cap: CapOperands);
109	}
110
111	if (!NoImp)
112	addImplicitDefUseOperands(MF);
113	}
114
115	/// MachineInstr ctor - Copies MachineInstr arg exactly.
116	/// Does not copy the number from debug instruction numbering, to preserve
117	/// uniqueness.
118	MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI)
119	: MCID(&MI.getDesc()), NumOperands(0), Flags(0), AsmPrinterFlags(0),
120	Info(MI.Info), DbgLoc(MI.getDebugLoc()), DebugInstrNum(0) {
121	assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor");
122
123	CapOperands = OperandCapacity::get(N: MI.getNumOperands());
124	Operands = MF.allocateOperandArray(Cap: CapOperands);
125
126	// Copy operands.
127	for (const MachineOperand &MO : MI.operands())
128	addOperand(MF, Op: MO);
129
130	// Replicate ties between the operands, which addOperand was not
131	// able to do reliably.
132	for (unsigned i = 0, e = getNumOperands(); i < e; ++i) {
133	MachineOperand &NewMO = getOperand(i);
134	const MachineOperand &OrigMO = MI.getOperand(i);
135	NewMO.TiedTo = OrigMO.TiedTo;
136	}
137
138	// Copy all the sensible flags.
139	setFlags(MI.Flags);
140	}
141
142	void MachineInstr::setDesc(const MCInstrDesc &TID) {
143	if (getParent())
144	getMF()->handleChangeDesc(MI&: *this, TID);
145	MCID = &TID;
146	}
147
148	void MachineInstr::moveBefore(MachineInstr *MovePos) {
149	MovePos->getParent()->splice(Where: MovePos, Other: getParent(), From: getIterator());
150	}
151
152	/// getRegInfo - If this instruction is embedded into a MachineFunction,
153	/// return the MachineRegisterInfo object for the current function, otherwise
154	/// return null.
155	MachineRegisterInfo *MachineInstr::getRegInfo() {
156	if (MachineBasicBlock *MBB = getParent())
157	return &MBB->getParent()->getRegInfo();
158	return nullptr;
159	}
160
161	const MachineRegisterInfo *MachineInstr::getRegInfo() const {
162	if (const MachineBasicBlock *MBB = getParent())
163	return &MBB->getParent()->getRegInfo();
164	return nullptr;
165	}
166
167	void MachineInstr::removeRegOperandsFromUseLists(MachineRegisterInfo &MRI) {
168	for (MachineOperand &MO : operands())
169	if (MO.isReg())
170	MRI.removeRegOperandFromUseList(MO: &MO);
171	}
172
173	void MachineInstr::addRegOperandsToUseLists(MachineRegisterInfo &MRI) {
174	for (MachineOperand &MO : operands())
175	if (MO.isReg())
176	MRI.addRegOperandToUseList(MO: &MO);
177	}
178
179	void MachineInstr::addOperand(const MachineOperand &Op) {
180	MachineBasicBlock *MBB = getParent();
181	assert(MBB && "Use MachineInstrBuilder to add operands to dangling instrs");
182	MachineFunction *MF = MBB->getParent();
183	assert(MF && "Use MachineInstrBuilder to add operands to dangling instrs");
184	addOperand(MF&: *MF, Op);
185	}
186
187	/// Move NumOps MachineOperands from Src to Dst, with support for overlapping
188	/// ranges. If MRI is non-null also update use-def chains.
189	static void moveOperands(MachineOperand *Dst, MachineOperand *Src,
190	unsigned NumOps, MachineRegisterInfo *MRI) {
191	if (MRI)
192	return MRI->moveOperands(Dst, Src, NumOps);
193	// MachineOperand is a trivially copyable type so we can just use memmove.
194	assert(Dst && Src && "Unknown operands");
195	std::memmove(dest: Dst, src: Src, n: NumOps * sizeof(MachineOperand));
196	}
197
198	/// addOperand - Add the specified operand to the instruction. If it is an
199	/// implicit operand, it is added to the end of the operand list. If it is
200	/// an explicit operand it is added at the end of the explicit operand list
201	/// (before the first implicit operand).
202	void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) {
203	assert(isUInt<LLVM_MI_NUMOPERANDS_BITS>(NumOperands + 1) &&
204	"Cannot add more operands.");
205	assert(MCID && "Cannot add operands before providing an instr descriptor");
206
207	// Check if we're adding one of our existing operands.
208	if (&Op >= Operands && &Op < Operands + NumOperands) {
209	// This is unusual: MI->addOperand(MI->getOperand(i)).
210	// If adding Op requires reallocating or moving existing operands around,
211	// the Op reference could go stale. Support it by copying Op.
212	MachineOperand CopyOp(Op);
213	return addOperand(MF, Op: CopyOp);
214	}
215
216	// Find the insert location for the new operand. Implicit registers go at
217	// the end, everything else goes before the implicit regs.
218	//
219	// FIXME: Allow mixed explicit and implicit operands on inline asm.
220	// InstrEmitter::EmitSpecialNode() is marking inline asm clobbers as
221	// implicit-defs, but they must not be moved around. See the FIXME in
222	// InstrEmitter.cpp.
223	unsigned OpNo = getNumOperands();
224	bool isImpReg = Op.isReg() && Op.isImplicit();
225	if (!isImpReg && !isInlineAsm()) {
226	while (OpNo && Operands[OpNo-1].isReg() && Operands[OpNo-1].isImplicit()) {
227	--OpNo;
228	assert(!Operands[OpNo].isTied() && "Cannot move tied operands");
229	}
230	}
231
232	// OpNo now points as the desired insertion point. Unless this is a variadic
233	// instruction, only implicit regs are allowed beyond MCID->getNumOperands().
234	// RegMask operands go between the explicit and implicit operands.
235	MachineRegisterInfo *MRI = getRegInfo();
236
237	// Determine if the Operands array needs to be reallocated.
238	// Save the old capacity and operand array.
239	OperandCapacity OldCap = CapOperands;
240	MachineOperand *OldOperands = Operands;
241	if (!OldOperands || OldCap.getSize() == getNumOperands()) {
242	CapOperands = OldOperands ? OldCap.getNext() : OldCap.get(N: 1);
243	Operands = MF.allocateOperandArray(Cap: CapOperands);
244	// Move the operands before the insertion point.
245	if (OpNo)
246	moveOperands(Dst: Operands, Src: OldOperands, NumOps: OpNo, MRI);
247	}
248
249	// Move the operands following the insertion point.
250	if (OpNo != NumOperands)
251	moveOperands(Dst: Operands + OpNo + 1, Src: OldOperands + OpNo, NumOps: NumOperands - OpNo,
252	MRI);
253	++NumOperands;
254
255	// Deallocate the old operand array.
256	if (OldOperands != Operands && OldOperands)
257	MF.deallocateOperandArray(Cap: OldCap, Array: OldOperands);
258
259	// Copy Op into place. It still needs to be inserted into the MRI use lists.
260	MachineOperand *NewMO = new (Operands + OpNo) MachineOperand(Op);
261	NewMO->ParentMI = this;
262
263	// When adding a register operand, tell MRI about it.
264	if (NewMO->isReg()) {
265	// Ensure isOnRegUseList() returns false, regardless of Op's status.
266	NewMO->Contents.Reg.Prev = nullptr;
267	// Ignore existing ties. This is not a property that can be copied.
268	NewMO->TiedTo = 0;
269	// Add the new operand to MRI, but only for instructions in an MBB.
270	if (MRI)
271	MRI->addRegOperandToUseList(MO: NewMO);
272	// The MCID operand information isn't accurate until we start adding
273	// explicit operands. The implicit operands are added first, then the
274	// explicits are inserted before them.
275	if (!isImpReg) {
276	// Tie uses to defs as indicated in MCInstrDesc.
277	if (NewMO->isUse()) {
278	int DefIdx = MCID->getOperandConstraint(OpNum: OpNo, Constraint: MCOI::TIED_TO);
279	if (DefIdx != -1)
280	tieOperands(DefIdx, UseIdx: OpNo);
281	}
282	// If the register operand is flagged as early, mark the operand as such.
283	if (MCID->getOperandConstraint(OpNum: OpNo, Constraint: MCOI::EARLY_CLOBBER) != -1)
284	NewMO->setIsEarlyClobber(true);
285	}
286	// Ensure debug instructions set debug flag on register uses.
287	if (NewMO->isUse() && isDebugInstr())
288	NewMO->setIsDebug();
289	}
290	}
291
292	void MachineInstr::removeOperand(unsigned OpNo) {
293	assert(OpNo < getNumOperands() && "Invalid operand number");
294	untieRegOperand(OpIdx: OpNo);
295
296	#ifndef NDEBUG
297	// Moving tied operands would break the ties.
298	for (unsigned i = OpNo + 1, e = getNumOperands(); i != e; ++i)
299	if (Operands[i].isReg())
300	assert(!Operands[i].isTied() && "Cannot move tied operands");
301	#endif
302
303	MachineRegisterInfo *MRI = getRegInfo();
304	if (MRI && Operands[OpNo].isReg())
305	MRI->removeRegOperandFromUseList(MO: Operands + OpNo);
306
307	// Don't call the MachineOperand destructor. A lot of this code depends on
308	// MachineOperand having a trivial destructor anyway, and adding a call here
309	// wouldn't make it 'destructor-correct'.
310
311	if (unsigned N = NumOperands - 1 - OpNo)
312	moveOperands(Dst: Operands + OpNo, Src: Operands + OpNo + 1, NumOps: N, MRI);
313	--NumOperands;
314	}
315
316	void MachineInstr::setExtraInfo(MachineFunction &MF,
317	ArrayRef<MachineMemOperand *> MMOs,
318	MCSymbol *PreInstrSymbol,
319	MCSymbol *PostInstrSymbol,
320	MDNode *HeapAllocMarker, MDNode *PCSections,
321	uint32_t CFIType, MDNode *MMRAs) {
322	bool HasPreInstrSymbol = PreInstrSymbol != nullptr;
323	bool HasPostInstrSymbol = PostInstrSymbol != nullptr;
324	bool HasHeapAllocMarker = HeapAllocMarker != nullptr;
325	bool HasPCSections = PCSections != nullptr;
326	bool HasCFIType = CFIType != 0;
327	bool HasMMRAs = MMRAs != nullptr;
328	int NumPointers = MMOs.size() + HasPreInstrSymbol + HasPostInstrSymbol +
329	HasHeapAllocMarker + HasPCSections + HasCFIType + HasMMRAs;
330
331	// Drop all extra info if there is none.
332	if (NumPointers <= 0) {
333	Info.clear();
334	return;
335	}
336
337	// If more than one pointer, then store out of line. Store heap alloc markers
338	// out of line because PointerSumType cannot hold more than 4 tag types with
339	// 32-bit pointers.
340	// FIXME: Maybe we should make the symbols in the extra info mutable?
341	else if (NumPointers > 1 || HasMMRAs || HasHeapAllocMarker || HasPCSections ||
342	HasCFIType) {
343	Info.set<EIIK_OutOfLine>(
344	MF.createMIExtraInfo(MMOs, PreInstrSymbol, PostInstrSymbol,
345	HeapAllocMarker, PCSections, CFIType, MMRAs));
346	return;
347	}
348
349	// Otherwise store the single pointer inline.
350	if (HasPreInstrSymbol)
351	Info.set<EIIK_PreInstrSymbol>(PreInstrSymbol);
352	else if (HasPostInstrSymbol)
353	Info.set<EIIK_PostInstrSymbol>(PostInstrSymbol);
354	else
355	Info.set<EIIK_MMO>(MMOs[0]);
356	}
357
358	void MachineInstr::dropMemRefs(MachineFunction &MF) {
359	if (memoperands_empty())
360	return;
361
362	setExtraInfo(MF, MMOs: {}, PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: getPostInstrSymbol(),
363	HeapAllocMarker: getHeapAllocMarker(), PCSections: getPCSections(), CFIType: getCFIType(),
364	MMRAs: getMMRAMetadata());
365	}
366
367	void MachineInstr::setMemRefs(MachineFunction &MF,
368	ArrayRef<MachineMemOperand *> MMOs) {
369	if (MMOs.empty()) {
370	dropMemRefs(MF);
371	return;
372	}
373
374	setExtraInfo(MF, MMOs, PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: getPostInstrSymbol(),
375	HeapAllocMarker: getHeapAllocMarker(), PCSections: getPCSections(), CFIType: getCFIType(),
376	MMRAs: getMMRAMetadata());
377	}
378
379	void MachineInstr::addMemOperand(MachineFunction &MF,
380	MachineMemOperand *MO) {
381	SmallVector<MachineMemOperand *, 2> MMOs;
382	MMOs.append(in_start: memoperands_begin(), in_end: memoperands_end());
383	MMOs.push_back(Elt: MO);
384	setMemRefs(MF, MMOs);
385	}
386
387	void MachineInstr::cloneMemRefs(MachineFunction &MF, const MachineInstr &MI) {
388	if (this == &MI)
389	// Nothing to do for a self-clone!
390	return;
391
392	assert(&MF == MI.getMF() &&
393	"Invalid machine functions when cloning memory refrences!");
394	// See if we can just steal the extra info already allocated for the
395	// instruction. We can do this whenever the pre- and post-instruction symbols
396	// are the same (including null).
397	if (getPreInstrSymbol() == MI.getPreInstrSymbol() &&
398	getPostInstrSymbol() == MI.getPostInstrSymbol() &&
399	getHeapAllocMarker() == MI.getHeapAllocMarker() &&
400	getPCSections() == MI.getPCSections() && getMMRAMetadata() &&
401	MI.getMMRAMetadata()) {
402	Info = MI.Info;
403	return;
404	}
405
406	// Otherwise, fall back on a copy-based clone.
407	setMemRefs(MF, MMOs: MI.memoperands());
408	}
409
410	/// Check to see if the MMOs pointed to by the two MemRefs arrays are
411	/// identical.
412	static bool hasIdenticalMMOs(ArrayRef<MachineMemOperand *> LHS,
413	ArrayRef<MachineMemOperand *> RHS) {
414	if (LHS.size() != RHS.size())
415	return false;
416
417	auto LHSPointees = make_pointee_range(Range&: LHS);
418	auto RHSPointees = make_pointee_range(Range&: RHS);
419	return std::equal(first1: LHSPointees.begin(), last1: LHSPointees.end(),
420	first2: RHSPointees.begin());
421	}
422
423	void MachineInstr::cloneMergedMemRefs(MachineFunction &MF,
424	ArrayRef<const MachineInstr *> MIs) {
425	// Try handling easy numbers of MIs with simpler mechanisms.
426	if (MIs.empty()) {
427	dropMemRefs(MF);
428	return;
429	}
430	if (MIs.size() == 1) {
431	cloneMemRefs(MF, MI: *MIs[0]);
432	return;
433	}
434	// Because an empty memoperands list provides *no* information and must be
435	// handled conservatively (assuming the instruction can do anything), the only
436	// way to merge with it is to drop all other memoperands.
437	if (MIs[0]->memoperands_empty()) {
438	dropMemRefs(MF);
439	return;
440	}
441
442	// Handle the general case.
443	SmallVector<MachineMemOperand *, 2> MergedMMOs;
444	// Start with the first instruction.
445	assert(&MF == MIs[0]->getMF() &&
446	"Invalid machine functions when cloning memory references!");
447	MergedMMOs.append(in_start: MIs[0]->memoperands_begin(), in_end: MIs[0]->memoperands_end());
448	// Now walk all the other instructions and accumulate any different MMOs.
449	for (const MachineInstr &MI : make_pointee_range(Range: MIs.slice(N: 1))) {
450	assert(&MF == MI.getMF() &&
451	"Invalid machine functions when cloning memory references!");
452
453	// Skip MIs with identical operands to the first. This is a somewhat
454	// arbitrary hack but will catch common cases without being quadratic.
455	// TODO: We could fully implement merge semantics here if needed.
456	if (hasIdenticalMMOs(LHS: MIs[0]->memoperands(), RHS: MI.memoperands()))
457	continue;
458
459	// Because an empty memoperands list provides *no* information and must be
460	// handled conservatively (assuming the instruction can do anything), the
461	// only way to merge with it is to drop all other memoperands.
462	if (MI.memoperands_empty()) {
463	dropMemRefs(MF);
464	return;
465	}
466
467	// Otherwise accumulate these into our temporary buffer of the merged state.
468	MergedMMOs.append(in_start: MI.memoperands_begin(), in_end: MI.memoperands_end());
469	}
470
471	setMemRefs(MF, MMOs: MergedMMOs);
472	}
473
474	void MachineInstr::setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) {
475	// Do nothing if old and new symbols are the same.
476	if (Symbol == getPreInstrSymbol())
477	return;
478
479	// If there was only one symbol and we're removing it, just clear info.
480	if (!Symbol && Info.is<EIIK_PreInstrSymbol>()) {
481	Info.clear();
482	return;
483	}
484
485	setExtraInfo(MF, MMOs: memoperands(), PreInstrSymbol: Symbol, PostInstrSymbol: getPostInstrSymbol(),
486	HeapAllocMarker: getHeapAllocMarker(), PCSections: getPCSections(), CFIType: getCFIType(),
487	MMRAs: getMMRAMetadata());
488	}
489
490	void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) {
491	// Do nothing if old and new symbols are the same.
492	if (Symbol == getPostInstrSymbol())
493	return;
494
495	// If there was only one symbol and we're removing it, just clear info.
496	if (!Symbol && Info.is<EIIK_PostInstrSymbol>()) {
497	Info.clear();
498	return;
499	}
500
501	setExtraInfo(MF, MMOs: memoperands(), PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: Symbol,
502	HeapAllocMarker: getHeapAllocMarker(), PCSections: getPCSections(), CFIType: getCFIType(),
503	MMRAs: getMMRAMetadata());
504	}
505
506	void MachineInstr::setHeapAllocMarker(MachineFunction &MF, MDNode *Marker) {
507	// Do nothing if old and new symbols are the same.
508	if (Marker == getHeapAllocMarker())
509	return;
510
511	setExtraInfo(MF, MMOs: memoperands(), PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: getPostInstrSymbol(),
512	HeapAllocMarker: Marker, PCSections: getPCSections(), CFIType: getCFIType(), MMRAs: getMMRAMetadata());
513	}
514
515	void MachineInstr::setPCSections(MachineFunction &MF, MDNode *PCSections) {
516	// Do nothing if old and new symbols are the same.
517	if (PCSections == getPCSections())
518	return;
519
520	setExtraInfo(MF, MMOs: memoperands(), PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: getPostInstrSymbol(),
521	HeapAllocMarker: getHeapAllocMarker(), PCSections, CFIType: getCFIType(),
522	MMRAs: getMMRAMetadata());
523	}
524
525	void MachineInstr::setCFIType(MachineFunction &MF, uint32_t Type) {
526	// Do nothing if old and new types are the same.
527	if (Type == getCFIType())
528	return;
529
530	setExtraInfo(MF, MMOs: memoperands(), PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: getPostInstrSymbol(),
531	HeapAllocMarker: getHeapAllocMarker(), PCSections: getPCSections(), CFIType: Type, MMRAs: getMMRAMetadata());
532	}
533
534	void MachineInstr::setMMRAMetadata(MachineFunction &MF, MDNode *MMRAs) {
535	// Do nothing if old and new symbols are the same.
536	if (MMRAs == getMMRAMetadata())
537	return;
538
539	setExtraInfo(MF, MMOs: memoperands(), PreInstrSymbol: getPreInstrSymbol(), PostInstrSymbol: getPostInstrSymbol(),
540	HeapAllocMarker: getHeapAllocMarker(), PCSections: getPCSections(), CFIType: getCFIType(), MMRAs);
541	}
542
543	void MachineInstr::cloneInstrSymbols(MachineFunction &MF,
544	const MachineInstr &MI) {
545	if (this == &MI)
546	// Nothing to do for a self-clone!
547	return;
548
549	assert(&MF == MI.getMF() &&
550	"Invalid machine functions when cloning instruction symbols!");
551
552	setPreInstrSymbol(MF, Symbol: MI.getPreInstrSymbol());
553	setPostInstrSymbol(MF, Symbol: MI.getPostInstrSymbol());
554	setHeapAllocMarker(MF, Marker: MI.getHeapAllocMarker());
555	setPCSections(MF, PCSections: MI.getPCSections());
556	setMMRAMetadata(MF, MMRAs: MI.getMMRAMetadata());
557	}
558
559	uint32_t MachineInstr::mergeFlagsWith(const MachineInstr &Other) const {
560	// For now, the just return the union of the flags. If the flags get more
561	// complicated over time, we might need more logic here.
562	return getFlags() | Other.getFlags();
563	}
564
565	uint32_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
566	uint32_t MIFlags = 0;
567	// Copy the wrapping flags.
568	if (const OverflowingBinaryOperator *OB =
569	dyn_cast<OverflowingBinaryOperator>(Val: &I)) {
570	if (OB->hasNoSignedWrap())
571	MIFlags |= MachineInstr::MIFlag::NoSWrap;
572	if (OB->hasNoUnsignedWrap())
573	MIFlags |= MachineInstr::MIFlag::NoUWrap;
574	} else if (const TruncInst *TI = dyn_cast<TruncInst>(Val: &I)) {
575	if (TI->hasNoSignedWrap())
576	MIFlags |= MachineInstr::MIFlag::NoSWrap;
577	if (TI->hasNoUnsignedWrap())
578	MIFlags |= MachineInstr::MIFlag::NoUWrap;
579	}
580
581	// Copy the nonneg flag.
582	if (const PossiblyNonNegInst *PNI = dyn_cast<PossiblyNonNegInst>(Val: &I)) {
583	if (PNI->hasNonNeg())
584	MIFlags |= MachineInstr::MIFlag::NonNeg;
585	// Copy the disjoint flag.
586	} else if (const PossiblyDisjointInst *PD =
587	dyn_cast<PossiblyDisjointInst>(Val: &I)) {
588	if (PD->isDisjoint())
589	MIFlags |= MachineInstr::MIFlag::Disjoint;
590	}
591
592	// Copy the exact flag.
593	if (const PossiblyExactOperator *PE = dyn_cast<PossiblyExactOperator>(Val: &I))
594	if (PE->isExact())
595	MIFlags |= MachineInstr::MIFlag::IsExact;
596
597	// Copy the fast-math flags.
598	if (const FPMathOperator *FP = dyn_cast<FPMathOperator>(Val: &I)) {
599	const FastMathFlags Flags = FP->getFastMathFlags();
600	if (Flags.noNaNs())
601	MIFlags |= MachineInstr::MIFlag::FmNoNans;
602	if (Flags.noInfs())
603	MIFlags |= MachineInstr::MIFlag::FmNoInfs;
604	if (Flags.noSignedZeros())
605	MIFlags |= MachineInstr::MIFlag::FmNsz;
606	if (Flags.allowReciprocal())
607	MIFlags |= MachineInstr::MIFlag::FmArcp;
608	if (Flags.allowContract())
609	MIFlags |= MachineInstr::MIFlag::FmContract;
610	if (Flags.approxFunc())
611	MIFlags |= MachineInstr::MIFlag::FmAfn;
612	if (Flags.allowReassoc())
613	MIFlags |= MachineInstr::MIFlag::FmReassoc;
614	}
615
616	if (I.getMetadata(KindID: LLVMContext::MD_unpredictable))
617	MIFlags |= MachineInstr::MIFlag::Unpredictable;
618
619	return MIFlags;
620	}
621
622	void MachineInstr::copyIRFlags(const Instruction &I) {
623	Flags = copyFlagsFromInstruction(I);
624	}
625
626	bool MachineInstr::hasPropertyInBundle(uint64_t Mask, QueryType Type) const {
627	assert(!isBundledWithPred() && "Must be called on bundle header");
628	for (MachineBasicBlock::const_instr_iterator MII = getIterator();; ++MII) {
629	if (MII->getDesc().getFlags() & Mask) {
630	if (Type == AnyInBundle)
631	return true;
632	} else {
633	if (Type == AllInBundle && !MII->isBundle())
634	return false;
635	}
636	// This was the last instruction in the bundle.
637	if (!MII->isBundledWithSucc())
638	return Type == AllInBundle;
639	}
640	}
641
642	bool MachineInstr::isIdenticalTo(const MachineInstr &Other,
643	MICheckType Check) const {
644	// If opcodes or number of operands are not the same then the two
645	// instructions are obviously not identical.
646	if (Other.getOpcode() != getOpcode() ||
647	Other.getNumOperands() != getNumOperands())
648	return false;
649
650	if (isBundle()) {
651	// We have passed the test above that both instructions have the same
652	// opcode, so we know that both instructions are bundles here. Let's compare
653	// MIs inside the bundle.
654	assert(Other.isBundle() && "Expected that both instructions are bundles.");
655	MachineBasicBlock::const_instr_iterator I1 = getIterator();
656	MachineBasicBlock::const_instr_iterator I2 = Other.getIterator();
657	// Loop until we analysed the last intruction inside at least one of the
658	// bundles.
659	while (I1->isBundledWithSucc() && I2->isBundledWithSucc()) {
660	++I1;
661	++I2;
662	if (!I1->isIdenticalTo(Other: *I2, Check))
663	return false;
664	}
665	// If we've reached the end of just one of the two bundles, but not both,
666	// the instructions are not identical.
667	if (I1->isBundledWithSucc() || I2->isBundledWithSucc())
668	return false;
669	}
670
671	// Check operands to make sure they match.
672	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
673	const MachineOperand &MO = getOperand(i);
674	const MachineOperand &OMO = Other.getOperand(i);
675	if (!MO.isReg()) {
676	if (!MO.isIdenticalTo(Other: OMO))
677	return false;
678	continue;
679	}
680
681	// Clients may or may not want to ignore defs when testing for equality.
682	// For example, machine CSE pass only cares about finding common
683	// subexpressions, so it's safe to ignore virtual register defs.
684	if (MO.isDef()) {
685	if (Check == IgnoreDefs)
686	continue;
687	else if (Check == IgnoreVRegDefs) {
688	if (!MO.getReg().isVirtual() || !OMO.getReg().isVirtual())
689	if (!MO.isIdenticalTo(Other: OMO))
690	return false;
691	} else {
692	if (!MO.isIdenticalTo(Other: OMO))
693	return false;
694	if (Check == CheckKillDead && MO.isDead() != OMO.isDead())
695	return false;
696	}
697	} else {
698	if (!MO.isIdenticalTo(Other: OMO))
699	return false;
700	if (Check == CheckKillDead && MO.isKill() != OMO.isKill())
701	return false;
702	}
703	}
704	// If DebugLoc does not match then two debug instructions are not identical.
705	if (isDebugInstr())
706	if (getDebugLoc() && Other.getDebugLoc() &&
707	getDebugLoc() != Other.getDebugLoc())
708	return false;
709	// If pre- or post-instruction symbols do not match then the two instructions
710	// are not identical.
711	if (getPreInstrSymbol() != Other.getPreInstrSymbol() ||
712	getPostInstrSymbol() != Other.getPostInstrSymbol())
713	return false;
714	// Call instructions with different CFI types are not identical.
715	if (isCall() && getCFIType() != Other.getCFIType())
716	return false;
717
718	return true;
719	}
720
721	bool MachineInstr::isEquivalentDbgInstr(const MachineInstr &Other) const {
722	if (!isDebugValueLike() || !Other.isDebugValueLike())
723	return false;
724	if (getDebugLoc() != Other.getDebugLoc())
725	return false;
726	if (getDebugVariable() != Other.getDebugVariable())
727	return false;
728	if (getNumDebugOperands() != Other.getNumDebugOperands())
729	return false;
730	for (unsigned OpIdx = 0; OpIdx < getNumDebugOperands(); ++OpIdx)
731	if (!getDebugOperand(Index: OpIdx).isIdenticalTo(Other: Other.getDebugOperand(Index: OpIdx)))
732	return false;
733	if (!DIExpression::isEqualExpression(
734	FirstExpr: getDebugExpression(), FirstIndirect: isIndirectDebugValue(),
735	SecondExpr: Other.getDebugExpression(), SecondIndirect: Other.isIndirectDebugValue()))
736	return false;
737	return true;
738	}
739
740	const MachineFunction *MachineInstr::getMF() const {
741	return getParent()->getParent();
742	}
743
744	MachineInstr *MachineInstr::removeFromParent() {
745	assert(getParent() && "Not embedded in a basic block!");
746	return getParent()->remove(I: this);
747	}
748
749	MachineInstr *MachineInstr::removeFromBundle() {
750	assert(getParent() && "Not embedded in a basic block!");
751	return getParent()->remove_instr(I: this);
752	}
753
754	void MachineInstr::eraseFromParent() {
755	assert(getParent() && "Not embedded in a basic block!");
756	getParent()->erase(I: this);
757	}
758
759	void MachineInstr::eraseFromBundle() {
760	assert(getParent() && "Not embedded in a basic block!");
761	getParent()->erase_instr(I: this);
762	}
763
764	bool MachineInstr::isCandidateForCallSiteEntry(QueryType Type) const {
765	if (!isCall(Type))
766	return false;
767	switch (getOpcode()) {
768	case TargetOpcode::PATCHPOINT:
769	case TargetOpcode::STACKMAP:
770	case TargetOpcode::STATEPOINT:
771	case TargetOpcode::FENTRY_CALL:
772	return false;
773	}
774	return true;
775	}
776
777	bool MachineInstr::shouldUpdateCallSiteInfo() const {
778	if (isBundle())
779	return isCandidateForCallSiteEntry(Type: MachineInstr::AnyInBundle);
780	return isCandidateForCallSiteEntry();
781	}
782
783	unsigned MachineInstr::getNumExplicitOperands() const {
784	unsigned NumOperands = MCID->getNumOperands();
785	if (!MCID->isVariadic())
786	return NumOperands;
787
788	for (unsigned I = NumOperands, E = getNumOperands(); I != E; ++I) {
789	const MachineOperand &MO = getOperand(i: I);
790	// The operands must always be in the following order:
791	// - explicit reg defs,
792	// - other explicit operands (reg uses, immediates, etc.),
793	// - implicit reg defs
794	// - implicit reg uses
795	if (MO.isReg() && MO.isImplicit())
796	break;
797	++NumOperands;
798	}
799	return NumOperands;
800	}
801
802	unsigned MachineInstr::getNumExplicitDefs() const {
803	unsigned NumDefs = MCID->getNumDefs();
804	if (!MCID->isVariadic())
805	return NumDefs;
806
807	for (unsigned I = NumDefs, E = getNumOperands(); I != E; ++I) {
808	const MachineOperand &MO = getOperand(i: I);
809	if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
810	break;
811	++NumDefs;
812	}
813	return NumDefs;
814	}
815
816	void MachineInstr::bundleWithPred() {
817	assert(!isBundledWithPred() && "MI is already bundled with its predecessor");
818	setFlag(BundledPred);
819	MachineBasicBlock::instr_iterator Pred = getIterator();
820	--Pred;
821	assert(!Pred->isBundledWithSucc() && "Inconsistent bundle flags");
822	Pred->setFlag(BundledSucc);
823	}
824
825	void MachineInstr::bundleWithSucc() {
826	assert(!isBundledWithSucc() && "MI is already bundled with its successor");
827	setFlag(BundledSucc);
828	MachineBasicBlock::instr_iterator Succ = getIterator();
829	++Succ;
830	assert(!Succ->isBundledWithPred() && "Inconsistent bundle flags");
831	Succ->setFlag(BundledPred);
832	}
833
834	void MachineInstr::unbundleFromPred() {
835	assert(isBundledWithPred() && "MI isn't bundled with its predecessor");
836	clearFlag(Flag: BundledPred);
837	MachineBasicBlock::instr_iterator Pred = getIterator();
838	--Pred;
839	assert(Pred->isBundledWithSucc() && "Inconsistent bundle flags");
840	Pred->clearFlag(Flag: BundledSucc);
841	}
842
843	void MachineInstr::unbundleFromSucc() {
844	assert(isBundledWithSucc() && "MI isn't bundled with its successor");
845	clearFlag(Flag: BundledSucc);
846	MachineBasicBlock::instr_iterator Succ = getIterator();
847	++Succ;
848	assert(Succ->isBundledWithPred() && "Inconsistent bundle flags");
849	Succ->clearFlag(Flag: BundledPred);
850	}
851
852	bool MachineInstr::isStackAligningInlineAsm() const {
853	if (isInlineAsm()) {
854	unsigned ExtraInfo = getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
855	if (ExtraInfo & InlineAsm::Extra_IsAlignStack)
856	return true;
857	}
858	return false;
859	}
860
861	InlineAsm::AsmDialect MachineInstr::getInlineAsmDialect() const {
862	assert(isInlineAsm() && "getInlineAsmDialect() only works for inline asms!");
863	unsigned ExtraInfo = getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
864	return InlineAsm::AsmDialect((ExtraInfo & InlineAsm::Extra_AsmDialect) != 0);
865	}
866
867	int MachineInstr::findInlineAsmFlagIdx(unsigned OpIdx,
868	unsigned *GroupNo) const {
869	assert(isInlineAsm() && "Expected an inline asm instruction");
870	assert(OpIdx < getNumOperands() && "OpIdx out of range");
871
872	// Ignore queries about the initial operands.
873	if (OpIdx < InlineAsm::MIOp_FirstOperand)
874	return -1;
875
876	unsigned Group = 0;
877	unsigned NumOps;
878	for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); i < e;
879	i += NumOps) {
880	const MachineOperand &FlagMO = getOperand(i);
881	// If we reach the implicit register operands, stop looking.
882	if (!FlagMO.isImm())
883	return -1;
884	const InlineAsm::Flag F(FlagMO.getImm());
885	NumOps = 1 + F.getNumOperandRegisters();
886	if (i + NumOps > OpIdx) {
887	if (GroupNo)
888	*GroupNo = Group;
889	return i;
890	}
891	++Group;
892	}
893	return -1;
894	}
895
896	const DILabel *MachineInstr::getDebugLabel() const {
897	assert(isDebugLabel() && "not a DBG_LABEL");
898	return cast<DILabel>(Val: getOperand(i: 0).getMetadata());
899	}
900
901	const MachineOperand &MachineInstr::getDebugVariableOp() const {
902	assert((isDebugValueLike()) && "not a DBG_VALUE*");
903	unsigned VariableOp = isNonListDebugValue() ? 2 : 0;
904	return getOperand(i: VariableOp);
905	}
906
907	MachineOperand &MachineInstr::getDebugVariableOp() {
908	assert((isDebugValueLike()) && "not a DBG_VALUE*");
909	unsigned VariableOp = isNonListDebugValue() ? 2 : 0;
910	return getOperand(i: VariableOp);
911	}
912
913	const DILocalVariable *MachineInstr::getDebugVariable() const {
914	return cast<DILocalVariable>(Val: getDebugVariableOp().getMetadata());
915	}
916
917	const MachineOperand &MachineInstr::getDebugExpressionOp() const {
918	assert((isDebugValueLike()) && "not a DBG_VALUE*");
919	unsigned ExpressionOp = isNonListDebugValue() ? 3 : 1;
920	return getOperand(i: ExpressionOp);
921	}
922
923	MachineOperand &MachineInstr::getDebugExpressionOp() {
924	assert((isDebugValueLike()) && "not a DBG_VALUE*");
925	unsigned ExpressionOp = isNonListDebugValue() ? 3 : 1;
926	return getOperand(i: ExpressionOp);
927	}
928
929	const DIExpression *MachineInstr::getDebugExpression() const {
930	return cast<DIExpression>(Val: getDebugExpressionOp().getMetadata());
931	}
932
933	bool MachineInstr::isDebugEntryValue() const {
934	return isDebugValue() && getDebugExpression()->isEntryValue();
935	}
936
937	const TargetRegisterClass*
938	MachineInstr::getRegClassConstraint(unsigned OpIdx,
939	const TargetInstrInfo *TII,
940	const TargetRegisterInfo *TRI) const {
941	assert(getParent() && "Can't have an MBB reference here!");
942	assert(getMF() && "Can't have an MF reference here!");
943	const MachineFunction &MF = *getMF();
944
945	// Most opcodes have fixed constraints in their MCInstrDesc.
946	if (!isInlineAsm())
947	return TII->getRegClass(MCID: getDesc(), OpNum: OpIdx, TRI, MF);
948
949	if (!getOperand(i: OpIdx).isReg())
950	return nullptr;
951
952	// For tied uses on inline asm, get the constraint from the def.
953	unsigned DefIdx;
954	if (getOperand(i: OpIdx).isUse() && isRegTiedToDefOperand(UseOpIdx: OpIdx, DefOpIdx: &DefIdx))
955	OpIdx = DefIdx;
956
957	// Inline asm stores register class constraints in the flag word.
958	int FlagIdx = findInlineAsmFlagIdx(OpIdx);
959	if (FlagIdx < 0)
960	return nullptr;
961
962	const InlineAsm::Flag F(getOperand(i: FlagIdx).getImm());
963	unsigned RCID;
964	if ((F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind()) &&
965	F.hasRegClassConstraint(RC&: RCID))
966	return TRI->getRegClass(i: RCID);
967
968	// Assume that all registers in a memory operand are pointers.
969	if (F.isMemKind())
970	return TRI->getPointerRegClass(MF);
971
972	return nullptr;
973	}
974
975	const TargetRegisterClass *MachineInstr::getRegClassConstraintEffectForVReg(
976	Register Reg, const TargetRegisterClass *CurRC, const TargetInstrInfo *TII,
977	const TargetRegisterInfo *TRI, bool ExploreBundle) const {
978	// Check every operands inside the bundle if we have
979	// been asked to.
980	if (ExploreBundle)
981	for (ConstMIBundleOperands OpndIt(*this); OpndIt.isValid() && CurRC;
982	++OpndIt)
983	CurRC = OpndIt->getParent()->getRegClassConstraintEffectForVRegImpl(
984	OpIdx: OpndIt.getOperandNo(), Reg, CurRC, TII, TRI);
985	else
986	// Otherwise, just check the current operands.
987	for (unsigned i = 0, e = NumOperands; i < e && CurRC; ++i)
988	CurRC = getRegClassConstraintEffectForVRegImpl(OpIdx: i, Reg, CurRC, TII, TRI);
989	return CurRC;
990	}
991
992	const TargetRegisterClass *MachineInstr::getRegClassConstraintEffectForVRegImpl(
993	unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC,
994	const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const {
995	assert(CurRC && "Invalid initial register class");
996	// Check if Reg is constrained by some of its use/def from MI.
997	const MachineOperand &MO = getOperand(i: OpIdx);
998	if (!MO.isReg() || MO.getReg() != Reg)
999	return CurRC;
1000	// If yes, accumulate the constraints through the operand.
1001	return getRegClassConstraintEffect(OpIdx, CurRC, TII, TRI);
1002	}
1003
1004	const TargetRegisterClass *MachineInstr::getRegClassConstraintEffect(
1005	unsigned OpIdx, const TargetRegisterClass *CurRC,
1006	const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const {
1007	const TargetRegisterClass *OpRC = getRegClassConstraint(OpIdx, TII, TRI);
1008	const MachineOperand &MO = getOperand(i: OpIdx);
1009	assert(MO.isReg() &&
1010	"Cannot get register constraints for non-register operand");
1011	assert(CurRC && "Invalid initial register class");
1012	if (unsigned SubIdx = MO.getSubReg()) {
1013	if (OpRC)
1014	CurRC = TRI->getMatchingSuperRegClass(A: CurRC, B: OpRC, Idx: SubIdx);
1015	else
1016	CurRC = TRI->getSubClassWithSubReg(RC: CurRC, Idx: SubIdx);
1017	} else if (OpRC)
1018	CurRC = TRI->getCommonSubClass(A: CurRC, B: OpRC);
1019	return CurRC;
1020	}
1021
1022	/// Return the number of instructions inside the MI bundle, not counting the
1023	/// header instruction.
1024	unsigned MachineInstr::getBundleSize() const {
1025	MachineBasicBlock::const_instr_iterator I = getIterator();
1026	unsigned Size = 0;
1027	while (I->isBundledWithSucc()) {
1028	++Size;
1029	++I;
1030	}
1031	return Size;
1032	}
1033
1034	/// Returns true if the MachineInstr has an implicit-use operand of exactly
1035	/// the given register (not considering sub/super-registers).
1036	bool MachineInstr::hasRegisterImplicitUseOperand(Register Reg) const {
1037	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
1038	const MachineOperand &MO = getOperand(i);
1039	if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == Reg)
1040	return true;
1041	}
1042	return false;
1043	}
1044
1045	/// findRegisterUseOperandIdx() - Returns the MachineOperand that is a use of
1046	/// the specific register or -1 if it is not found. It further tightens
1047	/// the search criteria to a use that kills the register if isKill is true.
1048	int MachineInstr::findRegisterUseOperandIdx(Register Reg,
1049	const TargetRegisterInfo *TRI,
1050	bool isKill) const {
1051	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
1052	const MachineOperand &MO = getOperand(i);
1053	if (!MO.isReg() || !MO.isUse())
1054	continue;
1055	Register MOReg = MO.getReg();
1056	if (!MOReg)
1057	continue;
1058	if (MOReg == Reg || (TRI && Reg && MOReg && TRI->regsOverlap(RegA: MOReg, RegB: Reg)))
1059	if (!isKill || MO.isKill())
1060	return i;
1061	}
1062	return -1;
1063	}
1064
1065	/// readsWritesVirtualRegister - Return a pair of bools (reads, writes)
1066	/// indicating if this instruction reads or writes Reg. This also considers
1067	/// partial defines.
1068	std::pair<bool,bool>
1069	MachineInstr::readsWritesVirtualRegister(Register Reg,
1070	SmallVectorImpl<unsigned> *Ops) const {
1071	bool PartDef = false; // Partial redefine.
1072	bool FullDef = false; // Full define.
1073	bool Use = false;
1074
1075	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
1076	const MachineOperand &MO = getOperand(i);
1077	if (!MO.isReg() || MO.getReg() != Reg)
1078	continue;
1079	if (Ops)
1080	Ops->push_back(Elt: i);
1081	if (MO.isUse())
1082	Use |= !MO.isUndef();
1083	else if (MO.getSubReg() && !MO.isUndef())
1084	// A partial def undef doesn't count as reading the register.
1085	PartDef = true;
1086	else
1087	FullDef = true;
1088	}
1089	// A partial redefine uses Reg unless there is also a full define.
1090	return std::make_pair(x: Use || (PartDef && !FullDef), y: PartDef || FullDef);
1091	}
1092
1093	/// findRegisterDefOperandIdx() - Returns the operand index that is a def of
1094	/// the specified register or -1 if it is not found. If isDead is true, defs
1095	/// that are not dead are skipped. If TargetRegisterInfo is non-null, then it
1096	/// also checks if there is a def of a super-register.
1097	int MachineInstr::findRegisterDefOperandIdx(Register Reg,
1098	const TargetRegisterInfo *TRI,
1099	bool isDead, bool Overlap) const {
1100	bool isPhys = Reg.isPhysical();
1101	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
1102	const MachineOperand &MO = getOperand(i);
1103	// Accept regmask operands when Overlap is set.
1104	// Ignore them when looking for a specific def operand (Overlap == false).
1105	if (isPhys && Overlap && MO.isRegMask() && MO.clobbersPhysReg(PhysReg: Reg))
1106	return i;
1107	if (!MO.isReg() || !MO.isDef())
1108	continue;
1109	Register MOReg = MO.getReg();
1110	bool Found = (MOReg == Reg);
1111	if (!Found && TRI && isPhys && MOReg.isPhysical()) {
1112	if (Overlap)
1113	Found = TRI->regsOverlap(RegA: MOReg, RegB: Reg);
1114	else
1115	Found = TRI->isSubRegister(RegA: MOReg, RegB: Reg);
1116	}
1117	if (Found && (!isDead || MO.isDead()))
1118	return i;
1119	}
1120	return -1;
1121	}
1122
1123	/// findFirstPredOperandIdx() - Find the index of the first operand in the
1124	/// operand list that is used to represent the predicate. It returns -1 if
1125	/// none is found.
1126	int MachineInstr::findFirstPredOperandIdx() const {
1127	// Don't call MCID.findFirstPredOperandIdx() because this variant
1128	// is sometimes called on an instruction that's not yet complete, and
1129	// so the number of operands is less than the MCID indicates. In
1130	// particular, the PTX target does this.
1131	const MCInstrDesc &MCID = getDesc();
1132	if (MCID.isPredicable()) {
1133	for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
1134	if (MCID.operands()[i].isPredicate())
1135	return i;
1136	}
1137
1138	return -1;
1139	}
1140
1141	// MachineOperand::TiedTo is 4 bits wide.
1142	const unsigned TiedMax = 15;
1143
1144	/// tieOperands - Mark operands at DefIdx and UseIdx as tied to each other.
1145	///
1146	/// Use and def operands can be tied together, indicated by a non-zero TiedTo
1147	/// field. TiedTo can have these values:
1148	///
1149	/// 0: Operand is not tied to anything.
1150	/// 1 to TiedMax-1: Tied to getOperand(TiedTo-1).
1151	/// TiedMax: Tied to an operand >= TiedMax-1.
1152	///
1153	/// The tied def must be one of the first TiedMax operands on a normal
1154	/// instruction. INLINEASM instructions allow more tied defs.
1155	///
1156	void MachineInstr::tieOperands(unsigned DefIdx, unsigned UseIdx) {
1157	MachineOperand &DefMO = getOperand(i: DefIdx);
1158	MachineOperand &UseMO = getOperand(i: UseIdx);
1159	assert(DefMO.isDef() && "DefIdx must be a def operand");
1160	assert(UseMO.isUse() && "UseIdx must be a use operand");
1161	assert(!DefMO.isTied() && "Def is already tied to another use");
1162	assert(!UseMO.isTied() && "Use is already tied to another def");
1163
1164	if (DefIdx < TiedMax)
1165	UseMO.TiedTo = DefIdx + 1;
1166	else {
1167	// Inline asm can use the group descriptors to find tied operands,
1168	// statepoint tied operands are trivial to match (1-1 reg def with reg use),
1169	// but on normal instruction, the tied def must be within the first TiedMax
1170	// operands.
1171	assert((isInlineAsm() || getOpcode() == TargetOpcode::STATEPOINT) &&
1172	"DefIdx out of range");
1173	UseMO.TiedTo = TiedMax;
1174	}
1175
1176	// UseIdx can be out of range, we'll search for it in findTiedOperandIdx().
1177	DefMO.TiedTo = std::min(a: UseIdx + 1, b: TiedMax);
1178	}
1179
1180	/// Given the index of a tied register operand, find the operand it is tied to.
1181	/// Defs are tied to uses and vice versa. Returns the index of the tied operand
1182	/// which must exist.
1183	unsigned MachineInstr::findTiedOperandIdx(unsigned OpIdx) const {
1184	const MachineOperand &MO = getOperand(i: OpIdx);
1185	assert(MO.isTied() && "Operand isn't tied");
1186
1187	// Normally TiedTo is in range.
1188	if (MO.TiedTo < TiedMax)
1189	return MO.TiedTo - 1;
1190
1191	// Uses on normal instructions can be out of range.
1192	if (!isInlineAsm() && getOpcode() != TargetOpcode::STATEPOINT) {
1193	// Normal tied defs must be in the 0..TiedMax-1 range.
1194	if (MO.isUse())
1195	return TiedMax - 1;
1196	// MO is a def. Search for the tied use.
1197	for (unsigned i = TiedMax - 1, e = getNumOperands(); i != e; ++i) {
1198	const MachineOperand &UseMO = getOperand(i);
1199	if (UseMO.isReg() && UseMO.isUse() && UseMO.TiedTo == OpIdx + 1)
1200	return i;
1201	}
1202	llvm_unreachable("Can't find tied use");
1203	}
1204
1205	if (getOpcode() == TargetOpcode::STATEPOINT) {
1206	// In STATEPOINT defs correspond 1-1 to GC pointer operands passed
1207	// on registers.
1208	StatepointOpers SO(this);
1209	unsigned CurUseIdx = SO.getFirstGCPtrIdx();
1210	assert(CurUseIdx != -1U && "only gc pointer statepoint operands can be tied");
1211	unsigned NumDefs = getNumDefs();
1212	for (unsigned CurDefIdx = 0; CurDefIdx < NumDefs; ++CurDefIdx) {
1213	while (!getOperand(i: CurUseIdx).isReg())
1214	CurUseIdx = StackMaps::getNextMetaArgIdx(MI: this, CurIdx: CurUseIdx);
1215	if (OpIdx == CurDefIdx)
1216	return CurUseIdx;
1217	if (OpIdx == CurUseIdx)
1218	return CurDefIdx;
1219	CurUseIdx = StackMaps::getNextMetaArgIdx(MI: this, CurIdx: CurUseIdx);
1220	}
1221	llvm_unreachable("Can't find tied use");
1222	}
1223
1224	// Now deal with inline asm by parsing the operand group descriptor flags.
1225	// Find the beginning of each operand group.
1226	SmallVector<unsigned, 8> GroupIdx;
1227	unsigned OpIdxGroup = ~0u;
1228	unsigned NumOps;
1229	for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); i < e;
1230	i += NumOps) {
1231	const MachineOperand &FlagMO = getOperand(i);
1232	assert(FlagMO.isImm() && "Invalid tied operand on inline asm");
1233	unsigned CurGroup = GroupIdx.size();
1234	GroupIdx.push_back(Elt: i);
1235	const InlineAsm::Flag F(FlagMO.getImm());
1236	NumOps = 1 + F.getNumOperandRegisters();
1237	// OpIdx belongs to this operand group.
1238	if (OpIdx > i && OpIdx < i + NumOps)
1239	OpIdxGroup = CurGroup;
1240	unsigned TiedGroup;
1241	if (!F.isUseOperandTiedToDef(Idx&: TiedGroup))
1242	continue;
1243	// Operands in this group are tied to operands in TiedGroup which must be
1244	// earlier. Find the number of operands between the two groups.
1245	unsigned Delta = i - GroupIdx[TiedGroup];
1246
1247	// OpIdx is a use tied to TiedGroup.
1248	if (OpIdxGroup == CurGroup)
1249	return OpIdx - Delta;
1250
1251	// OpIdx is a def tied to this use group.
1252	if (OpIdxGroup == TiedGroup)
1253	return OpIdx + Delta;
1254	}
1255	llvm_unreachable("Invalid tied operand on inline asm");
1256	}
1257
1258	/// clearKillInfo - Clears kill flags on all operands.
1259	///
1260	void MachineInstr::clearKillInfo() {
1261	for (MachineOperand &MO : operands()) {
1262	if (MO.isReg() && MO.isUse())
1263	MO.setIsKill(false);
1264	}
1265	}
1266
1267	void MachineInstr::substituteRegister(Register FromReg, Register ToReg,
1268	unsigned SubIdx,
1269	const TargetRegisterInfo &RegInfo) {
1270	if (ToReg.isPhysical()) {
1271	if (SubIdx)
1272	ToReg = RegInfo.getSubReg(Reg: ToReg, Idx: SubIdx);
1273	for (MachineOperand &MO : operands()) {
1274	if (!MO.isReg() || MO.getReg() != FromReg)
1275	continue;
1276	MO.substPhysReg(Reg: ToReg, RegInfo);
1277	}
1278	} else {
1279	for (MachineOperand &MO : operands()) {
1280	if (!MO.isReg() || MO.getReg() != FromReg)
1281	continue;
1282	MO.substVirtReg(Reg: ToReg, SubIdx, RegInfo);
1283	}
1284	}
1285	}
1286
1287	/// isSafeToMove - Return true if it is safe to move this instruction. If
1288	/// SawStore is set to true, it means that there is a store (or call) between
1289	/// the instruction's location and its intended destination.
1290	bool MachineInstr::isSafeToMove(AAResults *AA, bool &SawStore) const {
1291	// Ignore stuff that we obviously can't move.
1292	//
1293	// Treat volatile loads as stores. This is not strictly necessary for
1294	// volatiles, but it is required for atomic loads. It is not allowed to move
1295	// a load across an atomic load with Ordering > Monotonic.
1296	if (mayStore() || isCall() || isPHI() ||
1297	(mayLoad() && hasOrderedMemoryRef())) {
1298	SawStore = true;
1299	return false;
1300	}
1301
1302	if (isPosition() || isDebugInstr() || isTerminator() ||
1303	mayRaiseFPException() || hasUnmodeledSideEffects() ||
1304	isJumpTableDebugInfo())
1305	return false;
1306
1307	// See if this instruction does a load. If so, we have to guarantee that the
1308	// loaded value doesn't change between the load and the its intended
1309	// destination. The check for isInvariantLoad gives the target the chance to
1310	// classify the load as always returning a constant, e.g. a constant pool
1311	// load.
1312	if (mayLoad() && !isDereferenceableInvariantLoad())
1313	// Otherwise, this is a real load. If there is a store between the load and
1314	// end of block, we can't move it.
1315	return !SawStore;
1316
1317	return true;
1318	}
1319
1320	static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
1321	bool UseTBAA, const MachineMemOperand *MMOa,
1322	const MachineMemOperand *MMOb) {
1323	// The following interface to AA is fashioned after DAGCombiner::isAlias and
1324	// operates with MachineMemOperand offset with some important assumptions:
1325	// - LLVM fundamentally assumes flat address spaces.
1326	// - MachineOperand offset can *only* result from legalization and cannot
1327	// affect queries other than the trivial case of overlap checking.
1328	// - These offsets never wrap and never step outside of allocated objects.
1329	// - There should never be any negative offsets here.
1330	//
1331	// FIXME: Modify API to hide this math from "user"
1332	// Even before we go to AA we can reason locally about some memory objects. It
1333	// can save compile time, and possibly catch some corner cases not currently
1334	// covered.
1335
1336	int64_t OffsetA = MMOa->getOffset();
1337	int64_t OffsetB = MMOb->getOffset();
1338	int64_t MinOffset = std::min(a: OffsetA, b: OffsetB);
1339
1340	LocationSize WidthA = MMOa->getSize();
1341	LocationSize WidthB = MMOb->getSize();
1342	bool KnownWidthA = WidthA.hasValue();
1343	bool KnownWidthB = WidthB.hasValue();
1344	bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
1345
1346	const Value *ValA = MMOa->getValue();
1347	const Value *ValB = MMOb->getValue();
1348	bool SameVal = (ValA && ValB && (ValA == ValB));
1349	if (!SameVal) {
1350	const PseudoSourceValue *PSVa = MMOa->getPseudoValue();
1351	const PseudoSourceValue *PSVb = MMOb->getPseudoValue();
1352	if (PSVa && ValB && !PSVa->mayAlias(&MFI))
1353	return false;
1354	if (PSVb && ValA && !PSVb->mayAlias(&MFI))
1355	return false;
1356	if (PSVa && PSVb && (PSVa == PSVb))
1357	SameVal = true;
1358	}
1359
1360	if (SameVal && BothMMONonScalable) {
1361	if (!KnownWidthA || !KnownWidthB)
1362	return true;
1363	int64_t MaxOffset = std::max(a: OffsetA, b: OffsetB);
1364	int64_t LowWidth = (MinOffset == OffsetA)
1365	? WidthA.getValue().getKnownMinValue()
1366	: WidthB.getValue().getKnownMinValue();
1367	return (MinOffset + LowWidth > MaxOffset);
1368	}
1369
1370	if (!AA)
1371	return true;
1372
1373	if (!ValA || !ValB)
1374	return true;
1375
1376	assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
1377	assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
1378
1379	// If Scalable Location Size has non-zero offset, Width + Offset does not work
1380	// at the moment
1381	if ((WidthA.isScalable() && OffsetA > 0) ||
1382	(WidthB.isScalable() && OffsetB > 0))
1383	return true;
1384
1385	int64_t OverlapA =
1386	KnownWidthA ? WidthA.getValue().getKnownMinValue() + OffsetA - MinOffset
1387	: MemoryLocation::UnknownSize;
1388	int64_t OverlapB =
1389	KnownWidthB ? WidthB.getValue().getKnownMinValue() + OffsetB - MinOffset
1390	: MemoryLocation::UnknownSize;
1391
1392	LocationSize LocA = (WidthA.isScalable() || !KnownWidthA)
1393	? WidthA
1394	: LocationSize::precise(Value: OverlapA);
1395	LocationSize LocB = (WidthB.isScalable() || !KnownWidthB)
1396	? WidthB
1397	: LocationSize::precise(Value: OverlapB);
1398
1399	return !AA->isNoAlias(
1400	LocA: MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
1401	LocB: MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
1402	}
1403
1404	bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,
1405	bool UseTBAA) const {
1406	const MachineFunction *MF = getMF();
1407	const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1408	const MachineFrameInfo &MFI = MF->getFrameInfo();
1409
1410	// Exclude call instruction which may alter the memory but can not be handled
1411	// by this function.
1412	if (isCall() || Other.isCall())
1413	return true;
1414
1415	// If neither instruction stores to memory, they can't alias in any
1416	// meaningful way, even if they read from the same address.
1417	if (!mayStore() && !Other.mayStore())
1418	return false;
1419
1420	// Both instructions must be memory operations to be able to alias.
1421	if (!mayLoadOrStore() || !Other.mayLoadOrStore())
1422	return false;
1423
1424	// Let the target decide if memory accesses cannot possibly overlap.
1425	if (TII->areMemAccessesTriviallyDisjoint(MIa: *this, MIb: Other))
1426	return false;
1427
1428	// Memory operations without memory operands may access anything. Be
1429	// conservative and assume `MayAlias`.
1430	if (memoperands_empty() || Other.memoperands_empty())
1431	return true;
1432
1433	// Skip if there are too many memory operands.
1434	auto NumChecks = getNumMemOperands() * Other.getNumMemOperands();
1435	if (NumChecks > TII->getMemOperandAACheckLimit())
1436	return true;
1437
1438	// Check each pair of memory operands from both instructions, which can't
1439	// alias only if all pairs won't alias.
1440	for (auto *MMOa : memoperands())
1441	for (auto *MMOb : Other.memoperands())
1442	if (MemOperandsHaveAlias(MFI, AA, UseTBAA, MMOa, MMOb))
1443	return true;
1444
1445	return false;
1446	}
1447
1448	/// hasOrderedMemoryRef - Return true if this instruction may have an ordered
1449	/// or volatile memory reference, or if the information describing the memory
1450	/// reference is not available. Return false if it is known to have no ordered
1451	/// memory references.
1452	bool MachineInstr::hasOrderedMemoryRef() const {
1453	// An instruction known never to access memory won't have a volatile access.
1454	if (!mayStore() &&
1455	!mayLoad() &&
1456	!isCall() &&
1457	!hasUnmodeledSideEffects())
1458	return false;
1459
1460	// Otherwise, if the instruction has no memory reference information,
1461	// conservatively assume it wasn't preserved.
1462	if (memoperands_empty())
1463	return true;
1464
1465	// Check if any of our memory operands are ordered.
1466	return llvm::any_of(Range: memoperands(), P: [](const MachineMemOperand *MMO) {
1467	return !MMO->isUnordered();
1468	});
1469	}
1470
1471	/// isDereferenceableInvariantLoad - Return true if this instruction will never
1472	/// trap and is loading from a location whose value is invariant across a run of
1473	/// this function.
1474	bool MachineInstr::isDereferenceableInvariantLoad() const {
1475	// If the instruction doesn't load at all, it isn't an invariant load.
1476	if (!mayLoad())
1477	return false;
1478
1479	// If the instruction has lost its memoperands, conservatively assume that
1480	// it may not be an invariant load.
1481	if (memoperands_empty())
1482	return false;
1483
1484	const MachineFrameInfo &MFI = getParent()->getParent()->getFrameInfo();
1485
1486	for (MachineMemOperand *MMO : memoperands()) {
1487	if (!MMO->isUnordered())
1488	// If the memory operand has ordering side effects, we can't move the
1489	// instruction. Such an instruction is technically an invariant load,
1490	// but the caller code would need updated to expect that.
1491	return false;
1492	if (MMO->isStore()) return false;
1493	if (MMO->isInvariant() && MMO->isDereferenceable())
1494	continue;
1495
1496	// A load from a constant PseudoSourceValue is invariant.
1497	if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) {
1498	if (PSV->isConstant(&MFI))
1499	continue;
1500	}
1501
1502	// Otherwise assume conservatively.
1503	return false;
1504	}
1505
1506	// Everything checks out.
1507	return true;
1508	}
1509
1510	/// isConstantValuePHI - If the specified instruction is a PHI that always
1511	/// merges together the same virtual register, return the register, otherwise
1512	/// return 0.
1513	unsigned MachineInstr::isConstantValuePHI() const {
1514	if (!isPHI())
1515	return 0;
1516	assert(getNumOperands() >= 3 &&
1517	"It's illegal to have a PHI without source operands");
1518
1519	Register Reg = getOperand(i: 1).getReg();
1520	for (unsigned i = 3, e = getNumOperands(); i < e; i += 2)
1521	if (getOperand(i).getReg() != Reg)
1522	return 0;
1523	return Reg;
1524	}
1525
1526	bool MachineInstr::hasUnmodeledSideEffects() const {
1527	if (hasProperty(MCFlag: MCID::UnmodeledSideEffects))
1528	return true;
1529	if (isInlineAsm()) {
1530	unsigned ExtraInfo = getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
1531	if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
1532	return true;
1533	}
1534
1535	return false;
1536	}
1537
1538	bool MachineInstr::isLoadFoldBarrier() const {
1539	return mayStore() || isCall() ||
1540	(hasUnmodeledSideEffects() && !isPseudoProbe());
1541	}
1542
1543	/// allDefsAreDead - Return true if all the defs of this instruction are dead.
1544	///
1545	bool MachineInstr::allDefsAreDead() const {
1546	for (const MachineOperand &MO : operands()) {
1547	if (!MO.isReg() || MO.isUse())
1548	continue;
1549	if (!MO.isDead())
1550	return false;
1551	}
1552	return true;
1553	}
1554
1555	bool MachineInstr::allImplicitDefsAreDead() const {
1556	for (const MachineOperand &MO : implicit_operands()) {
1557	if (!MO.isReg() || MO.isUse())
1558	continue;
1559	if (!MO.isDead())
1560	return false;
1561	}
1562	return true;
1563	}
1564
1565	/// copyImplicitOps - Copy implicit register operands from specified
1566	/// instruction to this instruction.
1567	void MachineInstr::copyImplicitOps(MachineFunction &MF,
1568	const MachineInstr &MI) {
1569	for (const MachineOperand &MO :
1570	llvm::drop_begin(RangeOrContainer: MI.operands(), N: MI.getDesc().getNumOperands()))
1571	if ((MO.isReg() && MO.isImplicit()) || MO.isRegMask())
1572	addOperand(MF, Op: MO);
1573	}
1574
1575	bool MachineInstr::hasComplexRegisterTies() const {
1576	const MCInstrDesc &MCID = getDesc();
1577	if (MCID.Opcode == TargetOpcode::STATEPOINT)
1578	return true;
1579	for (unsigned I = 0, E = getNumOperands(); I < E; ++I) {
1580	const auto &Operand = getOperand(i: I);
1581	if (!Operand.isReg() || Operand.isDef())
1582	// Ignore the defined registers as MCID marks only the uses as tied.
1583	continue;
1584	int ExpectedTiedIdx = MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::TIED_TO);
1585	int TiedIdx = Operand.isTied() ? int(findTiedOperandIdx(OpIdx: I)) : -1;
1586	if (ExpectedTiedIdx != TiedIdx)
1587	return true;
1588	}
1589	return false;
1590	}
1591
1592	LLT MachineInstr::getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes,
1593	const MachineRegisterInfo &MRI) const {
1594	const MachineOperand &Op = getOperand(i: OpIdx);
1595	if (!Op.isReg())
1596	return LLT{};
1597
1598	if (isVariadic() || OpIdx >= getNumExplicitOperands())
1599	return MRI.getType(Reg: Op.getReg());
1600
1601	auto &OpInfo = getDesc().operands()[OpIdx];
1602	if (!OpInfo.isGenericType())
1603	return MRI.getType(Reg: Op.getReg());
1604
1605	if (PrintedTypes[OpInfo.getGenericTypeIndex()])
1606	return LLT{};
1607
1608	LLT TypeToPrint = MRI.getType(Reg: Op.getReg());
1609	// Don't mark the type index printed if it wasn't actually printed: maybe
1610	// another operand with the same type index has an actual type attached:
1611	if (TypeToPrint.isValid())
1612	PrintedTypes.set(OpInfo.getGenericTypeIndex());
1613	return TypeToPrint;
1614	}
1615
1616	#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1617	LLVM_DUMP_METHOD void MachineInstr::dump() const {
1618	dbgs() << " ";
1619	print(OS&: dbgs());
1620	}
1621
1622	LLVM_DUMP_METHOD void MachineInstr::dumprImpl(
1623	const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth,
1624	SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const {
1625	if (Depth >= MaxDepth)
1626	return;
1627	if (!AlreadySeenInstrs.insert(Ptr: this).second)
1628	return;
1629	// PadToColumn always inserts at least one space.
1630	// Don't mess up the alignment if we don't want any space.
1631	if (Depth)
1632	fdbgs().PadToColumn(NewCol: Depth * 2);
1633	print(OS&: fdbgs());
1634	for (const MachineOperand &MO : operands()) {
1635	if (!MO.isReg() || MO.isDef())
1636	continue;
1637	Register Reg = MO.getReg();
1638	if (Reg.isPhysical())
1639	continue;
1640	const MachineInstr *NewMI = MRI.getUniqueVRegDef(Reg);
1641	if (NewMI == nullptr)
1642	continue;
1643	NewMI->dumprImpl(MRI, Depth: Depth + 1, MaxDepth, AlreadySeenInstrs);
1644	}
1645	}
1646
1647	LLVM_DUMP_METHOD void MachineInstr::dumpr(const MachineRegisterInfo &MRI,
1648	unsigned MaxDepth) const {
1649	SmallPtrSet<const MachineInstr *, 16> AlreadySeenInstrs;
1650	dumprImpl(MRI, Depth: 0, MaxDepth, AlreadySeenInstrs);
1651	}
1652	#endif
1653
1654	void MachineInstr::print(raw_ostream &OS, bool IsStandalone, bool SkipOpers,
1655	bool SkipDebugLoc, bool AddNewLine,
1656	const TargetInstrInfo *TII) const {
1657	const Module *M = nullptr;
1658	const Function *F = nullptr;
1659	if (const MachineFunction *MF = getMFIfAvailable(MI: *this)) {
1660	F = &MF->getFunction();
1661	M = F->getParent();
1662	if (!TII)
1663	TII = MF->getSubtarget().getInstrInfo();
1664	}
1665
1666	ModuleSlotTracker MST(M);
1667	if (F)
1668	MST.incorporateFunction(F: *F);
1669	print(OS, MST, IsStandalone, SkipOpers, SkipDebugLoc, AddNewLine, TII);
1670	}
1671
1672	void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
1673	bool IsStandalone, bool SkipOpers, bool SkipDebugLoc,
1674	bool AddNewLine, const TargetInstrInfo *TII) const {
1675	// We can be a bit tidier if we know the MachineFunction.
1676	const TargetRegisterInfo *TRI = nullptr;
1677	const MachineRegisterInfo *MRI = nullptr;
1678	const TargetIntrinsicInfo *IntrinsicInfo = nullptr;
1679	tryToGetTargetInfo(MI: *this, TRI, MRI, IntrinsicInfo, TII);
1680
1681	if (isCFIInstruction())
1682	assert(getNumOperands() == 1 && "Expected 1 operand in CFI instruction");
1683
1684	SmallBitVector PrintedTypes(8);
1685	bool ShouldPrintRegisterTies = IsStandalone || hasComplexRegisterTies();
1686	auto getTiedOperandIdx = [&](unsigned OpIdx) {
1687	if (!ShouldPrintRegisterTies)
1688	return 0U;
1689	const MachineOperand &MO = getOperand(i: OpIdx);
1690	if (MO.isReg() && MO.isTied() && !MO.isDef())
1691	return findTiedOperandIdx(OpIdx);
1692	return 0U;
1693	};
1694	unsigned StartOp = 0;
1695	unsigned e = getNumOperands();
1696
1697	// Print explicitly defined operands on the left of an assignment syntax.
1698	while (StartOp < e) {
1699	const MachineOperand &MO = getOperand(i: StartOp);
1700	if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
1701	break;
1702
1703	if (StartOp != 0)
1704	OS << ", ";
1705
1706	LLT TypeToPrint = MRI ? getTypeToPrint(OpIdx: StartOp, PrintedTypes, MRI: *MRI) : LLT{};
1707	unsigned TiedOperandIdx = getTiedOperandIdx(StartOp);
1708	MO.print(os&: OS, MST, TypeToPrint, OpIdx: StartOp, /*PrintDef=*/false, IsStandalone,
1709	ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
1710	++StartOp;
1711	}
1712
1713	if (StartOp != 0)
1714	OS << " = ";
1715
1716	if (getFlag(Flag: MachineInstr::FrameSetup))
1717	OS << "frame-setup ";
1718	if (getFlag(Flag: MachineInstr::FrameDestroy))
1719	OS << "frame-destroy ";
1720	if (getFlag(Flag: MachineInstr::FmNoNans))
1721	OS << "nnan ";
1722	if (getFlag(Flag: MachineInstr::FmNoInfs))
1723	OS << "ninf ";
1724	if (getFlag(Flag: MachineInstr::FmNsz))
1725	OS << "nsz ";
1726	if (getFlag(Flag: MachineInstr::FmArcp))
1727	OS << "arcp ";
1728	if (getFlag(Flag: MachineInstr::FmContract))
1729	OS << "contract ";
1730	if (getFlag(Flag: MachineInstr::FmAfn))
1731	OS << "afn ";
1732	if (getFlag(Flag: MachineInstr::FmReassoc))
1733	OS << "reassoc ";
1734	if (getFlag(Flag: MachineInstr::NoUWrap))
1735	OS << "nuw ";
1736	if (getFlag(Flag: MachineInstr::NoSWrap))
1737	OS << "nsw ";
1738	if (getFlag(Flag: MachineInstr::IsExact))
1739	OS << "exact ";
1740	if (getFlag(Flag: MachineInstr::NoFPExcept))
1741	OS << "nofpexcept ";
1742	if (getFlag(Flag: MachineInstr::NoMerge))
1743	OS << "nomerge ";
1744	if (getFlag(Flag: MachineInstr::NonNeg))
1745	OS << "nneg ";
1746	if (getFlag(Flag: MachineInstr::Disjoint))
1747	OS << "disjoint ";
1748
1749	// Print the opcode name.
1750	if (TII)
1751	OS << TII->getName(Opcode: getOpcode());
1752	else
1753	OS << "UNKNOWN";
1754
1755	if (SkipOpers)
1756	return;
1757
1758	// Print the rest of the operands.
1759	bool FirstOp = true;
1760	unsigned AsmDescOp = ~0u;
1761	unsigned AsmOpCount = 0;
1762
1763	if (isInlineAsm() && e >= InlineAsm::MIOp_FirstOperand) {
1764	// Print asm string.
1765	OS << " ";
1766	const unsigned OpIdx = InlineAsm::MIOp_AsmString;
1767	LLT TypeToPrint = MRI ? getTypeToPrint(OpIdx, PrintedTypes, MRI: *MRI) : LLT{};
1768	unsigned TiedOperandIdx = getTiedOperandIdx(OpIdx);
1769	getOperand(i: OpIdx).print(os&: OS, MST, TypeToPrint, OpIdx, /*PrintDef=*/true, IsStandalone,
1770	ShouldPrintRegisterTies, TiedOperandIdx, TRI,
1771	IntrinsicInfo);
1772
1773	// Print HasSideEffects, MayLoad, MayStore, IsAlignStack
1774	unsigned ExtraInfo = getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
1775	if (ExtraInfo & InlineAsm::Extra_HasSideEffects)
1776	OS << " [sideeffect]";
1777	if (ExtraInfo & InlineAsm::Extra_MayLoad)
1778	OS << " [mayload]";
1779	if (ExtraInfo & InlineAsm::Extra_MayStore)
1780	OS << " [maystore]";
1781	if (ExtraInfo & InlineAsm::Extra_IsConvergent)
1782	OS << " [isconvergent]";
1783	if (ExtraInfo & InlineAsm::Extra_IsAlignStack)
1784	OS << " [alignstack]";
1785	if (getInlineAsmDialect() == InlineAsm::AD_ATT)
1786	OS << " [attdialect]";
1787	if (getInlineAsmDialect() == InlineAsm::AD_Intel)
1788	OS << " [inteldialect]";
1789
1790	StartOp = AsmDescOp = InlineAsm::MIOp_FirstOperand;
1791	FirstOp = false;
1792	}
1793
1794	for (unsigned i = StartOp, e = getNumOperands(); i != e; ++i) {
1795	const MachineOperand &MO = getOperand(i);
1796
1797	if (FirstOp) FirstOp = false; else OS << ",";
1798	OS << " ";
1799
1800	if (isDebugValueLike() && MO.isMetadata()) {
1801	// Pretty print DBG_VALUE* instructions.
1802	auto *DIV = dyn_cast<DILocalVariable>(Val: MO.getMetadata());
1803	if (DIV && !DIV->getName().empty())
1804	OS << "!\"" << DIV->getName() << '\"';
1805	else {
1806	LLT TypeToPrint = MRI ? getTypeToPrint(OpIdx: i, PrintedTypes, MRI: *MRI) : LLT{};
1807	unsigned TiedOperandIdx = getTiedOperandIdx(i);
1808	MO.print(os&: OS, MST, TypeToPrint, OpIdx: i, /*PrintDef=*/true, IsStandalone,
1809	ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
1810	}
1811	} else if (isDebugLabel() && MO.isMetadata()) {
1812	// Pretty print DBG_LABEL instructions.
1813	auto *DIL = dyn_cast<DILabel>(Val: MO.getMetadata());
1814	if (DIL && !DIL->getName().empty())
1815	OS << "\"" << DIL->getName() << '\"';
1816	else {
1817	LLT TypeToPrint = MRI ? getTypeToPrint(OpIdx: i, PrintedTypes, MRI: *MRI) : LLT{};
1818	unsigned TiedOperandIdx = getTiedOperandIdx(i);
1819	MO.print(os&: OS, MST, TypeToPrint, OpIdx: i, /*PrintDef=*/true, IsStandalone,
1820	ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
1821	}
1822	} else if (i == AsmDescOp && MO.isImm()) {
1823	// Pretty print the inline asm operand descriptor.
1824	OS << '$' << AsmOpCount++;
1825	unsigned Flag = MO.getImm();
1826	const InlineAsm::Flag F(Flag);
1827	OS << ":[";
1828	OS << F.getKindName();
1829
1830	unsigned RCID;
1831	if (!F.isImmKind() && !F.isMemKind() && F.hasRegClassConstraint(RC&: RCID)) {
1832	if (TRI) {
1833	OS << ':' << TRI->getRegClassName(Class: TRI->getRegClass(i: RCID));
1834	} else
1835	OS << ":RC" << RCID;
1836	}
1837
1838	if (F.isMemKind()) {
1839	const InlineAsm::ConstraintCode MCID = F.getMemoryConstraintID();
1840	OS << ":" << InlineAsm::getMemConstraintName(C: MCID);
1841	}
1842
1843	unsigned TiedTo;
1844	if (F.isUseOperandTiedToDef(Idx&: TiedTo))
1845	OS << " tiedto:$" << TiedTo;
1846
1847	if ((F.isRegDefKind() || F.isRegDefEarlyClobberKind() ||
1848	F.isRegUseKind()) &&
1849	F.getRegMayBeFolded()) {
1850	OS << " foldable";
1851	}
1852
1853	OS << ']';
1854
1855	// Compute the index of the next operand descriptor.
1856	AsmDescOp += 1 + F.getNumOperandRegisters();
1857	} else {
1858	LLT TypeToPrint = MRI ? getTypeToPrint(OpIdx: i, PrintedTypes, MRI: *MRI) : LLT{};
1859	unsigned TiedOperandIdx = getTiedOperandIdx(i);
1860	if (MO.isImm() && isOperandSubregIdx(OpIdx: i))
1861	MachineOperand::printSubRegIdx(OS, Index: MO.getImm(), TRI);
1862	else
1863	MO.print(os&: OS, MST, TypeToPrint, OpIdx: i, /*PrintDef=*/true, IsStandalone,
1864	ShouldPrintRegisterTies, TiedOperandIdx, TRI, IntrinsicInfo);
1865	}
1866	}
1867
1868	// Print any optional symbols attached to this instruction as-if they were
1869	// operands.
1870	if (MCSymbol *PreInstrSymbol = getPreInstrSymbol()) {
1871	if (!FirstOp) {
1872	FirstOp = false;
1873	OS << ',';
1874	}
1875	OS << " pre-instr-symbol ";
1876	MachineOperand::printSymbol(OS, Sym&: *PreInstrSymbol);
1877	}
1878	if (MCSymbol *PostInstrSymbol = getPostInstrSymbol()) {
1879	if (!FirstOp) {
1880	FirstOp = false;
1881	OS << ',';
1882	}
1883	OS << " post-instr-symbol ";
1884	MachineOperand::printSymbol(OS, Sym&: *PostInstrSymbol);
1885	}
1886	if (MDNode *HeapAllocMarker = getHeapAllocMarker()) {
1887	if (!FirstOp) {
1888	FirstOp = false;
1889	OS << ',';
1890	}
1891	OS << " heap-alloc-marker ";
1892	HeapAllocMarker->printAsOperand(OS, MST);
1893	}
1894	if (MDNode *PCSections = getPCSections()) {
1895	if (!FirstOp) {
1896	FirstOp = false;
1897	OS << ',';
1898	}
1899	OS << " pcsections ";
1900	PCSections->printAsOperand(OS, MST);
1901	}
1902	if (MDNode *MMRA = getMMRAMetadata()) {
1903	if (!FirstOp) {
1904	FirstOp = false;
1905	OS << ',';
1906	}
1907	OS << " mmra ";
1908	MMRA->printAsOperand(OS, MST);
1909	}
1910	if (uint32_t CFIType = getCFIType()) {
1911	if (!FirstOp)
1912	OS << ',';
1913	OS << " cfi-type " << CFIType;
1914	}
1915
1916	if (DebugInstrNum) {
1917	if (!FirstOp)
1918	OS << ",";
1919	OS << " debug-instr-number " << DebugInstrNum;
1920	}
1921
1922	if (!SkipDebugLoc) {
1923	if (const DebugLoc &DL = getDebugLoc()) {
1924	if (!FirstOp)
1925	OS << ',';
1926	OS << " debug-location ";
1927	DL->printAsOperand(OS, MST);
1928	}
1929	}
1930
1931	if (!memoperands_empty()) {
1932	SmallVector<StringRef, 0> SSNs;
1933	const LLVMContext *Context = nullptr;
1934	std::unique_ptr<LLVMContext> CtxPtr;
1935	const MachineFrameInfo *MFI = nullptr;
1936	if (const MachineFunction *MF = getMFIfAvailable(MI: *this)) {
1937	MFI = &MF->getFrameInfo();
1938	Context = &MF->getFunction().getContext();
1939	} else {
1940	CtxPtr = std::make_unique<LLVMContext>();
1941	Context = CtxPtr.get();
1942	}
1943
1944	OS << " :: ";
1945	bool NeedComma = false;
1946	for (const MachineMemOperand *Op : memoperands()) {
1947	if (NeedComma)
1948	OS << ", ";
1949	Op->print(OS, MST, SSNs, Context: *Context, MFI, TII);
1950	NeedComma = true;
1951	}
1952	}
1953
1954	if (SkipDebugLoc)
1955	return;
1956
1957	bool HaveSemi = false;
1958
1959	// Print debug location information.
1960	if (const DebugLoc &DL = getDebugLoc()) {
1961	if (!HaveSemi) {
1962	OS << ';';
1963	HaveSemi = true;
1964	}
1965	OS << ' ';
1966	DL.print(OS);
1967	}
1968
1969	// Print extra comments for DEBUG_VALUE and friends if they are well-formed.
1970	if ((isNonListDebugValue() && getNumOperands() >= 4) ||
1971	(isDebugValueList() && getNumOperands() >= 2) ||
1972	(isDebugRef() && getNumOperands() >= 3)) {
1973	if (getDebugVariableOp().isMetadata()) {
1974	if (!HaveSemi) {
1975	OS << ";";
1976	HaveSemi = true;
1977	}
1978	auto *DV = getDebugVariable();
1979	OS << " line no:" << DV->getLine();
1980	if (isIndirectDebugValue())
1981	OS << " indirect";
1982	}
1983	}
1984	// TODO: DBG_LABEL
1985
1986	if (AddNewLine)
1987	OS << '\n';
1988	}
1989
1990	bool MachineInstr::addRegisterKilled(Register IncomingReg,
1991	const TargetRegisterInfo *RegInfo,
1992	bool AddIfNotFound) {
1993	bool isPhysReg = IncomingReg.isPhysical();
1994	bool hasAliases = isPhysReg &&
1995	MCRegAliasIterator(IncomingReg, RegInfo, false).isValid();
1996	bool Found = false;
1997	SmallVector<unsigned,4> DeadOps;
1998	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
1999	MachineOperand &MO = getOperand(i);
2000	if (!MO.isReg() || !MO.isUse() || MO.isUndef())
2001	continue;
2002
2003	// DEBUG_VALUE nodes do not contribute to code generation and should
2004	// always be ignored. Failure to do so may result in trying to modify
2005	// KILL flags on DEBUG_VALUE nodes.
2006	if (MO.isDebug())
2007	continue;
2008
2009	Register Reg = MO.getReg();
2010	if (!Reg)
2011	continue;
2012
2013	if (Reg == IncomingReg) {
2014	if (!Found) {
2015	if (MO.isKill())
2016	// The register is already marked kill.
2017	return true;
2018	if (isPhysReg && isRegTiedToDefOperand(UseOpIdx: i))
2019	// Two-address uses of physregs must not be marked kill.
2020	return true;
2021	MO.setIsKill();
2022	Found = true;
2023	}
2024	} else if (hasAliases && MO.isKill() && Reg.isPhysical()) {
2025	// A super-register kill already exists.
2026	if (RegInfo->isSuperRegister(RegA: IncomingReg, RegB: Reg))
2027	return true;
2028	if (RegInfo->isSubRegister(RegA: IncomingReg, RegB: Reg))
2029	DeadOps.push_back(Elt: i);
2030	}
2031	}
2032
2033	// Trim unneeded kill operands.
2034	while (!DeadOps.empty()) {
2035	unsigned OpIdx = DeadOps.back();
2036	if (getOperand(i: OpIdx).isImplicit() &&
2037	(!isInlineAsm() || findInlineAsmFlagIdx(OpIdx) < 0))
2038	removeOperand(OpNo: OpIdx);
2039	else
2040	getOperand(i: OpIdx).setIsKill(false);
2041	DeadOps.pop_back();
2042	}
2043
2044	// If not found, this means an alias of one of the operands is killed. Add a
2045	// new implicit operand if required.
2046	if (!Found && AddIfNotFound) {
2047	addOperand(Op: MachineOperand::CreateReg(Reg: IncomingReg,
2048	isDef: false /*IsDef*/,
2049	isImp: true /*IsImp*/,
2050	isKill: true /*IsKill*/));
2051	return true;
2052	}
2053	return Found;
2054	}
2055
2056	void MachineInstr::clearRegisterKills(Register Reg,
2057	const TargetRegisterInfo *RegInfo) {
2058	if (!Reg.isPhysical())
2059	RegInfo = nullptr;
2060	for (MachineOperand &MO : operands()) {
2061	if (!MO.isReg() || !MO.isUse() || !MO.isKill())
2062	continue;
2063	Register OpReg = MO.getReg();
2064	if ((RegInfo && RegInfo->regsOverlap(RegA: Reg, RegB: OpReg)) || Reg == OpReg)
2065	MO.setIsKill(false);
2066	}
2067	}
2068
2069	bool MachineInstr::addRegisterDead(Register Reg,
2070	const TargetRegisterInfo *RegInfo,
2071	bool AddIfNotFound) {
2072	bool isPhysReg = Reg.isPhysical();
2073	bool hasAliases = isPhysReg &&
2074	MCRegAliasIterator(Reg, RegInfo, false).isValid();
2075	bool Found = false;
2076	SmallVector<unsigned,4> DeadOps;
2077	for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
2078	MachineOperand &MO = getOperand(i);
2079	if (!MO.isReg() || !MO.isDef())
2080	continue;
2081	Register MOReg = MO.getReg();
2082	if (!MOReg)
2083	continue;
2084
2085	if (MOReg == Reg) {
2086	MO.setIsDead();
2087	Found = true;
2088	} else if (hasAliases && MO.isDead() && MOReg.isPhysical()) {
2089	// There exists a super-register that's marked dead.
2090	if (RegInfo->isSuperRegister(RegA: Reg, RegB: MOReg))
2091	return true;
2092	if (RegInfo->isSubRegister(RegA: Reg, RegB: MOReg))
2093	DeadOps.push_back(Elt: i);
2094	}
2095	}
2096
2097	// Trim unneeded dead operands.
2098	while (!DeadOps.empty()) {
2099	unsigned OpIdx = DeadOps.back();
2100	if (getOperand(i: OpIdx).isImplicit() &&
2101	(!isInlineAsm() || findInlineAsmFlagIdx(OpIdx) < 0))
2102	removeOperand(OpNo: OpIdx);
2103	else
2104	getOperand(i: OpIdx).setIsDead(false);
2105	DeadOps.pop_back();
2106	}
2107
2108	// If not found, this means an alias of one of the operands is dead. Add a
2109	// new implicit operand if required.
2110	if (Found || !AddIfNotFound)
2111	return Found;
2112
2113	addOperand(Op: MachineOperand::CreateReg(Reg,
2114	isDef: true /*IsDef*/,
2115	isImp: true /*IsImp*/,
2116	isKill: false /*IsKill*/,
2117	isDead: true /*IsDead*/));
2118	return true;
2119	}
2120
2121	void MachineInstr::clearRegisterDeads(Register Reg) {
2122	for (MachineOperand &MO : operands()) {
2123	if (!MO.isReg() || !MO.isDef() || MO.getReg() != Reg)
2124	continue;
2125	MO.setIsDead(false);
2126	}
2127	}
2128
2129	void MachineInstr::setRegisterDefReadUndef(Register Reg, bool IsUndef) {
2130	for (MachineOperand &MO : operands()) {
2131	if (!MO.isReg() || !MO.isDef() || MO.getReg() != Reg || MO.getSubReg() == 0)
2132	continue;
2133	MO.setIsUndef(IsUndef);
2134	}
2135	}
2136
2137	void MachineInstr::addRegisterDefined(Register Reg,
2138	const TargetRegisterInfo *RegInfo) {
2139	if (Reg.isPhysical()) {
2140	MachineOperand *MO = findRegisterDefOperand(Reg, TRI: RegInfo, isDead: false, Overlap: false);
2141	if (MO)
2142	return;
2143	} else {
2144	for (const MachineOperand &MO : operands()) {
2145	if (MO.isReg() && MO.getReg() == Reg && MO.isDef() &&
2146	MO.getSubReg() == 0)
2147	return;
2148	}
2149	}
2150	addOperand(Op: MachineOperand::CreateReg(Reg,
2151	isDef: true /*IsDef*/,
2152	isImp: true /*IsImp*/));
2153	}
2154
2155	void MachineInstr::setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs,
2156	const TargetRegisterInfo &TRI) {
2157	bool HasRegMask = false;
2158	for (MachineOperand &MO : operands()) {
2159	if (MO.isRegMask()) {
2160	HasRegMask = true;
2161	continue;
2162	}
2163	if (!MO.isReg() || !MO.isDef()) continue;
2164	Register Reg = MO.getReg();
2165	if (!Reg.isPhysical())
2166	continue;
2167	// If there are no uses, including partial uses, the def is dead.
2168	if (llvm::none_of(Range&: UsedRegs,
2169	P: [&](MCRegister Use) { return TRI.regsOverlap(RegA: Use, RegB: Reg); }))
2170	MO.setIsDead();
2171	}
2172
2173	// This is a call with a register mask operand.
2174	// Mask clobbers are always dead, so add defs for the non-dead defines.
2175	if (HasRegMask)
2176	for (const Register &UsedReg : UsedRegs)
2177	addRegisterDefined(Reg: UsedReg, RegInfo: &TRI);
2178	}
2179
2180	unsigned
2181	MachineInstrExpressionTrait::getHashValue(const MachineInstr* const &MI) {
2182	// Build up a buffer of hash code components.
2183	SmallVector<size_t, 16> HashComponents;
2184	HashComponents.reserve(N: MI->getNumOperands() + 1);
2185	HashComponents.push_back(Elt: MI->getOpcode());
2186	for (const MachineOperand &MO : MI->operands()) {
2187	if (MO.isReg() && MO.isDef() && MO.getReg().isVirtual())
2188	continue; // Skip virtual register defs.
2189
2190	HashComponents.push_back(Elt: hash_value(MO));
2191	}
2192	return hash_combine_range(first: HashComponents.begin(), last: HashComponents.end());
2193	}
2194
2195	void MachineInstr::emitError(StringRef Msg) const {
2196	// Find the source location cookie.
2197	uint64_t LocCookie = 0;
2198	const MDNode *LocMD = nullptr;
2199	for (unsigned i = getNumOperands(); i != 0; --i) {
2200	if (getOperand(i: i-1).isMetadata() &&
2201	(LocMD = getOperand(i: i-1).getMetadata()) &&
2202	LocMD->getNumOperands() != 0) {
2203	if (const ConstantInt *CI =
2204	mdconst::dyn_extract<ConstantInt>(MD: LocMD->getOperand(I: 0))) {
2205	LocCookie = CI->getZExtValue();
2206	break;
2207	}
2208	}
2209	}
2210
2211	if (const MachineBasicBlock *MBB = getParent())
2212	if (const MachineFunction *MF = MBB->getParent())
2213	return MF->getMMI().getModule()->getContext().emitError(LocCookie, ErrorStr: Msg);
2214	report_fatal_error(reason: Msg);
2215	}
2216
2217	MachineInstrBuilder llvm::BuildMI(MachineFunction &MF, const DebugLoc &DL,
2218	const MCInstrDesc &MCID, bool IsIndirect,
2219	Register Reg, const MDNode *Variable,
2220	const MDNode *Expr) {
2221	assert(isa<DILocalVariable>(Variable) && "not a variable");
2222	assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
2223	assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
2224	"Expected inlined-at fields to agree");
2225	auto MIB = BuildMI(MF, MIMD: DL, MCID).addReg(RegNo: Reg);
2226	if (IsIndirect)
2227	MIB.addImm(Val: 0U);
2228	else
2229	MIB.addReg(RegNo: 0U);
2230	return MIB.addMetadata(MD: Variable).addMetadata(MD: Expr);
2231	}
2232
2233	MachineInstrBuilder llvm::BuildMI(MachineFunction &MF, const DebugLoc &DL,
2234	const MCInstrDesc &MCID, bool IsIndirect,
2235	ArrayRef<MachineOperand> DebugOps,
2236	const MDNode *Variable, const MDNode *Expr) {
2237	assert(isa<DILocalVariable>(Variable) && "not a variable");
2238	assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
2239	assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
2240	"Expected inlined-at fields to agree");
2241	if (MCID.Opcode == TargetOpcode::DBG_VALUE) {
2242	assert(DebugOps.size() == 1 &&
2243	"DBG_VALUE must contain exactly one debug operand");
2244	MachineOperand DebugOp = DebugOps[0];
2245	if (DebugOp.isReg())
2246	return BuildMI(MF, DL, MCID, IsIndirect, Reg: DebugOp.getReg(), Variable,
2247	Expr);
2248
2249	auto MIB = BuildMI(MF, MIMD: DL, MCID).add(MO: DebugOp);
2250	if (IsIndirect)
2251	MIB.addImm(Val: 0U);
2252	else
2253	MIB.addReg(RegNo: 0U);
2254	return MIB.addMetadata(MD: Variable).addMetadata(MD: Expr);
2255	}
2256
2257	auto MIB = BuildMI(MF, MIMD: DL, MCID);
2258	MIB.addMetadata(MD: Variable).addMetadata(MD: Expr);
2259	for (const MachineOperand &DebugOp : DebugOps)
2260	if (DebugOp.isReg())
2261	MIB.addReg(RegNo: DebugOp.getReg());
2262	else
2263	MIB.add(MO: DebugOp);
2264	return MIB;
2265	}
2266
2267	MachineInstrBuilder llvm::BuildMI(MachineBasicBlock &BB,
2268	MachineBasicBlock::iterator I,
2269	const DebugLoc &DL, const MCInstrDesc &MCID,
2270	bool IsIndirect, Register Reg,
2271	const MDNode *Variable, const MDNode *Expr) {
2272	MachineFunction &MF = *BB.getParent();
2273	MachineInstr *MI = BuildMI(MF, DL, MCID, IsIndirect, Reg, Variable, Expr);
2274	BB.insert(I, MI);
2275	return MachineInstrBuilder(MF, MI);
2276	}
2277
2278	MachineInstrBuilder llvm::BuildMI(MachineBasicBlock &BB,
2279	MachineBasicBlock::iterator I,
2280	const DebugLoc &DL, const MCInstrDesc &MCID,
2281	bool IsIndirect,
2282	ArrayRef<MachineOperand> DebugOps,
2283	const MDNode *Variable, const MDNode *Expr) {
2284	MachineFunction &MF = *BB.getParent();
2285	MachineInstr *MI =
2286	BuildMI(MF, DL, MCID, IsIndirect, DebugOps, Variable, Expr);
2287	BB.insert(I, MI);
2288	return MachineInstrBuilder(MF, *MI);
2289	}
2290
2291	/// Compute the new DIExpression to use with a DBG_VALUE for a spill slot.
2292	/// This prepends DW_OP_deref when spilling an indirect DBG_VALUE.
2293	static const DIExpression *
2294	computeExprForSpill(const MachineInstr &MI,
2295	SmallVectorImpl<const MachineOperand *> &SpilledOperands) {
2296	assert(MI.getDebugVariable()->isValidLocationForIntrinsic(MI.getDebugLoc()) &&
2297	"Expected inlined-at fields to agree");
2298
2299	const DIExpression *Expr = MI.getDebugExpression();
2300	if (MI.isIndirectDebugValue()) {
2301	assert(MI.getDebugOffset().getImm() == 0 &&
2302	"DBG_VALUE with nonzero offset");
2303	Expr = DIExpression::prepend(Expr, Flags: DIExpression::DerefBefore);
2304	} else if (MI.isDebugValueList()) {
2305	// We will replace the spilled register with a frame index, so
2306	// immediately deref all references to the spilled register.
2307	std::array<uint64_t, 1> Ops{._M_elems: {dwarf::DW_OP_deref}};
2308	for (const MachineOperand *Op : SpilledOperands) {
2309	unsigned OpIdx = MI.getDebugOperandIndex(Op);
2310	Expr = DIExpression::appendOpsToArg(Expr, Ops, ArgNo: OpIdx);
2311	}
2312	}
2313	return Expr;
2314	}
2315	static const DIExpression *computeExprForSpill(const MachineInstr &MI,
2316	Register SpillReg) {
2317	assert(MI.hasDebugOperandForReg(SpillReg) && "Spill Reg is not used in MI.");
2318	SmallVector<const MachineOperand *> SpillOperands;
2319	for (const MachineOperand &Op : MI.getDebugOperandsForReg(Reg: SpillReg))
2320	SpillOperands.push_back(Elt: &Op);
2321	return computeExprForSpill(MI, SpilledOperands&: SpillOperands);
2322	}
2323
2324	MachineInstr *llvm::buildDbgValueForSpill(MachineBasicBlock &BB,
2325	MachineBasicBlock::iterator I,
2326	const MachineInstr &Orig,
2327	int FrameIndex, Register SpillReg) {
2328	assert(!Orig.isDebugRef() &&
2329	"DBG_INSTR_REF should not reference a virtual register.");
2330	const DIExpression *Expr = computeExprForSpill(MI: Orig, SpillReg);
2331	MachineInstrBuilder NewMI =
2332	BuildMI(BB, I, MIMD: Orig.getDebugLoc(), MCID: Orig.getDesc());
2333	// Non-Variadic Operands: Location, Offset, Variable, Expression
2334	// Variadic Operands: Variable, Expression, Locations...
2335	if (Orig.isNonListDebugValue())
2336	NewMI.addFrameIndex(Idx: FrameIndex).addImm(Val: 0U);
2337	NewMI.addMetadata(MD: Orig.getDebugVariable()).addMetadata(MD: Expr);
2338	if (Orig.isDebugValueList()) {
2339	for (const MachineOperand &Op : Orig.debug_operands())
2340	if (Op.isReg() && Op.getReg() == SpillReg)
2341	NewMI.addFrameIndex(Idx: FrameIndex);
2342	else
2343	NewMI.add(MO: MachineOperand(Op));
2344	}
2345	return NewMI;
2346	}
2347	MachineInstr *llvm::buildDbgValueForSpill(
2348	MachineBasicBlock &BB, MachineBasicBlock::iterator I,
2349	const MachineInstr &Orig, int FrameIndex,
2350	SmallVectorImpl<const MachineOperand *> &SpilledOperands) {
2351	const DIExpression *Expr = computeExprForSpill(MI: Orig, SpilledOperands);
2352	MachineInstrBuilder NewMI =
2353	BuildMI(BB, I, MIMD: Orig.getDebugLoc(), MCID: Orig.getDesc());
2354	// Non-Variadic Operands: Location, Offset, Variable, Expression
2355	// Variadic Operands: Variable, Expression, Locations...
2356	if (Orig.isNonListDebugValue())
2357	NewMI.addFrameIndex(Idx: FrameIndex).addImm(Val: 0U);
2358	NewMI.addMetadata(MD: Orig.getDebugVariable()).addMetadata(MD: Expr);
2359	if (Orig.isDebugValueList()) {
2360	for (const MachineOperand &Op : Orig.debug_operands())
2361	if (is_contained(Range&: SpilledOperands, Element: &Op))
2362	NewMI.addFrameIndex(Idx: FrameIndex);
2363	else
2364	NewMI.add(MO: MachineOperand(Op));
2365	}
2366	return NewMI;
2367	}
2368
2369	void llvm::updateDbgValueForSpill(MachineInstr &Orig, int FrameIndex,
2370	Register Reg) {
2371	const DIExpression *Expr = computeExprForSpill(MI: Orig, SpillReg: Reg);
2372	if (Orig.isNonListDebugValue())
2373	Orig.getDebugOffset().ChangeToImmediate(ImmVal: 0U);
2374	for (MachineOperand &Op : Orig.getDebugOperandsForReg(Reg))
2375	Op.ChangeToFrameIndex(Idx: FrameIndex);
2376	Orig.getDebugExpressionOp().setMetadata(Expr);
2377	}
2378
2379	void MachineInstr::collectDebugValues(
2380	SmallVectorImpl<MachineInstr *> &DbgValues) {
2381	MachineInstr &MI = *this;
2382	if (!MI.getOperand(i: 0).isReg())
2383	return;
2384
2385	MachineBasicBlock::iterator DI = MI; ++DI;
2386	for (MachineBasicBlock::iterator DE = MI.getParent()->end();
2387	DI != DE; ++DI) {
2388	if (!DI->isDebugValue())
2389	return;
2390	if (DI->hasDebugOperandForReg(Reg: MI.getOperand(i: 0).getReg()))
2391	DbgValues.push_back(Elt: &*DI);
2392	}
2393	}
2394
2395	void MachineInstr::changeDebugValuesDefReg(Register Reg) {
2396	// Collect matching debug values.
2397	SmallVector<MachineInstr *, 2> DbgValues;
2398
2399	if (!getOperand(i: 0).isReg())
2400	return;
2401
2402	Register DefReg = getOperand(i: 0).getReg();
2403	auto *MRI = getRegInfo();
2404	for (auto &MO : MRI->use_operands(Reg: DefReg)) {
2405	auto *DI = MO.getParent();
2406	if (!DI->isDebugValue())
2407	continue;
2408	if (DI->hasDebugOperandForReg(Reg: DefReg)) {
2409	DbgValues.push_back(Elt: DI);
2410	}
2411	}
2412
2413	// Propagate Reg to debug value instructions.
2414	for (auto *DBI : DbgValues)
2415	for (MachineOperand &Op : DBI->getDebugOperandsForReg(Reg: DefReg))
2416	Op.setReg(Reg);
2417	}
2418
2419	using MMOList = SmallVector<const MachineMemOperand *, 2>;
2420
2421	static LocationSize getSpillSlotSize(const MMOList &Accesses,
2422	const MachineFrameInfo &MFI) {
2423	uint64_t Size = 0;
2424	for (const auto *A : Accesses) {
2425	if (MFI.isSpillSlotObjectIndex(
2426	ObjectIdx: cast<FixedStackPseudoSourceValue>(Val: A->getPseudoValue())
2427	->getFrameIndex())) {
2428	LocationSize S = A->getSize();
2429	if (!S.hasValue())
2430	return LocationSize::beforeOrAfterPointer();
2431	Size += S.getValue();
2432	}
2433	}
2434	return Size;
2435	}
2436
2437	std::optional<LocationSize>
2438	MachineInstr::getSpillSize(const TargetInstrInfo *TII) const {
2439	int FI;
2440	if (TII->isStoreToStackSlotPostFE(MI: *this, FrameIndex&: FI)) {
2441	const MachineFrameInfo &MFI = getMF()->getFrameInfo();
2442	if (MFI.isSpillSlotObjectIndex(ObjectIdx: FI))
2443	return (*memoperands_begin())->getSize();
2444	}
2445	return std::nullopt;
2446	}
2447
2448	std::optional<LocationSize>
2449	MachineInstr::getFoldedSpillSize(const TargetInstrInfo *TII) const {
2450	MMOList Accesses;
2451	if (TII->hasStoreToStackSlot(MI: *this, Accesses))
2452	return getSpillSlotSize(Accesses, MFI: getMF()->getFrameInfo());
2453	return std::nullopt;
2454	}
2455
2456	std::optional<LocationSize>
2457	MachineInstr::getRestoreSize(const TargetInstrInfo *TII) const {
2458	int FI;
2459	if (TII->isLoadFromStackSlotPostFE(MI: *this, FrameIndex&: FI)) {
2460	const MachineFrameInfo &MFI = getMF()->getFrameInfo();
2461	if (MFI.isSpillSlotObjectIndex(ObjectIdx: FI))
2462	return (*memoperands_begin())->getSize();
2463	}
2464	return std::nullopt;
2465	}
2466
2467	std::optional<LocationSize>
2468	MachineInstr::getFoldedRestoreSize(const TargetInstrInfo *TII) const {
2469	MMOList Accesses;
2470	if (TII->hasLoadFromStackSlot(MI: *this, Accesses))
2471	return getSpillSlotSize(Accesses, MFI: getMF()->getFrameInfo());
2472	return std::nullopt;
2473	}
2474
2475	unsigned MachineInstr::getDebugInstrNum() {
2476	if (DebugInstrNum == 0)
2477	DebugInstrNum = getParent()->getParent()->getNewDebugInstrNum();
2478	return DebugInstrNum;
2479	}
2480
2481	unsigned MachineInstr::getDebugInstrNum(MachineFunction &MF) {
2482	if (DebugInstrNum == 0)
2483	DebugInstrNum = MF.getNewDebugInstrNum();
2484	return DebugInstrNum;
2485	}
2486
2487	std::tuple<LLT, LLT> MachineInstr::getFirst2LLTs() const {
2488	return std::tuple(getRegInfo()->getType(Reg: getOperand(i: 0).getReg()),
2489	getRegInfo()->getType(Reg: getOperand(i: 1).getReg()));
2490	}
2491
2492	std::tuple<LLT, LLT, LLT> MachineInstr::getFirst3LLTs() const {
2493	return std::tuple(getRegInfo()->getType(Reg: getOperand(i: 0).getReg()),
2494	getRegInfo()->getType(Reg: getOperand(i: 1).getReg()),
2495	getRegInfo()->getType(Reg: getOperand(i: 2).getReg()));
2496	}
2497
2498	std::tuple<LLT, LLT, LLT, LLT> MachineInstr::getFirst4LLTs() const {
2499	return std::tuple(getRegInfo()->getType(Reg: getOperand(i: 0).getReg()),
2500	getRegInfo()->getType(Reg: getOperand(i: 1).getReg()),
2501	getRegInfo()->getType(Reg: getOperand(i: 2).getReg()),
2502	getRegInfo()->getType(Reg: getOperand(i: 3).getReg()));
2503	}
2504
2505	std::tuple<LLT, LLT, LLT, LLT, LLT> MachineInstr::getFirst5LLTs() const {
2506	return std::tuple(getRegInfo()->getType(Reg: getOperand(i: 0).getReg()),
2507	getRegInfo()->getType(Reg: getOperand(i: 1).getReg()),
2508	getRegInfo()->getType(Reg: getOperand(i: 2).getReg()),
2509	getRegInfo()->getType(Reg: getOperand(i: 3).getReg()),
2510	getRegInfo()->getType(Reg: getOperand(i: 4).getReg()));
2511	}
2512
2513	std::tuple<Register, LLT, Register, LLT>
2514	MachineInstr::getFirst2RegLLTs() const {
2515	Register Reg0 = getOperand(i: 0).getReg();
2516	Register Reg1 = getOperand(i: 1).getReg();
2517	return std::tuple(Reg0, getRegInfo()->getType(Reg: Reg0), Reg1,
2518	getRegInfo()->getType(Reg: Reg1));
2519	}
2520
2521	std::tuple<Register, LLT, Register, LLT, Register, LLT>
2522	MachineInstr::getFirst3RegLLTs() const {
2523	Register Reg0 = getOperand(i: 0).getReg();
2524	Register Reg1 = getOperand(i: 1).getReg();
2525	Register Reg2 = getOperand(i: 2).getReg();
2526	return std::tuple(Reg0, getRegInfo()->getType(Reg: Reg0), Reg1,
2527	getRegInfo()->getType(Reg: Reg1), Reg2,
2528	getRegInfo()->getType(Reg: Reg2));
2529	}
2530
2531	std::tuple<Register, LLT, Register, LLT, Register, LLT, Register, LLT>
2532	MachineInstr::getFirst4RegLLTs() const {
2533	Register Reg0 = getOperand(i: 0).getReg();
2534	Register Reg1 = getOperand(i: 1).getReg();
2535	Register Reg2 = getOperand(i: 2).getReg();
2536	Register Reg3 = getOperand(i: 3).getReg();
2537	return std::tuple(
2538	Reg0, getRegInfo()->getType(Reg: Reg0), Reg1, getRegInfo()->getType(Reg: Reg1),
2539	Reg2, getRegInfo()->getType(Reg: Reg2), Reg3, getRegInfo()->getType(Reg: Reg3));
2540	}
2541
2542	std::tuple<Register, LLT, Register, LLT, Register, LLT, Register, LLT, Register,
2543	LLT>
2544	MachineInstr::getFirst5RegLLTs() const {
2545	Register Reg0 = getOperand(i: 0).getReg();
2546	Register Reg1 = getOperand(i: 1).getReg();
2547	Register Reg2 = getOperand(i: 2).getReg();
2548	Register Reg3 = getOperand(i: 3).getReg();
2549	Register Reg4 = getOperand(i: 4).getReg();
2550	return std::tuple(
2551	Reg0, getRegInfo()->getType(Reg: Reg0), Reg1, getRegInfo()->getType(Reg: Reg1),
2552	Reg2, getRegInfo()->getType(Reg: Reg2), Reg3, getRegInfo()->getType(Reg: Reg3),
2553	Reg4, getRegInfo()->getType(Reg: Reg4));
2554	}
2555
2556	void MachineInstr::insert(mop_iterator InsertBefore,
2557	ArrayRef<MachineOperand> Ops) {
2558	assert(InsertBefore != nullptr && "invalid iterator");
2559	assert(InsertBefore->getParent() == this &&
2560	"iterator points to operand of other inst");
2561	if (Ops.empty())
2562	return;
2563
2564	// Do one pass to untie operands.
2565	SmallDenseMap<unsigned, unsigned> TiedOpIndices;
2566	for (const MachineOperand &MO : operands()) {
2567	if (MO.isReg() && MO.isTied()) {
2568	unsigned OpNo = getOperandNo(I: &MO);
2569	unsigned TiedTo = findTiedOperandIdx(OpIdx: OpNo);
2570	TiedOpIndices[OpNo] = TiedTo;
2571	untieRegOperand(OpIdx: OpNo);
2572	}
2573	}
2574
2575	unsigned OpIdx = getOperandNo(I: InsertBefore);
2576	unsigned NumOperands = getNumOperands();
2577	unsigned OpsToMove = NumOperands - OpIdx;
2578
2579	SmallVector<MachineOperand> MovingOps;
2580	MovingOps.reserve(N: OpsToMove);
2581
2582	for (unsigned I = 0; I < OpsToMove; ++I) {
2583	MovingOps.emplace_back(Args&: getOperand(i: OpIdx));
2584	removeOperand(OpNo: OpIdx);
2585	}
2586	for (const MachineOperand &MO : Ops)
2587	addOperand(Op: MO);
2588	for (const MachineOperand &OpMoved : MovingOps)
2589	addOperand(Op: OpMoved);
2590
2591	// Re-tie operands.
2592	for (auto [Tie1, Tie2] : TiedOpIndices) {
2593	if (Tie1 >= OpIdx)
2594	Tie1 += Ops.size();
2595	if (Tie2 >= OpIdx)
2596	Tie2 += Ops.size();
2597	tieOperands(DefIdx: Tie1, UseIdx: Tie2);
2598	}
2599	}
2600
2601	bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
2602	assert(OpId && "expected non-zero operand id");
2603	assert(isInlineAsm() && "should only be used on inline asm");
2604
2605	if (!getOperand(i: OpId).isReg())
2606	return false;
2607
2608	const MachineOperand &MD = getOperand(i: OpId - 1);
2609	if (!MD.isImm())
2610	return false;
2611
2612	InlineAsm::Flag F(MD.getImm());
2613	if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind())
2614	return F.getRegMayBeFolded();
2615	return false;
2616	}
2617