1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/CodeGenTypes/MachineValueType.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DebugLoc.h"
38#include "llvm/IR/Instruction.h"
39#include "llvm/IR/Instructions.h"
40#include "llvm/IR/Metadata.h"
41#include "llvm/IR/Operator.h"
42#include "llvm/Support/AlignOf.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56#include <utility>
57
58namespace llvm {
59
60class APInt;
61class Constant;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Returns true if the specified node is a vector where all elements can
122/// be truncated to the specified element size without a loss in meaning.
123bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed);
124
125/// Return true if the node has at least one operand and all operands of the
126/// specified node are ISD::UNDEF.
127bool allOperandsUndef(const SDNode *N);
128
129/// Return true if the specified node is FREEZE(UNDEF).
130bool isFreezeUndef(const SDNode *N);
131
132} // end namespace ISD
133
134//===----------------------------------------------------------------------===//
135/// Unlike LLVM values, Selection DAG nodes may return multiple
136/// values as the result of a computation. Many nodes return multiple values,
137/// from loads (which define a token and a return value) to ADDC (which returns
138/// a result and a carry value), to calls (which may return an arbitrary number
139/// of values).
140///
141/// As such, each use of a SelectionDAG computation must indicate the node that
142/// computes it as well as which return value to use from that node. This pair
143/// of information is represented with the SDValue value type.
144///
145class SDValue {
146 friend struct DenseMapInfo<SDValue>;
147
148 SDNode *Node = nullptr; // The node defining the value we are using.
149 unsigned ResNo = 0; // Which return value of the node we are using.
150
151public:
152 SDValue() = default;
153 SDValue(SDNode *node, unsigned resno);
154
155 /// get the index which selects a specific result in the SDNode
156 unsigned getResNo() const { return ResNo; }
157
158 /// get the SDNode which holds the desired result
159 SDNode *getNode() const { return Node; }
160
161 /// set the SDNode
162 void setNode(SDNode *N) { Node = N; }
163
164 inline SDNode *operator->() const { return Node; }
165
166 bool operator==(const SDValue &O) const {
167 return Node == O.Node && ResNo == O.ResNo;
168 }
169 bool operator!=(const SDValue &O) const {
170 return !operator==(O);
171 }
172 bool operator<(const SDValue &O) const {
173 return std::tie(args: Node, args: ResNo) < std::tie(args: O.Node, args: O.ResNo);
174 }
175 explicit operator bool() const {
176 return Node != nullptr;
177 }
178
179 SDValue getValue(unsigned R) const {
180 return SDValue(Node, R);
181 }
182
183 /// Return true if this node is an operand of N.
184 bool isOperandOf(const SDNode *N) const;
185
186 /// Return the ValueType of the referenced return value.
187 inline EVT getValueType() const;
188
189 /// Return the simple ValueType of the referenced return value.
190 MVT getSimpleValueType() const {
191 return getValueType().getSimpleVT();
192 }
193
194 /// Returns the size of the value in bits.
195 ///
196 /// If the value type is a scalable vector type, the scalable property will
197 /// be set and the runtime size will be a positive integer multiple of the
198 /// base size.
199 TypeSize getValueSizeInBits() const {
200 return getValueType().getSizeInBits();
201 }
202
203 uint64_t getScalarValueSizeInBits() const {
204 return getValueType().getScalarType().getFixedSizeInBits();
205 }
206
207 // Forwarding methods - These forward to the corresponding methods in SDNode.
208 inline unsigned getOpcode() const;
209 inline unsigned getNumOperands() const;
210 inline const SDValue &getOperand(unsigned i) const;
211 inline uint64_t getConstantOperandVal(unsigned i) const;
212 inline const APInt &getConstantOperandAPInt(unsigned i) const;
213 inline bool isTargetMemoryOpcode() const;
214 inline bool isTargetOpcode() const;
215 inline bool isMachineOpcode() const;
216 inline bool isUndef() const;
217 inline unsigned getMachineOpcode() const;
218 inline const DebugLoc &getDebugLoc() const;
219 inline void dump() const;
220 inline void dump(const SelectionDAG *G) const;
221 inline void dumpr() const;
222 inline void dumpr(const SelectionDAG *G) const;
223
224 /// Return true if this operand (which must be a chain) reaches the
225 /// specified operand without crossing any side-effecting instructions.
226 /// In practice, this looks through token factors and non-volatile loads.
227 /// In order to remain efficient, this only
228 /// looks a couple of nodes in, it does not do an exhaustive search.
229 bool reachesChainWithoutSideEffects(SDValue Dest,
230 unsigned Depth = 2) const;
231
232 /// Return true if there are no nodes using value ResNo of Node.
233 inline bool use_empty() const;
234
235 /// Return true if there is exactly one node using value ResNo of Node.
236 inline bool hasOneUse() const;
237};
238
239template<> struct DenseMapInfo<SDValue> {
240 static inline SDValue getEmptyKey() {
241 SDValue V;
242 V.ResNo = -1U;
243 return V;
244 }
245
246 static inline SDValue getTombstoneKey() {
247 SDValue V;
248 V.ResNo = -2U;
249 return V;
250 }
251
252 static unsigned getHashValue(const SDValue &Val) {
253 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
254 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
255 }
256
257 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
258 return LHS == RHS;
259 }
260};
261
262/// Allow casting operators to work directly on
263/// SDValues as if they were SDNode*'s.
264template<> struct simplify_type<SDValue> {
265 using SimpleType = SDNode *;
266
267 static SimpleType getSimplifiedValue(SDValue &Val) {
268 return Val.getNode();
269 }
270};
271template<> struct simplify_type<const SDValue> {
272 using SimpleType = /*const*/ SDNode *;
273
274 static SimpleType getSimplifiedValue(const SDValue &Val) {
275 return Val.getNode();
276 }
277};
278
279/// Represents a use of a SDNode. This class holds an SDValue,
280/// which records the SDNode being used and the result number, a
281/// pointer to the SDNode using the value, and Next and Prev pointers,
282/// which link together all the uses of an SDNode.
283///
284class SDUse {
285 /// Val - The value being used.
286 SDValue Val;
287 /// User - The user of this value.
288 SDNode *User = nullptr;
289 /// Prev, Next - Pointers to the uses list of the SDNode referred by
290 /// this operand.
291 SDUse **Prev = nullptr;
292 SDUse *Next = nullptr;
293
294public:
295 SDUse() = default;
296 SDUse(const SDUse &U) = delete;
297 SDUse &operator=(const SDUse &) = delete;
298
299 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
300 operator const SDValue&() const { return Val; }
301
302 /// If implicit conversion to SDValue doesn't work, the get() method returns
303 /// the SDValue.
304 const SDValue &get() const { return Val; }
305
306 /// This returns the SDNode that contains this Use.
307 SDNode *getUser() { return User; }
308 const SDNode *getUser() const { return User; }
309
310 /// Get the next SDUse in the use list.
311 SDUse *getNext() const { return Next; }
312
313 /// Convenience function for get().getNode().
314 SDNode *getNode() const { return Val.getNode(); }
315 /// Convenience function for get().getResNo().
316 unsigned getResNo() const { return Val.getResNo(); }
317 /// Convenience function for get().getValueType().
318 EVT getValueType() const { return Val.getValueType(); }
319
320 /// Convenience function for get().operator==
321 bool operator==(const SDValue &V) const {
322 return Val == V;
323 }
324
325 /// Convenience function for get().operator!=
326 bool operator!=(const SDValue &V) const {
327 return Val != V;
328 }
329
330 /// Convenience function for get().operator<
331 bool operator<(const SDValue &V) const {
332 return Val < V;
333 }
334
335private:
336 friend class SelectionDAG;
337 friend class SDNode;
338 // TODO: unfriend HandleSDNode once we fix its operand handling.
339 friend class HandleSDNode;
340
341 void setUser(SDNode *p) { User = p; }
342
343 /// Remove this use from its existing use list, assign it the
344 /// given value, and add it to the new value's node's use list.
345 inline void set(const SDValue &V);
346 /// Like set, but only supports initializing a newly-allocated
347 /// SDUse with a non-null value.
348 inline void setInitial(const SDValue &V);
349 /// Like set, but only sets the Node portion of the value,
350 /// leaving the ResNo portion unmodified.
351 inline void setNode(SDNode *N);
352
353 void addToList(SDUse **List) {
354 Next = *List;
355 if (Next) Next->Prev = &Next;
356 Prev = List;
357 *List = this;
358 }
359
360 void removeFromList() {
361 *Prev = Next;
362 if (Next) Next->Prev = Prev;
363 }
364};
365
366/// simplify_type specializations - Allow casting operators to work directly on
367/// SDValues as if they were SDNode*'s.
368template<> struct simplify_type<SDUse> {
369 using SimpleType = SDNode *;
370
371 static SimpleType getSimplifiedValue(SDUse &Val) {
372 return Val.getNode();
373 }
374};
375
376/// These are IR-level optimization flags that may be propagated to SDNodes.
377/// TODO: This data structure should be shared by the IR optimizer and the
378/// the backend.
379struct SDNodeFlags {
380private:
381 bool NoUnsignedWrap : 1;
382 bool NoSignedWrap : 1;
383 bool Exact : 1;
384 bool Disjoint : 1;
385 bool NonNeg : 1;
386 bool NoNaNs : 1;
387 bool NoInfs : 1;
388 bool NoSignedZeros : 1;
389 bool AllowReciprocal : 1;
390 bool AllowContract : 1;
391 bool ApproximateFuncs : 1;
392 bool AllowReassociation : 1;
393
394 // We assume instructions do not raise floating-point exceptions by default,
395 // and only those marked explicitly may do so. We could choose to represent
396 // this via a positive "FPExcept" flags like on the MI level, but having a
397 // negative "NoFPExcept" flag here makes the flag intersection logic more
398 // straightforward.
399 bool NoFPExcept : 1;
400 // Instructions with attached 'unpredictable' metadata on IR level.
401 bool Unpredictable : 1;
402
403public:
404 /// Default constructor turns off all optimization flags.
405 SDNodeFlags()
406 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false),
407 Disjoint(false), NonNeg(false), NoNaNs(false), NoInfs(false),
408 NoSignedZeros(false), AllowReciprocal(false), AllowContract(false),
409 ApproximateFuncs(false), AllowReassociation(false), NoFPExcept(false),
410 Unpredictable(false) {}
411
412 /// Propagate the fast-math-flags from an IR FPMathOperator.
413 void copyFMF(const FPMathOperator &FPMO) {
414 setNoNaNs(FPMO.hasNoNaNs());
415 setNoInfs(FPMO.hasNoInfs());
416 setNoSignedZeros(FPMO.hasNoSignedZeros());
417 setAllowReciprocal(FPMO.hasAllowReciprocal());
418 setAllowContract(FPMO.hasAllowContract());
419 setApproximateFuncs(FPMO.hasApproxFunc());
420 setAllowReassociation(FPMO.hasAllowReassoc());
421 }
422
423 // These are mutators for each flag.
424 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
425 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
426 void setExact(bool b) { Exact = b; }
427 void setDisjoint(bool b) { Disjoint = b; }
428 void setNonNeg(bool b) { NonNeg = b; }
429 void setNoNaNs(bool b) { NoNaNs = b; }
430 void setNoInfs(bool b) { NoInfs = b; }
431 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
432 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
433 void setAllowContract(bool b) { AllowContract = b; }
434 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
435 void setAllowReassociation(bool b) { AllowReassociation = b; }
436 void setNoFPExcept(bool b) { NoFPExcept = b; }
437 void setUnpredictable(bool b) { Unpredictable = b; }
438
439 // These are accessors for each flag.
440 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
441 bool hasNoSignedWrap() const { return NoSignedWrap; }
442 bool hasExact() const { return Exact; }
443 bool hasDisjoint() const { return Disjoint; }
444 bool hasNonNeg() const { return NonNeg; }
445 bool hasNoNaNs() const { return NoNaNs; }
446 bool hasNoInfs() const { return NoInfs; }
447 bool hasNoSignedZeros() const { return NoSignedZeros; }
448 bool hasAllowReciprocal() const { return AllowReciprocal; }
449 bool hasAllowContract() const { return AllowContract; }
450 bool hasApproximateFuncs() const { return ApproximateFuncs; }
451 bool hasAllowReassociation() const { return AllowReassociation; }
452 bool hasNoFPExcept() const { return NoFPExcept; }
453 bool hasUnpredictable() const { return Unpredictable; }
454
455 /// Clear any flags in this flag set that aren't also set in Flags. All
456 /// flags will be cleared if Flags are undefined.
457 void intersectWith(const SDNodeFlags Flags) {
458 NoUnsignedWrap &= Flags.NoUnsignedWrap;
459 NoSignedWrap &= Flags.NoSignedWrap;
460 Exact &= Flags.Exact;
461 Disjoint &= Flags.Disjoint;
462 NonNeg &= Flags.NonNeg;
463 NoNaNs &= Flags.NoNaNs;
464 NoInfs &= Flags.NoInfs;
465 NoSignedZeros &= Flags.NoSignedZeros;
466 AllowReciprocal &= Flags.AllowReciprocal;
467 AllowContract &= Flags.AllowContract;
468 ApproximateFuncs &= Flags.ApproximateFuncs;
469 AllowReassociation &= Flags.AllowReassociation;
470 NoFPExcept &= Flags.NoFPExcept;
471 Unpredictable &= Flags.Unpredictable;
472 }
473};
474
475/// Represents one node in the SelectionDAG.
476///
477class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
478private:
479 /// The operation that this node performs.
480 int32_t NodeType;
481
482public:
483 /// Unique and persistent id per SDNode in the DAG. Used for debug printing.
484 /// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS`
485 /// intentionally because it adds unneeded complexity without noticeable
486 /// benefits (see discussion with @thakis in D120714).
487 uint16_t PersistentId = 0xffff;
488
489protected:
490 // We define a set of mini-helper classes to help us interpret the bits in our
491 // SubclassData. These are designed to fit within a uint16_t so they pack
492 // with PersistentId.
493
494#if defined(_AIX) && (!defined(__GNUC__) || defined(__clang__))
495// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
496// and give the `pack` pragma push semantics.
497#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")
498#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")
499#else
500#define BEGIN_TWO_BYTE_PACK()
501#define END_TWO_BYTE_PACK()
502#endif
503
504BEGIN_TWO_BYTE_PACK()
505 class SDNodeBitfields {
506 friend class SDNode;
507 friend class MemIntrinsicSDNode;
508 friend class MemSDNode;
509 friend class SelectionDAG;
510
511 uint16_t HasDebugValue : 1;
512 uint16_t IsMemIntrinsic : 1;
513 uint16_t IsDivergent : 1;
514 };
515 enum { NumSDNodeBits = 3 };
516
517 class ConstantSDNodeBitfields {
518 friend class ConstantSDNode;
519
520 uint16_t : NumSDNodeBits;
521
522 uint16_t IsOpaque : 1;
523 };
524
525 class MemSDNodeBitfields {
526 friend class MemSDNode;
527 friend class MemIntrinsicSDNode;
528 friend class AtomicSDNode;
529
530 uint16_t : NumSDNodeBits;
531
532 uint16_t IsVolatile : 1;
533 uint16_t IsNonTemporal : 1;
534 uint16_t IsDereferenceable : 1;
535 uint16_t IsInvariant : 1;
536 };
537 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
538
539 class LSBaseSDNodeBitfields {
540 friend class LSBaseSDNode;
541 friend class VPBaseLoadStoreSDNode;
542 friend class MaskedLoadStoreSDNode;
543 friend class MaskedGatherScatterSDNode;
544 friend class VPGatherScatterSDNode;
545
546 uint16_t : NumMemSDNodeBits;
547
548 // This storage is shared between disparate class hierarchies to hold an
549 // enumeration specific to the class hierarchy in use.
550 // LSBaseSDNode => enum ISD::MemIndexedMode
551 // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
552 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
553 // VPGatherScatterSDNode => enum ISD::MemIndexType
554 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
555 uint16_t AddressingMode : 3;
556 };
557 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
558
559 class LoadSDNodeBitfields {
560 friend class LoadSDNode;
561 friend class VPLoadSDNode;
562 friend class VPStridedLoadSDNode;
563 friend class MaskedLoadSDNode;
564 friend class MaskedGatherSDNode;
565 friend class VPGatherSDNode;
566
567 uint16_t : NumLSBaseSDNodeBits;
568
569 uint16_t ExtTy : 2; // enum ISD::LoadExtType
570 uint16_t IsExpanding : 1;
571 };
572
573 class StoreSDNodeBitfields {
574 friend class StoreSDNode;
575 friend class VPStoreSDNode;
576 friend class VPStridedStoreSDNode;
577 friend class MaskedStoreSDNode;
578 friend class MaskedScatterSDNode;
579 friend class VPScatterSDNode;
580
581 uint16_t : NumLSBaseSDNodeBits;
582
583 uint16_t IsTruncating : 1;
584 uint16_t IsCompressing : 1;
585 };
586
587 union {
588 char RawSDNodeBits[sizeof(uint16_t)];
589 SDNodeBitfields SDNodeBits;
590 ConstantSDNodeBitfields ConstantSDNodeBits;
591 MemSDNodeBitfields MemSDNodeBits;
592 LSBaseSDNodeBitfields LSBaseSDNodeBits;
593 LoadSDNodeBitfields LoadSDNodeBits;
594 StoreSDNodeBitfields StoreSDNodeBits;
595 };
596END_TWO_BYTE_PACK()
597#undef BEGIN_TWO_BYTE_PACK
598#undef END_TWO_BYTE_PACK
599
600 // RawSDNodeBits must cover the entirety of the union. This means that all of
601 // the union's members must have size <= RawSDNodeBits. We write the RHS as
602 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
603 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
604 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
605 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
606 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
607 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
608 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
609
610private:
611 friend class SelectionDAG;
612 // TODO: unfriend HandleSDNode once we fix its operand handling.
613 friend class HandleSDNode;
614
615 /// Unique id per SDNode in the DAG.
616 int NodeId = -1;
617
618 /// The values that are used by this operation.
619 SDUse *OperandList = nullptr;
620
621 /// The types of the values this node defines. SDNode's may
622 /// define multiple values simultaneously.
623 const EVT *ValueList;
624
625 /// List of uses for this SDNode.
626 SDUse *UseList = nullptr;
627
628 /// The number of entries in the Operand/Value list.
629 unsigned short NumOperands = 0;
630 unsigned short NumValues;
631
632 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
633 // original LLVM instructions.
634 // This is used for turning off scheduling, because we'll forgo
635 // the normal scheduling algorithms and output the instructions according to
636 // this ordering.
637 unsigned IROrder;
638
639 /// Source line information.
640 DebugLoc debugLoc;
641
642 /// Return a pointer to the specified value type.
643 static const EVT *getValueTypeList(EVT VT);
644
645 SDNodeFlags Flags;
646
647 uint32_t CFIType = 0;
648
649public:
650 //===--------------------------------------------------------------------===//
651 // Accessors
652 //
653
654 /// Return the SelectionDAG opcode value for this node. For
655 /// pre-isel nodes (those for which isMachineOpcode returns false), these
656 /// are the opcode values in the ISD and <target>ISD namespaces. For
657 /// post-isel opcodes, see getMachineOpcode.
658 unsigned getOpcode() const { return (unsigned)NodeType; }
659
660 /// Test if this node has a target-specific opcode (in the
661 /// \<target\>ISD namespace).
662 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
663
664 /// Test if this node has a target-specific opcode that may raise
665 /// FP exceptions (in the \<target\>ISD namespace and greater than
666 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
667 /// opcode are currently automatically considered to possibly raise
668 /// FP exceptions as well.
669 bool isTargetStrictFPOpcode() const {
670 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
671 }
672
673 /// Test if this node has a target-specific
674 /// memory-referencing opcode (in the \<target\>ISD namespace and
675 /// greater than FIRST_TARGET_MEMORY_OPCODE).
676 bool isTargetMemoryOpcode() const {
677 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
678 }
679
680 /// Return true if the type of the node type undefined.
681 bool isUndef() const { return NodeType == ISD::UNDEF; }
682
683 /// Test if this node is a memory intrinsic (with valid pointer information).
684 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
685 /// non-memory intrinsics (with chains) that are not really instances of
686 /// MemSDNode. For such nodes, we need some extra state to determine the
687 /// proper classof relationship.
688 bool isMemIntrinsic() const {
689 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
690 NodeType == ISD::INTRINSIC_VOID) &&
691 SDNodeBits.IsMemIntrinsic;
692 }
693
694 /// Test if this node is a strict floating point pseudo-op.
695 bool isStrictFPOpcode() {
696 switch (NodeType) {
697 default:
698 return false;
699 case ISD::STRICT_FP16_TO_FP:
700 case ISD::STRICT_FP_TO_FP16:
701#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
702 case ISD::STRICT_##DAGN:
703#include "llvm/IR/ConstrainedOps.def"
704 return true;
705 }
706 }
707
708 /// Test if this node is a vector predication operation.
709 bool isVPOpcode() const { return ISD::isVPOpcode(Opcode: getOpcode()); }
710
711 /// Test if this node has a post-isel opcode, directly
712 /// corresponding to a MachineInstr opcode.
713 bool isMachineOpcode() const { return NodeType < 0; }
714
715 /// This may only be called if isMachineOpcode returns
716 /// true. It returns the MachineInstr opcode value that the node's opcode
717 /// corresponds to.
718 unsigned getMachineOpcode() const {
719 assert(isMachineOpcode() && "Not a MachineInstr opcode!");
720 return ~NodeType;
721 }
722
723 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
724 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
725
726 bool isDivergent() const { return SDNodeBits.IsDivergent; }
727
728 /// Return true if there are no uses of this node.
729 bool use_empty() const { return UseList == nullptr; }
730
731 /// Return true if there is exactly one use of this node.
732 bool hasOneUse() const { return hasSingleElement(C: uses()); }
733
734 /// Return the number of uses of this node. This method takes
735 /// time proportional to the number of uses.
736 size_t use_size() const { return std::distance(first: use_begin(), last: use_end()); }
737
738 /// Return the unique node id.
739 int getNodeId() const { return NodeId; }
740
741 /// Set unique node id.
742 void setNodeId(int Id) { NodeId = Id; }
743
744 /// Return the node ordering.
745 unsigned getIROrder() const { return IROrder; }
746
747 /// Set the node ordering.
748 void setIROrder(unsigned Order) { IROrder = Order; }
749
750 /// Return the source location info.
751 const DebugLoc &getDebugLoc() const { return debugLoc; }
752
753 /// Set source location info. Try to avoid this, putting
754 /// it in the constructor is preferable.
755 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
756
757 /// This class provides iterator support for SDUse
758 /// operands that use a specific SDNode.
759 class use_iterator {
760 friend class SDNode;
761
762 SDUse *Op = nullptr;
763
764 explicit use_iterator(SDUse *op) : Op(op) {}
765
766 public:
767 using iterator_category = std::forward_iterator_tag;
768 using value_type = SDUse;
769 using difference_type = std::ptrdiff_t;
770 using pointer = value_type *;
771 using reference = value_type &;
772
773 use_iterator() = default;
774 use_iterator(const use_iterator &I) = default;
775 use_iterator &operator=(const use_iterator &) = default;
776
777 bool operator==(const use_iterator &x) const { return Op == x.Op; }
778 bool operator!=(const use_iterator &x) const {
779 return !operator==(x);
780 }
781
782 /// Return true if this iterator is at the end of uses list.
783 bool atEnd() const { return Op == nullptr; }
784
785 // Iterator traversal: forward iteration only.
786 use_iterator &operator++() { // Preincrement
787 assert(Op && "Cannot increment end iterator!");
788 Op = Op->getNext();
789 return *this;
790 }
791
792 use_iterator operator++(int) { // Postincrement
793 use_iterator tmp = *this; ++*this; return tmp;
794 }
795
796 /// Retrieve a pointer to the current user node.
797 SDNode *operator*() const {
798 assert(Op && "Cannot dereference end iterator!");
799 return Op->getUser();
800 }
801
802 SDNode *operator->() const { return operator*(); }
803
804 SDUse &getUse() const { return *Op; }
805
806 /// Retrieve the operand # of this use in its user.
807 unsigned getOperandNo() const {
808 assert(Op && "Cannot dereference end iterator!");
809 return (unsigned)(Op - Op->getUser()->OperandList);
810 }
811 };
812
813 /// Provide iteration support to walk over all uses of an SDNode.
814 use_iterator use_begin() const {
815 return use_iterator(UseList);
816 }
817
818 static use_iterator use_end() { return use_iterator(nullptr); }
819
820 inline iterator_range<use_iterator> uses() {
821 return make_range(x: use_begin(), y: use_end());
822 }
823 inline iterator_range<use_iterator> uses() const {
824 return make_range(x: use_begin(), y: use_end());
825 }
826
827 /// Return true if there are exactly NUSES uses of the indicated value.
828 /// This method ignores uses of other values defined by this operation.
829 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
830
831 /// Return true if there are any use of the indicated value.
832 /// This method ignores uses of other values defined by this operation.
833 bool hasAnyUseOfValue(unsigned Value) const;
834
835 /// Return true if this node is the only use of N.
836 bool isOnlyUserOf(const SDNode *N) const;
837
838 /// Return true if this node is an operand of N.
839 bool isOperandOf(const SDNode *N) const;
840
841 /// Return true if this node is a predecessor of N.
842 /// NOTE: Implemented on top of hasPredecessor and every bit as
843 /// expensive. Use carefully.
844 bool isPredecessorOf(const SDNode *N) const {
845 return N->hasPredecessor(N: this);
846 }
847
848 /// Return true if N is a predecessor of this node.
849 /// N is either an operand of this node, or can be reached by recursively
850 /// traversing up the operands.
851 /// NOTE: This is an expensive method. Use it carefully.
852 bool hasPredecessor(const SDNode *N) const;
853
854 /// Returns true if N is a predecessor of any node in Worklist. This
855 /// helper keeps Visited and Worklist sets externally to allow unions
856 /// searches to be performed in parallel, caching of results across
857 /// queries and incremental addition to Worklist. Stops early if N is
858 /// found but will resume. Remember to clear Visited and Worklists
859 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
860 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
861 /// topologically ordered (Operands have strictly smaller node id) and search
862 /// can be pruned leveraging this.
863 static bool hasPredecessorHelper(const SDNode *N,
864 SmallPtrSetImpl<const SDNode *> &Visited,
865 SmallVectorImpl<const SDNode *> &Worklist,
866 unsigned int MaxSteps = 0,
867 bool TopologicalPrune = false) {
868 SmallVector<const SDNode *, 8> DeferredNodes;
869 if (Visited.count(Ptr: N))
870 return true;
871
872 // Node Id's are assigned in three places: As a topological
873 // ordering (> 0), during legalization (results in values set to
874 // 0), new nodes (set to -1). If N has a topolgical id then we
875 // know that all nodes with ids smaller than it cannot be
876 // successors and we need not check them. Filter out all node
877 // that can't be matches. We add them to the worklist before exit
878 // in case of multiple calls. Note that during selection the topological id
879 // may be violated if a node's predecessor is selected before it. We mark
880 // this at selection negating the id of unselected successors and
881 // restricting topological pruning to positive ids.
882
883 int NId = N->getNodeId();
884 // If we Invalidated the Id, reconstruct original NId.
885 if (NId < -1)
886 NId = -(NId + 1);
887
888 bool Found = false;
889 while (!Worklist.empty()) {
890 const SDNode *M = Worklist.pop_back_val();
891 int MId = M->getNodeId();
892 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
893 (MId > 0) && (MId < NId)) {
894 DeferredNodes.push_back(Elt: M);
895 continue;
896 }
897 for (const SDValue &OpV : M->op_values()) {
898 SDNode *Op = OpV.getNode();
899 if (Visited.insert(Ptr: Op).second)
900 Worklist.push_back(Elt: Op);
901 if (Op == N)
902 Found = true;
903 }
904 if (Found)
905 break;
906 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
907 break;
908 }
909 // Push deferred nodes back on worklist.
910 Worklist.append(in_start: DeferredNodes.begin(), in_end: DeferredNodes.end());
911 // If we bailed early, conservatively return found.
912 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
913 return true;
914 return Found;
915 }
916
917 /// Return true if all the users of N are contained in Nodes.
918 /// NOTE: Requires at least one match, but doesn't require them all.
919 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
920
921 /// Return the number of values used by this operation.
922 unsigned getNumOperands() const { return NumOperands; }
923
924 /// Return the maximum number of operands that a SDNode can hold.
925 static constexpr size_t getMaxNumOperands() {
926 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
927 }
928
929 /// Helper method returns the integer value of a ConstantSDNode operand.
930 inline uint64_t getConstantOperandVal(unsigned Num) const;
931
932 /// Helper method returns the zero-extended integer value of a ConstantSDNode.
933 inline uint64_t getAsZExtVal() const;
934
935 /// Helper method returns the APInt of a ConstantSDNode operand.
936 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
937
938 /// Helper method returns the APInt value of a ConstantSDNode.
939 inline const APInt &getAsAPIntVal() const;
940
941 const SDValue &getOperand(unsigned Num) const {
942 assert(Num < NumOperands && "Invalid child # of SDNode!");
943 return OperandList[Num];
944 }
945
946 using op_iterator = SDUse *;
947
948 op_iterator op_begin() const { return OperandList; }
949 op_iterator op_end() const { return OperandList+NumOperands; }
950 ArrayRef<SDUse> ops() const { return ArrayRef(op_begin(), op_end()); }
951
952 /// Iterator for directly iterating over the operand SDValue's.
953 struct value_op_iterator
954 : iterator_adaptor_base<value_op_iterator, op_iterator,
955 std::random_access_iterator_tag, SDValue,
956 ptrdiff_t, value_op_iterator *,
957 value_op_iterator *> {
958 explicit value_op_iterator(SDUse *U = nullptr)
959 : iterator_adaptor_base(U) {}
960
961 const SDValue &operator*() const { return I->get(); }
962 };
963
964 iterator_range<value_op_iterator> op_values() const {
965 return make_range(x: value_op_iterator(op_begin()),
966 y: value_op_iterator(op_end()));
967 }
968
969 SDVTList getVTList() const {
970 SDVTList X = { .VTs: ValueList, .NumVTs: NumValues };
971 return X;
972 }
973
974 /// If this node has a glue operand, return the node
975 /// to which the glue operand points. Otherwise return NULL.
976 SDNode *getGluedNode() const {
977 if (getNumOperands() != 0 &&
978 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
979 return getOperand(Num: getNumOperands()-1).getNode();
980 return nullptr;
981 }
982
983 /// If this node has a glue value with a user, return
984 /// the user (there is at most one). Otherwise return NULL.
985 SDNode *getGluedUser() const {
986 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
987 if (UI.getUse().get().getValueType() == MVT::Glue)
988 return *UI;
989 return nullptr;
990 }
991
992 SDNodeFlags getFlags() const { return Flags; }
993 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
994
995 /// Clear any flags in this node that aren't also set in Flags.
996 /// If Flags is not in a defined state then this has no effect.
997 void intersectFlagsWith(const SDNodeFlags Flags);
998
999 void setCFIType(uint32_t Type) { CFIType = Type; }
1000 uint32_t getCFIType() const { return CFIType; }
1001
1002 /// Return the number of values defined/returned by this operator.
1003 unsigned getNumValues() const { return NumValues; }
1004
1005 /// Return the type of a specified result.
1006 EVT getValueType(unsigned ResNo) const {
1007 assert(ResNo < NumValues && "Illegal result number!");
1008 return ValueList[ResNo];
1009 }
1010
1011 /// Return the type of a specified result as a simple type.
1012 MVT getSimpleValueType(unsigned ResNo) const {
1013 return getValueType(ResNo).getSimpleVT();
1014 }
1015
1016 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
1017 ///
1018 /// If the value type is a scalable vector type, the scalable property will
1019 /// be set and the runtime size will be a positive integer multiple of the
1020 /// base size.
1021 TypeSize getValueSizeInBits(unsigned ResNo) const {
1022 return getValueType(ResNo).getSizeInBits();
1023 }
1024
1025 using value_iterator = const EVT *;
1026
1027 value_iterator value_begin() const { return ValueList; }
1028 value_iterator value_end() const { return ValueList+NumValues; }
1029 iterator_range<value_iterator> values() const {
1030 return llvm::make_range(x: value_begin(), y: value_end());
1031 }
1032
1033 /// Return the opcode of this operation for printing.
1034 std::string getOperationName(const SelectionDAG *G = nullptr) const;
1035 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
1036 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
1037 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
1038 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1039 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1040
1041 /// Print a SelectionDAG node and all children down to
1042 /// the leaves. The given SelectionDAG allows target-specific nodes
1043 /// to be printed in human-readable form. Unlike printr, this will
1044 /// print the whole DAG, including children that appear multiple
1045 /// times.
1046 ///
1047 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1048
1049 /// Print a SelectionDAG node and children up to
1050 /// depth "depth." The given SelectionDAG allows target-specific
1051 /// nodes to be printed in human-readable form. Unlike printr, this
1052 /// will print children that appear multiple times wherever they are
1053 /// used.
1054 ///
1055 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1056 unsigned depth = 100) const;
1057
1058 /// Dump this node, for debugging.
1059 void dump() const;
1060
1061 /// Dump (recursively) this node and its use-def subgraph.
1062 void dumpr() const;
1063
1064 /// Dump this node, for debugging.
1065 /// The given SelectionDAG allows target-specific nodes to be printed
1066 /// in human-readable form.
1067 void dump(const SelectionDAG *G) const;
1068
1069 /// Dump (recursively) this node and its use-def subgraph.
1070 /// The given SelectionDAG allows target-specific nodes to be printed
1071 /// in human-readable form.
1072 void dumpr(const SelectionDAG *G) const;
1073
1074 /// printrFull to dbgs(). The given SelectionDAG allows
1075 /// target-specific nodes to be printed in human-readable form.
1076 /// Unlike dumpr, this will print the whole DAG, including children
1077 /// that appear multiple times.
1078 void dumprFull(const SelectionDAG *G = nullptr) const;
1079
1080 /// printrWithDepth to dbgs(). The given
1081 /// SelectionDAG allows target-specific nodes to be printed in
1082 /// human-readable form. Unlike dumpr, this will print children
1083 /// that appear multiple times wherever they are used.
1084 ///
1085 void dumprWithDepth(const SelectionDAG *G = nullptr,
1086 unsigned depth = 100) const;
1087
1088 /// Gather unique data for the node.
1089 void Profile(FoldingSetNodeID &ID) const;
1090
1091 /// This method should only be used by the SDUse class.
1092 void addUse(SDUse &U) { U.addToList(List: &UseList); }
1093
1094protected:
1095 static SDVTList getSDVTList(EVT VT) {
1096 SDVTList Ret = { .VTs: getValueTypeList(VT), .NumVTs: 1 };
1097 return Ret;
1098 }
1099
1100 /// Create an SDNode.
1101 ///
1102 /// SDNodes are created without any operands, and never own the operand
1103 /// storage. To add operands, see SelectionDAG::createOperands.
1104 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1105 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1106 IROrder(Order), debugLoc(std::move(dl)) {
1107 memset(s: &RawSDNodeBits, c: 0, n: sizeof(RawSDNodeBits));
1108 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
1109 assert(NumValues == VTs.NumVTs &&
1110 "NumValues wasn't wide enough for its operands!");
1111 }
1112
1113 /// Release the operands and set this node to have zero operands.
1114 void DropOperands();
1115};
1116
1117/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1118/// into SDNode creation functions.
1119/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1120/// from the original Instruction, and IROrder is the ordinal position of
1121/// the instruction.
1122/// When an SDNode is created after the DAG is being built, both DebugLoc and
1123/// the IROrder are propagated from the original SDNode.
1124/// So SDLoc class provides two constructors besides the default one, one to
1125/// be used by the DAGBuilder, the other to be used by others.
1126class SDLoc {
1127private:
1128 DebugLoc DL;
1129 int IROrder = 0;
1130
1131public:
1132 SDLoc() = default;
1133 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1134 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1135 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1136 assert(Order >= 0 && "bad IROrder");
1137 if (I)
1138 DL = I->getDebugLoc();
1139 }
1140
1141 unsigned getIROrder() const { return IROrder; }
1142 const DebugLoc &getDebugLoc() const { return DL; }
1143};
1144
1145// Define inline functions from the SDValue class.
1146
1147inline SDValue::SDValue(SDNode *node, unsigned resno)
1148 : Node(node), ResNo(resno) {
1149 // Explicitly check for !ResNo to avoid use-after-free, because there are
1150 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1151 // combines.
1152 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&
1153 "Invalid result number for the given node!");
1154 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.");
1155}
1156
1157inline unsigned SDValue::getOpcode() const {
1158 return Node->getOpcode();
1159}
1160
1161inline EVT SDValue::getValueType() const {
1162 return Node->getValueType(ResNo);
1163}
1164
1165inline unsigned SDValue::getNumOperands() const {
1166 return Node->getNumOperands();
1167}
1168
1169inline const SDValue &SDValue::getOperand(unsigned i) const {
1170 return Node->getOperand(Num: i);
1171}
1172
1173inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1174 return Node->getConstantOperandVal(Num: i);
1175}
1176
1177inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1178 return Node->getConstantOperandAPInt(Num: i);
1179}
1180
1181inline bool SDValue::isTargetOpcode() const {
1182 return Node->isTargetOpcode();
1183}
1184
1185inline bool SDValue::isTargetMemoryOpcode() const {
1186 return Node->isTargetMemoryOpcode();
1187}
1188
1189inline bool SDValue::isMachineOpcode() const {
1190 return Node->isMachineOpcode();
1191}
1192
1193inline unsigned SDValue::getMachineOpcode() const {
1194 return Node->getMachineOpcode();
1195}
1196
1197inline bool SDValue::isUndef() const {
1198 return Node->isUndef();
1199}
1200
1201inline bool SDValue::use_empty() const {
1202 return !Node->hasAnyUseOfValue(Value: ResNo);
1203}
1204
1205inline bool SDValue::hasOneUse() const {
1206 return Node->hasNUsesOfValue(NUses: 1, Value: ResNo);
1207}
1208
1209inline const DebugLoc &SDValue::getDebugLoc() const {
1210 return Node->getDebugLoc();
1211}
1212
1213inline void SDValue::dump() const {
1214 return Node->dump();
1215}
1216
1217inline void SDValue::dump(const SelectionDAG *G) const {
1218 return Node->dump(G);
1219}
1220
1221inline void SDValue::dumpr() const {
1222 return Node->dumpr();
1223}
1224
1225inline void SDValue::dumpr(const SelectionDAG *G) const {
1226 return Node->dumpr(G);
1227}
1228
1229// Define inline functions from the SDUse class.
1230
1231inline void SDUse::set(const SDValue &V) {
1232 if (Val.getNode()) removeFromList();
1233 Val = V;
1234 if (V.getNode())
1235 V->addUse(U&: *this);
1236}
1237
1238inline void SDUse::setInitial(const SDValue &V) {
1239 Val = V;
1240 V->addUse(U&: *this);
1241}
1242
1243inline void SDUse::setNode(SDNode *N) {
1244 if (Val.getNode()) removeFromList();
1245 Val.setNode(N);
1246 if (N) N->addUse(U&: *this);
1247}
1248
1249/// This class is used to form a handle around another node that
1250/// is persistent and is updated across invocations of replaceAllUsesWith on its
1251/// operand. This node should be directly created by end-users and not added to
1252/// the AllNodes list.
1253class HandleSDNode : public SDNode {
1254 SDUse Op;
1255
1256public:
1257 explicit HandleSDNode(SDValue X)
1258 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1259 // HandleSDNodes are never inserted into the DAG, so they won't be
1260 // auto-numbered. Use ID 65535 as a sentinel.
1261 PersistentId = 0xffff;
1262
1263 // Manually set up the operand list. This node type is special in that it's
1264 // always stack allocated and SelectionDAG does not manage its operands.
1265 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1266 // be so special.
1267 Op.setUser(this);
1268 Op.setInitial(X);
1269 NumOperands = 1;
1270 OperandList = &Op;
1271 }
1272 ~HandleSDNode();
1273
1274 const SDValue &getValue() const { return Op; }
1275};
1276
1277class AddrSpaceCastSDNode : public SDNode {
1278private:
1279 unsigned SrcAddrSpace;
1280 unsigned DestAddrSpace;
1281
1282public:
1283 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1284 unsigned SrcAS, unsigned DestAS);
1285
1286 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1287 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1288
1289 static bool classof(const SDNode *N) {
1290 return N->getOpcode() == ISD::ADDRSPACECAST;
1291 }
1292};
1293
1294/// This is an abstract virtual class for memory operations.
1295class MemSDNode : public SDNode {
1296private:
1297 // VT of in-memory value.
1298 EVT MemoryVT;
1299
1300protected:
1301 /// Memory reference information.
1302 MachineMemOperand *MMO;
1303
1304public:
1305 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1306 EVT memvt, MachineMemOperand *MMO);
1307
1308 bool readMem() const { return MMO->isLoad(); }
1309 bool writeMem() const { return MMO->isStore(); }
1310
1311 /// Returns alignment and volatility of the memory access
1312 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1313 Align getAlign() const { return MMO->getAlign(); }
1314
1315 /// Return the SubclassData value, without HasDebugValue. This contains an
1316 /// encoding of the volatile flag, as well as bits used by subclasses. This
1317 /// function should only be used to compute a FoldingSetNodeID value.
1318 /// The HasDebugValue bit is masked out because CSE map needs to match
1319 /// nodes with debug info with nodes without debug info. Same is about
1320 /// isDivergent bit.
1321 unsigned getRawSubclassData() const {
1322 uint16_t Data;
1323 union {
1324 char RawSDNodeBits[sizeof(uint16_t)];
1325 SDNodeBitfields SDNodeBits;
1326 };
1327 memcpy(dest: &RawSDNodeBits, src: &this->RawSDNodeBits, n: sizeof(this->RawSDNodeBits));
1328 SDNodeBits.HasDebugValue = 0;
1329 SDNodeBits.IsDivergent = false;
1330 memcpy(dest: &Data, src: &RawSDNodeBits, n: sizeof(RawSDNodeBits));
1331 return Data;
1332 }
1333
1334 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1335 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1336 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1337 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1338
1339 // Returns the offset from the location of the access.
1340 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1341
1342 /// Returns the AA info that describes the dereference.
1343 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1344
1345 /// Returns the Ranges that describes the dereference.
1346 const MDNode *getRanges() const { return MMO->getRanges(); }
1347
1348 /// Returns the synchronization scope ID for this memory operation.
1349 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1350
1351 /// Return the atomic ordering requirements for this memory operation. For
1352 /// cmpxchg atomic operations, return the atomic ordering requirements when
1353 /// store occurs.
1354 AtomicOrdering getSuccessOrdering() const {
1355 return MMO->getSuccessOrdering();
1356 }
1357
1358 /// Return a single atomic ordering that is at least as strong as both the
1359 /// success and failure orderings for an atomic operation. (For operations
1360 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1361 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1362
1363 /// Return true if the memory operation ordering is Unordered or higher.
1364 bool isAtomic() const { return MMO->isAtomic(); }
1365
1366 /// Returns true if the memory operation doesn't imply any ordering
1367 /// constraints on surrounding memory operations beyond the normal memory
1368 /// aliasing rules.
1369 bool isUnordered() const { return MMO->isUnordered(); }
1370
1371 /// Returns true if the memory operation is neither atomic or volatile.
1372 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1373
1374 /// Return the type of the in-memory value.
1375 EVT getMemoryVT() const { return MemoryVT; }
1376
1377 /// Return a MachineMemOperand object describing the memory
1378 /// reference performed by operation.
1379 MachineMemOperand *getMemOperand() const { return MMO; }
1380
1381 const MachinePointerInfo &getPointerInfo() const {
1382 return MMO->getPointerInfo();
1383 }
1384
1385 /// Return the address space for the associated pointer
1386 unsigned getAddressSpace() const {
1387 return getPointerInfo().getAddrSpace();
1388 }
1389
1390 /// Update this MemSDNode's MachineMemOperand information
1391 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1392 /// This must only be used when the new alignment applies to all users of
1393 /// this MachineMemOperand.
1394 void refineAlignment(const MachineMemOperand *NewMMO) {
1395 MMO->refineAlignment(MMO: NewMMO);
1396 }
1397
1398 const SDValue &getChain() const { return getOperand(Num: 0); }
1399
1400 const SDValue &getBasePtr() const {
1401 switch (getOpcode()) {
1402 case ISD::STORE:
1403 case ISD::ATOMIC_STORE:
1404 case ISD::VP_STORE:
1405 case ISD::MSTORE:
1406 case ISD::VP_SCATTER:
1407 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1408 return getOperand(Num: 2);
1409 case ISD::MGATHER:
1410 case ISD::MSCATTER:
1411 return getOperand(Num: 3);
1412 default:
1413 return getOperand(Num: 1);
1414 }
1415 }
1416
1417 // Methods to support isa and dyn_cast
1418 static bool classof(const SDNode *N) {
1419 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1420 // with either an intrinsic or a target opcode.
1421 switch (N->getOpcode()) {
1422 case ISD::LOAD:
1423 case ISD::STORE:
1424 case ISD::PREFETCH:
1425 case ISD::ATOMIC_CMP_SWAP:
1426 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1427 case ISD::ATOMIC_SWAP:
1428 case ISD::ATOMIC_LOAD_ADD:
1429 case ISD::ATOMIC_LOAD_SUB:
1430 case ISD::ATOMIC_LOAD_AND:
1431 case ISD::ATOMIC_LOAD_CLR:
1432 case ISD::ATOMIC_LOAD_OR:
1433 case ISD::ATOMIC_LOAD_XOR:
1434 case ISD::ATOMIC_LOAD_NAND:
1435 case ISD::ATOMIC_LOAD_MIN:
1436 case ISD::ATOMIC_LOAD_MAX:
1437 case ISD::ATOMIC_LOAD_UMIN:
1438 case ISD::ATOMIC_LOAD_UMAX:
1439 case ISD::ATOMIC_LOAD_FADD:
1440 case ISD::ATOMIC_LOAD_FSUB:
1441 case ISD::ATOMIC_LOAD_FMAX:
1442 case ISD::ATOMIC_LOAD_FMIN:
1443 case ISD::ATOMIC_LOAD_UINC_WRAP:
1444 case ISD::ATOMIC_LOAD_UDEC_WRAP:
1445 case ISD::ATOMIC_LOAD:
1446 case ISD::ATOMIC_STORE:
1447 case ISD::MLOAD:
1448 case ISD::MSTORE:
1449 case ISD::MGATHER:
1450 case ISD::MSCATTER:
1451 case ISD::VP_LOAD:
1452 case ISD::VP_STORE:
1453 case ISD::VP_GATHER:
1454 case ISD::VP_SCATTER:
1455 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1456 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1457 case ISD::GET_FPENV_MEM:
1458 case ISD::SET_FPENV_MEM:
1459 return true;
1460 default:
1461 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1462 }
1463 }
1464};
1465
1466/// This is an SDNode representing atomic operations.
1467class AtomicSDNode : public MemSDNode {
1468public:
1469 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1470 EVT MemVT, MachineMemOperand *MMO)
1471 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1472 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||
1473 MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
1474 }
1475
1476 const SDValue &getBasePtr() const {
1477 return getOpcode() == ISD::ATOMIC_STORE ? getOperand(Num: 2) : getOperand(Num: 1);
1478 }
1479 const SDValue &getVal() const {
1480 return getOpcode() == ISD::ATOMIC_STORE ? getOperand(Num: 1) : getOperand(Num: 2);
1481 }
1482
1483 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1484 /// otherwise.
1485 bool isCompareAndSwap() const {
1486 unsigned Op = getOpcode();
1487 return Op == ISD::ATOMIC_CMP_SWAP ||
1488 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1489 }
1490
1491 /// For cmpxchg atomic operations, return the atomic ordering requirements
1492 /// when store does not occur.
1493 AtomicOrdering getFailureOrdering() const {
1494 assert(isCompareAndSwap() && "Must be cmpxchg operation");
1495 return MMO->getFailureOrdering();
1496 }
1497
1498 // Methods to support isa and dyn_cast
1499 static bool classof(const SDNode *N) {
1500 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1501 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1502 N->getOpcode() == ISD::ATOMIC_SWAP ||
1503 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1504 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1505 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1506 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1507 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1508 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1509 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1510 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1511 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1512 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1513 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1514 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1515 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1516 N->getOpcode() == ISD::ATOMIC_LOAD_FMAX ||
1517 N->getOpcode() == ISD::ATOMIC_LOAD_FMIN ||
1518 N->getOpcode() == ISD::ATOMIC_LOAD_UINC_WRAP ||
1519 N->getOpcode() == ISD::ATOMIC_LOAD_UDEC_WRAP ||
1520 N->getOpcode() == ISD::ATOMIC_LOAD ||
1521 N->getOpcode() == ISD::ATOMIC_STORE;
1522 }
1523};
1524
1525/// This SDNode is used for target intrinsics that touch
1526/// memory and need an associated MachineMemOperand. Its opcode may be
1527/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1528/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1529class MemIntrinsicSDNode : public MemSDNode {
1530public:
1531 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1532 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1533 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1534 SDNodeBits.IsMemIntrinsic = true;
1535 }
1536
1537 // Methods to support isa and dyn_cast
1538 static bool classof(const SDNode *N) {
1539 // We lower some target intrinsics to their target opcode
1540 // early a node with a target opcode can be of this class
1541 return N->isMemIntrinsic() ||
1542 N->getOpcode() == ISD::PREFETCH ||
1543 N->isTargetMemoryOpcode();
1544 }
1545};
1546
1547/// This SDNode is used to implement the code generator
1548/// support for the llvm IR shufflevector instruction. It combines elements
1549/// from two input vectors into a new input vector, with the selection and
1550/// ordering of elements determined by an array of integers, referred to as
1551/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1552/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1553/// An index of -1 is treated as undef, such that the code generator may put
1554/// any value in the corresponding element of the result.
1555class ShuffleVectorSDNode : public SDNode {
1556 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1557 // is freed when the SelectionDAG object is destroyed.
1558 const int *Mask;
1559
1560protected:
1561 friend class SelectionDAG;
1562
1563 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1564 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1565
1566public:
1567 ArrayRef<int> getMask() const {
1568 EVT VT = getValueType(ResNo: 0);
1569 return ArrayRef(Mask, VT.getVectorNumElements());
1570 }
1571
1572 int getMaskElt(unsigned Idx) const {
1573 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!");
1574 return Mask[Idx];
1575 }
1576
1577 bool isSplat() const { return isSplatMask(Mask, VT: getValueType(ResNo: 0)); }
1578
1579 int getSplatIndex() const {
1580 assert(isSplat() && "Cannot get splat index for non-splat!");
1581 EVT VT = getValueType(ResNo: 0);
1582 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1583 if (Mask[i] >= 0)
1584 return Mask[i];
1585
1586 // We can choose any index value here and be correct because all elements
1587 // are undefined. Return 0 for better potential for callers to simplify.
1588 return 0;
1589 }
1590
1591 static bool isSplatMask(const int *Mask, EVT VT);
1592
1593 /// Change values in a shuffle permute mask assuming
1594 /// the two vector operands have swapped position.
1595 static void commuteMask(MutableArrayRef<int> Mask) {
1596 unsigned NumElems = Mask.size();
1597 for (unsigned i = 0; i != NumElems; ++i) {
1598 int idx = Mask[i];
1599 if (idx < 0)
1600 continue;
1601 else if (idx < (int)NumElems)
1602 Mask[i] = idx + NumElems;
1603 else
1604 Mask[i] = idx - NumElems;
1605 }
1606 }
1607
1608 static bool classof(const SDNode *N) {
1609 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1610 }
1611};
1612
1613class ConstantSDNode : public SDNode {
1614 friend class SelectionDAG;
1615
1616 const ConstantInt *Value;
1617
1618 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1619 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1620 getSDVTList(VT)),
1621 Value(val) {
1622 ConstantSDNodeBits.IsOpaque = isOpaque;
1623 }
1624
1625public:
1626 const ConstantInt *getConstantIntValue() const { return Value; }
1627 const APInt &getAPIntValue() const { return Value->getValue(); }
1628 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1629 int64_t getSExtValue() const { return Value->getSExtValue(); }
1630 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) {
1631 return Value->getLimitedValue(Limit);
1632 }
1633 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1634 Align getAlignValue() const { return Value->getAlignValue(); }
1635
1636 bool isOne() const { return Value->isOne(); }
1637 bool isZero() const { return Value->isZero(); }
1638 bool isAllOnes() const { return Value->isMinusOne(); }
1639 bool isMaxSignedValue() const { return Value->isMaxValue(IsSigned: true); }
1640 bool isMinSignedValue() const { return Value->isMinValue(IsSigned: true); }
1641
1642 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1643
1644 static bool classof(const SDNode *N) {
1645 return N->getOpcode() == ISD::Constant ||
1646 N->getOpcode() == ISD::TargetConstant;
1647 }
1648};
1649
1650uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1651 return cast<ConstantSDNode>(Val: getOperand(Num))->getZExtValue();
1652}
1653
1654uint64_t SDNode::getAsZExtVal() const {
1655 return cast<ConstantSDNode>(Val: this)->getZExtValue();
1656}
1657
1658const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1659 return cast<ConstantSDNode>(Val: getOperand(Num))->getAPIntValue();
1660}
1661
1662const APInt &SDNode::getAsAPIntVal() const {
1663 return cast<ConstantSDNode>(Val: this)->getAPIntValue();
1664}
1665
1666class ConstantFPSDNode : public SDNode {
1667 friend class SelectionDAG;
1668
1669 const ConstantFP *Value;
1670
1671 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1672 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1673 DebugLoc(), getSDVTList(VT)),
1674 Value(val) {}
1675
1676public:
1677 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1678 const ConstantFP *getConstantFPValue() const { return Value; }
1679
1680 /// Return true if the value is positive or negative zero.
1681 bool isZero() const { return Value->isZero(); }
1682
1683 /// Return true if the value is a NaN.
1684 bool isNaN() const { return Value->isNaN(); }
1685
1686 /// Return true if the value is an infinity
1687 bool isInfinity() const { return Value->isInfinity(); }
1688
1689 /// Return true if the value is negative.
1690 bool isNegative() const { return Value->isNegative(); }
1691
1692 /// We don't rely on operator== working on double values, as
1693 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1694 /// As such, this method can be used to do an exact bit-for-bit comparison of
1695 /// two floating point values.
1696
1697 /// We leave the version with the double argument here because it's just so
1698 /// convenient to write "2.0" and the like. Without this function we'd
1699 /// have to duplicate its logic everywhere it's called.
1700 bool isExactlyValue(double V) const {
1701 return Value->getValueAPF().isExactlyValue(V);
1702 }
1703 bool isExactlyValue(const APFloat& V) const;
1704
1705 static bool isValueValidForType(EVT VT, const APFloat& Val);
1706
1707 static bool classof(const SDNode *N) {
1708 return N->getOpcode() == ISD::ConstantFP ||
1709 N->getOpcode() == ISD::TargetConstantFP;
1710 }
1711};
1712
1713/// Returns true if \p V is a constant integer zero.
1714bool isNullConstant(SDValue V);
1715
1716/// Returns true if \p V is an FP constant with a value of positive zero.
1717bool isNullFPConstant(SDValue V);
1718
1719/// Returns true if \p V is an integer constant with all bits set.
1720bool isAllOnesConstant(SDValue V);
1721
1722/// Returns true if \p V is a constant integer one.
1723bool isOneConstant(SDValue V);
1724
1725/// Returns true if \p V is a constant min signed integer value.
1726bool isMinSignedConstant(SDValue V);
1727
1728/// Returns true if \p V is a neutral element of Opc with Flags.
1729/// When OperandNo is 0, it checks that V is a left identity. Otherwise, it
1730/// checks that V is a right identity.
1731bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V,
1732 unsigned OperandNo);
1733
1734/// Return the non-bitcasted source operand of \p V if it exists.
1735/// If \p V is not a bitcasted value, it is returned as-is.
1736SDValue peekThroughBitcasts(SDValue V);
1737
1738/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1739/// If \p V is not a bitcasted one-use value, it is returned as-is.
1740SDValue peekThroughOneUseBitcasts(SDValue V);
1741
1742/// Return the non-extracted vector source operand of \p V if it exists.
1743/// If \p V is not an extracted subvector, it is returned as-is.
1744SDValue peekThroughExtractSubvectors(SDValue V);
1745
1746/// Return the non-truncated source operand of \p V if it exists.
1747/// If \p V is not a truncation, it is returned as-is.
1748SDValue peekThroughTruncates(SDValue V);
1749
1750/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1751/// constant is canonicalized to be operand 1.
1752bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1753
1754/// If \p V is a bitwise not, returns the inverted operand. Otherwise returns
1755/// an empty SDValue. Only bits set in \p Mask are required to be inverted,
1756/// other bits may be arbitrary.
1757SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs);
1758
1759/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1760ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1761 bool AllowTruncation = false);
1762
1763/// Returns the SDNode if it is a demanded constant splat BuildVector or
1764/// constant int.
1765ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1766 bool AllowUndefs = false,
1767 bool AllowTruncation = false);
1768
1769/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1770ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1771
1772/// Returns the SDNode if it is a demanded constant splat BuildVector or
1773/// constant float.
1774ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1775 bool AllowUndefs = false);
1776
1777/// Return true if the value is a constant 0 integer or a splatted vector of
1778/// a constant 0 integer (with no undefs by default).
1779/// Build vector implicit truncation is not an issue for null values.
1780bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1781
1782/// Return true if the value is a constant 1 integer or a splatted vector of a
1783/// constant 1 integer (with no undefs).
1784/// Build vector implicit truncation is allowed, but the truncated bits need to
1785/// be zero.
1786bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1787
1788/// Return true if the value is a constant -1 integer or a splatted vector of a
1789/// constant -1 integer (with no undefs).
1790/// Does not permit build vector implicit truncation.
1791bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1792
1793/// Return true if \p V is either a integer or FP constant.
1794inline bool isIntOrFPConstant(SDValue V) {
1795 return isa<ConstantSDNode>(Val: V) || isa<ConstantFPSDNode>(Val: V);
1796}
1797
1798class GlobalAddressSDNode : public SDNode {
1799 friend class SelectionDAG;
1800
1801 const GlobalValue *TheGlobal;
1802 int64_t Offset;
1803 unsigned TargetFlags;
1804
1805 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1806 const GlobalValue *GA, EVT VT, int64_t o,
1807 unsigned TF);
1808
1809public:
1810 const GlobalValue *getGlobal() const { return TheGlobal; }
1811 int64_t getOffset() const { return Offset; }
1812 unsigned getTargetFlags() const { return TargetFlags; }
1813 // Return the address space this GlobalAddress belongs to.
1814 unsigned getAddressSpace() const;
1815
1816 static bool classof(const SDNode *N) {
1817 return N->getOpcode() == ISD::GlobalAddress ||
1818 N->getOpcode() == ISD::TargetGlobalAddress ||
1819 N->getOpcode() == ISD::GlobalTLSAddress ||
1820 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1821 }
1822};
1823
1824class FrameIndexSDNode : public SDNode {
1825 friend class SelectionDAG;
1826
1827 int FI;
1828
1829 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1830 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1831 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1832 }
1833
1834public:
1835 int getIndex() const { return FI; }
1836
1837 static bool classof(const SDNode *N) {
1838 return N->getOpcode() == ISD::FrameIndex ||
1839 N->getOpcode() == ISD::TargetFrameIndex;
1840 }
1841};
1842
1843/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1844/// the offet and size that are started/ended in the underlying FrameIndex.
1845class LifetimeSDNode : public SDNode {
1846 friend class SelectionDAG;
1847 int64_t Size;
1848 int64_t Offset; // -1 if offset is unknown.
1849
1850 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1851 SDVTList VTs, int64_t Size, int64_t Offset)
1852 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1853public:
1854 int64_t getFrameIndex() const {
1855 return cast<FrameIndexSDNode>(Val: getOperand(Num: 1))->getIndex();
1856 }
1857
1858 bool hasOffset() const { return Offset >= 0; }
1859 int64_t getOffset() const {
1860 assert(hasOffset() && "offset is unknown");
1861 return Offset;
1862 }
1863 int64_t getSize() const {
1864 assert(hasOffset() && "offset is unknown");
1865 return Size;
1866 }
1867
1868 // Methods to support isa and dyn_cast
1869 static bool classof(const SDNode *N) {
1870 return N->getOpcode() == ISD::LIFETIME_START ||
1871 N->getOpcode() == ISD::LIFETIME_END;
1872 }
1873};
1874
1875/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1876/// the index of the basic block being probed. A pseudo probe serves as a place
1877/// holder and will be removed at the end of compilation. It does not have any
1878/// operand because we do not want the instruction selection to deal with any.
1879class PseudoProbeSDNode : public SDNode {
1880 friend class SelectionDAG;
1881 uint64_t Guid;
1882 uint64_t Index;
1883 uint32_t Attributes;
1884
1885 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1886 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1887 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1888 Attributes(Attr) {}
1889
1890public:
1891 uint64_t getGuid() const { return Guid; }
1892 uint64_t getIndex() const { return Index; }
1893 uint32_t getAttributes() const { return Attributes; }
1894
1895 // Methods to support isa and dyn_cast
1896 static bool classof(const SDNode *N) {
1897 return N->getOpcode() == ISD::PSEUDO_PROBE;
1898 }
1899};
1900
1901class JumpTableSDNode : public SDNode {
1902 friend class SelectionDAG;
1903
1904 int JTI;
1905 unsigned TargetFlags;
1906
1907 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1908 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1909 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1910 }
1911
1912public:
1913 int getIndex() const { return JTI; }
1914 unsigned getTargetFlags() const { return TargetFlags; }
1915
1916 static bool classof(const SDNode *N) {
1917 return N->getOpcode() == ISD::JumpTable ||
1918 N->getOpcode() == ISD::TargetJumpTable;
1919 }
1920};
1921
1922class ConstantPoolSDNode : public SDNode {
1923 friend class SelectionDAG;
1924
1925 union {
1926 const Constant *ConstVal;
1927 MachineConstantPoolValue *MachineCPVal;
1928 } Val;
1929 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1930 Align Alignment; // Minimum alignment requirement of CP.
1931 unsigned TargetFlags;
1932
1933 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1934 Align Alignment, unsigned TF)
1935 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1936 DebugLoc(), getSDVTList(VT)),
1937 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1938 assert(Offset >= 0 && "Offset is too large");
1939 Val.ConstVal = c;
1940 }
1941
1942 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1943 Align Alignment, unsigned TF)
1944 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1945 DebugLoc(), getSDVTList(VT)),
1946 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1947 assert(Offset >= 0 && "Offset is too large");
1948 Val.MachineCPVal = v;
1949 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1);
1950 }
1951
1952public:
1953 bool isMachineConstantPoolEntry() const {
1954 return Offset < 0;
1955 }
1956
1957 const Constant *getConstVal() const {
1958 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type");
1959 return Val.ConstVal;
1960 }
1961
1962 MachineConstantPoolValue *getMachineCPVal() const {
1963 assert(isMachineConstantPoolEntry() && "Wrong constantpool type");
1964 return Val.MachineCPVal;
1965 }
1966
1967 int getOffset() const {
1968 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT-1));
1969 }
1970
1971 // Return the alignment of this constant pool object, which is either 0 (for
1972 // default alignment) or the desired value.
1973 Align getAlign() const { return Alignment; }
1974 unsigned getTargetFlags() const { return TargetFlags; }
1975
1976 Type *getType() const;
1977
1978 static bool classof(const SDNode *N) {
1979 return N->getOpcode() == ISD::ConstantPool ||
1980 N->getOpcode() == ISD::TargetConstantPool;
1981 }
1982};
1983
1984/// Completely target-dependent object reference.
1985class TargetIndexSDNode : public SDNode {
1986 friend class SelectionDAG;
1987
1988 unsigned TargetFlags;
1989 int Index;
1990 int64_t Offset;
1991
1992public:
1993 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1994 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1995 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1996
1997 unsigned getTargetFlags() const { return TargetFlags; }
1998 int getIndex() const { return Index; }
1999 int64_t getOffset() const { return Offset; }
2000
2001 static bool classof(const SDNode *N) {
2002 return N->getOpcode() == ISD::TargetIndex;
2003 }
2004};
2005
2006class BasicBlockSDNode : public SDNode {
2007 friend class SelectionDAG;
2008
2009 MachineBasicBlock *MBB;
2010
2011 /// Debug info is meaningful and potentially useful here, but we create
2012 /// blocks out of order when they're jumped to, which makes it a bit
2013 /// harder. Let's see if we need it first.
2014 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
2015 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
2016 {}
2017
2018public:
2019 MachineBasicBlock *getBasicBlock() const { return MBB; }
2020
2021 static bool classof(const SDNode *N) {
2022 return N->getOpcode() == ISD::BasicBlock;
2023 }
2024};
2025
2026/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
2027class BuildVectorSDNode : public SDNode {
2028public:
2029 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
2030 explicit BuildVectorSDNode() = delete;
2031
2032 /// Check if this is a constant splat, and if so, find the
2033 /// smallest element size that splats the vector. If MinSplatBits is
2034 /// nonzero, the element size must be at least that large. Note that the
2035 /// splat element may be the entire vector (i.e., a one element vector).
2036 /// Returns the splat element value in SplatValue. Any undefined bits in
2037 /// that value are zero, and the corresponding bits in the SplatUndef mask
2038 /// are set. The SplatBitSize value is set to the splat element size in
2039 /// bits. HasAnyUndefs is set to true if any bits in the vector are
2040 /// undefined. isBigEndian describes the endianness of the target.
2041 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
2042 unsigned &SplatBitSize, bool &HasAnyUndefs,
2043 unsigned MinSplatBits = 0,
2044 bool isBigEndian = false) const;
2045
2046 /// Returns the demanded splatted value or a null value if this is not a
2047 /// splat.
2048 ///
2049 /// The DemandedElts mask indicates the elements that must be in the splat.
2050 /// If passed a non-null UndefElements bitvector, it will resize it to match
2051 /// the vector width and set the bits where elements are undef.
2052 SDValue getSplatValue(const APInt &DemandedElts,
2053 BitVector *UndefElements = nullptr) const;
2054
2055 /// Returns the splatted value or a null value if this is not a splat.
2056 ///
2057 /// If passed a non-null UndefElements bitvector, it will resize it to match
2058 /// the vector width and set the bits where elements are undef.
2059 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
2060
2061 /// Find the shortest repeating sequence of values in the build vector.
2062 ///
2063 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
2064 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
2065 ///
2066 /// Currently this must be a power-of-2 build vector.
2067 /// The DemandedElts mask indicates the elements that must be present,
2068 /// undemanded elements in Sequence may be null (SDValue()). If passed a
2069 /// non-null UndefElements bitvector, it will resize it to match the original
2070 /// vector width and set the bits where elements are undef. If result is
2071 /// false, Sequence will be empty.
2072 bool getRepeatedSequence(const APInt &DemandedElts,
2073 SmallVectorImpl<SDValue> &Sequence,
2074 BitVector *UndefElements = nullptr) const;
2075
2076 /// Find the shortest repeating sequence of values in the build vector.
2077 ///
2078 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
2079 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
2080 ///
2081 /// Currently this must be a power-of-2 build vector.
2082 /// If passed a non-null UndefElements bitvector, it will resize it to match
2083 /// the original vector width and set the bits where elements are undef.
2084 /// If result is false, Sequence will be empty.
2085 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
2086 BitVector *UndefElements = nullptr) const;
2087
2088 /// Returns the demanded splatted constant or null if this is not a constant
2089 /// splat.
2090 ///
2091 /// The DemandedElts mask indicates the elements that must be in the splat.
2092 /// If passed a non-null UndefElements bitvector, it will resize it to match
2093 /// the vector width and set the bits where elements are undef.
2094 ConstantSDNode *
2095 getConstantSplatNode(const APInt &DemandedElts,
2096 BitVector *UndefElements = nullptr) const;
2097
2098 /// Returns the splatted constant or null if this is not a constant
2099 /// splat.
2100 ///
2101 /// If passed a non-null UndefElements bitvector, it will resize it to match
2102 /// the vector width and set the bits where elements are undef.
2103 ConstantSDNode *
2104 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2105
2106 /// Returns the demanded splatted constant FP or null if this is not a
2107 /// constant FP splat.
2108 ///
2109 /// The DemandedElts mask indicates the elements that must be in the splat.
2110 /// If passed a non-null UndefElements bitvector, it will resize it to match
2111 /// the vector width and set the bits where elements are undef.
2112 ConstantFPSDNode *
2113 getConstantFPSplatNode(const APInt &DemandedElts,
2114 BitVector *UndefElements = nullptr) const;
2115
2116 /// Returns the splatted constant FP or null if this is not a constant
2117 /// FP splat.
2118 ///
2119 /// If passed a non-null UndefElements bitvector, it will resize it to match
2120 /// the vector width and set the bits where elements are undef.
2121 ConstantFPSDNode *
2122 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2123
2124 /// If this is a constant FP splat and the splatted constant FP is an
2125 /// exact power or 2, return the log base 2 integer value. Otherwise,
2126 /// return -1.
2127 ///
2128 /// The BitWidth specifies the necessary bit precision.
2129 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2130 uint32_t BitWidth) const;
2131
2132 /// Extract the raw bit data from a build vector of Undef, Constant or
2133 /// ConstantFP node elements. Each raw bit element will be \p
2134 /// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
2135 /// undefined elements are flagged in \p UndefElements.
2136 bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
2137 SmallVectorImpl<APInt> &RawBitElements,
2138 BitVector &UndefElements) const;
2139
2140 bool isConstant() const;
2141
2142 /// If this BuildVector is constant and represents the numerical series
2143 /// "<a, a+n, a+2n, a+3n, ...>" where a is integer and n is a non-zero integer,
2144 /// the value "<a,n>" is returned.
2145 std::optional<std::pair<APInt, APInt>> isConstantSequence() const;
2146
2147 /// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements.
2148 /// Undef elements are treated as zero, and entirely undefined elements are
2149 /// flagged in \p DstUndefElements.
2150 static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
2151 SmallVectorImpl<APInt> &DstBitElements,
2152 ArrayRef<APInt> SrcBitElements,
2153 BitVector &DstUndefElements,
2154 const BitVector &SrcUndefElements);
2155
2156 static bool classof(const SDNode *N) {
2157 return N->getOpcode() == ISD::BUILD_VECTOR;
2158 }
2159};
2160
2161/// An SDNode that holds an arbitrary LLVM IR Value. This is
2162/// used when the SelectionDAG needs to make a simple reference to something
2163/// in the LLVM IR representation.
2164///
2165class SrcValueSDNode : public SDNode {
2166 friend class SelectionDAG;
2167
2168 const Value *V;
2169
2170 /// Create a SrcValue for a general value.
2171 explicit SrcValueSDNode(const Value *v)
2172 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2173
2174public:
2175 /// Return the contained Value.
2176 const Value *getValue() const { return V; }
2177
2178 static bool classof(const SDNode *N) {
2179 return N->getOpcode() == ISD::SRCVALUE;
2180 }
2181};
2182
2183class MDNodeSDNode : public SDNode {
2184 friend class SelectionDAG;
2185
2186 const MDNode *MD;
2187
2188 explicit MDNodeSDNode(const MDNode *md)
2189 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2190 {}
2191
2192public:
2193 const MDNode *getMD() const { return MD; }
2194
2195 static bool classof(const SDNode *N) {
2196 return N->getOpcode() == ISD::MDNODE_SDNODE;
2197 }
2198};
2199
2200class RegisterSDNode : public SDNode {
2201 friend class SelectionDAG;
2202
2203 Register Reg;
2204
2205 RegisterSDNode(Register reg, EVT VT)
2206 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2207
2208public:
2209 Register getReg() const { return Reg; }
2210
2211 static bool classof(const SDNode *N) {
2212 return N->getOpcode() == ISD::Register;
2213 }
2214};
2215
2216class RegisterMaskSDNode : public SDNode {
2217 friend class SelectionDAG;
2218
2219 // The memory for RegMask is not owned by the node.
2220 const uint32_t *RegMask;
2221
2222 RegisterMaskSDNode(const uint32_t *mask)
2223 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2224 RegMask(mask) {}
2225
2226public:
2227 const uint32_t *getRegMask() const { return RegMask; }
2228
2229 static bool classof(const SDNode *N) {
2230 return N->getOpcode() == ISD::RegisterMask;
2231 }
2232};
2233
2234class BlockAddressSDNode : public SDNode {
2235 friend class SelectionDAG;
2236
2237 const BlockAddress *BA;
2238 int64_t Offset;
2239 unsigned TargetFlags;
2240
2241 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2242 int64_t o, unsigned Flags)
2243 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2244 BA(ba), Offset(o), TargetFlags(Flags) {}
2245
2246public:
2247 const BlockAddress *getBlockAddress() const { return BA; }
2248 int64_t getOffset() const { return Offset; }
2249 unsigned getTargetFlags() const { return TargetFlags; }
2250
2251 static bool classof(const SDNode *N) {
2252 return N->getOpcode() == ISD::BlockAddress ||
2253 N->getOpcode() == ISD::TargetBlockAddress;
2254 }
2255};
2256
2257class LabelSDNode : public SDNode {
2258 friend class SelectionDAG;
2259
2260 MCSymbol *Label;
2261
2262 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2263 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2264 assert(LabelSDNode::classof(this) && "not a label opcode");
2265 }
2266
2267public:
2268 MCSymbol *getLabel() const { return Label; }
2269
2270 static bool classof(const SDNode *N) {
2271 return N->getOpcode() == ISD::EH_LABEL ||
2272 N->getOpcode() == ISD::ANNOTATION_LABEL;
2273 }
2274};
2275
2276class ExternalSymbolSDNode : public SDNode {
2277 friend class SelectionDAG;
2278
2279 const char *Symbol;
2280 unsigned TargetFlags;
2281
2282 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2283 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2284 DebugLoc(), getSDVTList(VT)),
2285 Symbol(Sym), TargetFlags(TF) {}
2286
2287public:
2288 const char *getSymbol() const { return Symbol; }
2289 unsigned getTargetFlags() const { return TargetFlags; }
2290
2291 static bool classof(const SDNode *N) {
2292 return N->getOpcode() == ISD::ExternalSymbol ||
2293 N->getOpcode() == ISD::TargetExternalSymbol;
2294 }
2295};
2296
2297class MCSymbolSDNode : public SDNode {
2298 friend class SelectionDAG;
2299
2300 MCSymbol *Symbol;
2301
2302 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2303 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2304
2305public:
2306 MCSymbol *getMCSymbol() const { return Symbol; }
2307
2308 static bool classof(const SDNode *N) {
2309 return N->getOpcode() == ISD::MCSymbol;
2310 }
2311};
2312
2313class CondCodeSDNode : public SDNode {
2314 friend class SelectionDAG;
2315
2316 ISD::CondCode Condition;
2317
2318 explicit CondCodeSDNode(ISD::CondCode Cond)
2319 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2320 Condition(Cond) {}
2321
2322public:
2323 ISD::CondCode get() const { return Condition; }
2324
2325 static bool classof(const SDNode *N) {
2326 return N->getOpcode() == ISD::CONDCODE;
2327 }
2328};
2329
2330/// This class is used to represent EVT's, which are used
2331/// to parameterize some operations.
2332class VTSDNode : public SDNode {
2333 friend class SelectionDAG;
2334
2335 EVT ValueType;
2336
2337 explicit VTSDNode(EVT VT)
2338 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2339 ValueType(VT) {}
2340
2341public:
2342 EVT getVT() const { return ValueType; }
2343
2344 static bool classof(const SDNode *N) {
2345 return N->getOpcode() == ISD::VALUETYPE;
2346 }
2347};
2348
2349/// Base class for LoadSDNode and StoreSDNode
2350class LSBaseSDNode : public MemSDNode {
2351public:
2352 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2353 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2354 MachineMemOperand *MMO)
2355 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2356 LSBaseSDNodeBits.AddressingMode = AM;
2357 assert(getAddressingMode() == AM && "Value truncated");
2358 }
2359
2360 const SDValue &getOffset() const {
2361 return getOperand(Num: getOpcode() == ISD::LOAD ? 2 : 3);
2362 }
2363
2364 /// Return the addressing mode for this load or store:
2365 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2366 ISD::MemIndexedMode getAddressingMode() const {
2367 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2368 }
2369
2370 /// Return true if this is a pre/post inc/dec load/store.
2371 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2372
2373 /// Return true if this is NOT a pre/post inc/dec load/store.
2374 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2375
2376 static bool classof(const SDNode *N) {
2377 return N->getOpcode() == ISD::LOAD ||
2378 N->getOpcode() == ISD::STORE;
2379 }
2380};
2381
2382/// This class is used to represent ISD::LOAD nodes.
2383class LoadSDNode : public LSBaseSDNode {
2384 friend class SelectionDAG;
2385
2386 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2387 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2388 MachineMemOperand *MMO)
2389 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2390 LoadSDNodeBits.ExtTy = ETy;
2391 assert(readMem() && "Load MachineMemOperand is not a load!");
2392 assert(!writeMem() && "Load MachineMemOperand is a store!");
2393 }
2394
2395public:
2396 /// Return whether this is a plain node,
2397 /// or one of the varieties of value-extending loads.
2398 ISD::LoadExtType getExtensionType() const {
2399 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2400 }
2401
2402 const SDValue &getBasePtr() const { return getOperand(Num: 1); }
2403 const SDValue &getOffset() const { return getOperand(Num: 2); }
2404
2405 static bool classof(const SDNode *N) {
2406 return N->getOpcode() == ISD::LOAD;
2407 }
2408};
2409
2410/// This class is used to represent ISD::STORE nodes.
2411class StoreSDNode : public LSBaseSDNode {
2412 friend class SelectionDAG;
2413
2414 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2415 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2416 MachineMemOperand *MMO)
2417 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2418 StoreSDNodeBits.IsTruncating = isTrunc;
2419 assert(!readMem() && "Store MachineMemOperand is a load!");
2420 assert(writeMem() && "Store MachineMemOperand is not a store!");
2421 }
2422
2423public:
2424 /// Return true if the op does a truncation before store.
2425 /// For integers this is the same as doing a TRUNCATE and storing the result.
2426 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2427 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2428 void setTruncatingStore(bool Truncating) {
2429 StoreSDNodeBits.IsTruncating = Truncating;
2430 }
2431
2432 const SDValue &getValue() const { return getOperand(Num: 1); }
2433 const SDValue &getBasePtr() const { return getOperand(Num: 2); }
2434 const SDValue &getOffset() const { return getOperand(Num: 3); }
2435
2436 static bool classof(const SDNode *N) {
2437 return N->getOpcode() == ISD::STORE;
2438 }
2439};
2440
2441/// This base class is used to represent VP_LOAD, VP_STORE,
2442/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes
2443class VPBaseLoadStoreSDNode : public MemSDNode {
2444public:
2445 friend class SelectionDAG;
2446
2447 VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2448 const DebugLoc &DL, SDVTList VTs,
2449 ISD::MemIndexedMode AM, EVT MemVT,
2450 MachineMemOperand *MMO)
2451 : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) {
2452 LSBaseSDNodeBits.AddressingMode = AM;
2453 assert(getAddressingMode() == AM && "Value truncated");
2454 }
2455
2456 // VPStridedStoreSDNode (Chain, Data, Ptr, Offset, Stride, Mask, EVL)
2457 // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
2458 // VPStridedLoadSDNode (Chain, Ptr, Offset, Stride, Mask, EVL)
2459 // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
2460 // Mask is a vector of i1 elements;
2461 // the type of EVL is TLI.getVPExplicitVectorLengthTy().
2462 const SDValue &getOffset() const {
2463 return getOperand(Num: (getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
2464 getOpcode() == ISD::VP_LOAD)
2465 ? 2
2466 : 3);
2467 }
2468 const SDValue &getBasePtr() const {
2469 return getOperand(Num: (getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
2470 getOpcode() == ISD::VP_LOAD)
2471 ? 1
2472 : 2);
2473 }
2474 const SDValue &getMask() const {
2475 switch (getOpcode()) {
2476 default:
2477 llvm_unreachable("Invalid opcode");
2478 case ISD::VP_LOAD:
2479 return getOperand(Num: 3);
2480 case ISD::VP_STORE:
2481 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2482 return getOperand(Num: 4);
2483 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2484 return getOperand(Num: 5);
2485 }
2486 }
2487 const SDValue &getVectorLength() const {
2488 switch (getOpcode()) {
2489 default:
2490 llvm_unreachable("Invalid opcode");
2491 case ISD::VP_LOAD:
2492 return getOperand(Num: 4);
2493 case ISD::VP_STORE:
2494 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2495 return getOperand(Num: 5);
2496 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2497 return getOperand(Num: 6);
2498 }
2499 }
2500
2501 /// Return the addressing mode for this load or store:
2502 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2503 ISD::MemIndexedMode getAddressingMode() const {
2504 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2505 }
2506
2507 /// Return true if this is a pre/post inc/dec load/store.
2508 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2509
2510 /// Return true if this is NOT a pre/post inc/dec load/store.
2511 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2512
2513 static bool classof(const SDNode *N) {
2514 return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
2515 N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE ||
2516 N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
2517 }
2518};
2519
2520/// This class is used to represent a VP_LOAD node
2521class VPLoadSDNode : public VPBaseLoadStoreSDNode {
2522public:
2523 friend class SelectionDAG;
2524
2525 VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2526 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
2527 EVT MemVT, MachineMemOperand *MMO)
2528 : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2529 LoadSDNodeBits.ExtTy = ETy;
2530 LoadSDNodeBits.IsExpanding = isExpanding;
2531 }
2532
2533 ISD::LoadExtType getExtensionType() const {
2534 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2535 }
2536
2537 const SDValue &getBasePtr() const { return getOperand(Num: 1); }
2538 const SDValue &getOffset() const { return getOperand(Num: 2); }
2539 const SDValue &getMask() const { return getOperand(Num: 3); }
2540 const SDValue &getVectorLength() const { return getOperand(Num: 4); }
2541
2542 static bool classof(const SDNode *N) {
2543 return N->getOpcode() == ISD::VP_LOAD;
2544 }
2545 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2546};
2547
2548/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
2549class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode {
2550public:
2551 friend class SelectionDAG;
2552
2553 VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
2554 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2555 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2556 : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs,
2557 AM, MemVT, MMO) {
2558 LoadSDNodeBits.ExtTy = ETy;
2559 LoadSDNodeBits.IsExpanding = IsExpanding;
2560 }
2561
2562 ISD::LoadExtType getExtensionType() const {
2563 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2564 }
2565
2566 const SDValue &getBasePtr() const { return getOperand(Num: 1); }
2567 const SDValue &getOffset() const { return getOperand(Num: 2); }
2568 const SDValue &getStride() const { return getOperand(Num: 3); }
2569 const SDValue &getMask() const { return getOperand(Num: 4); }
2570 const SDValue &getVectorLength() const { return getOperand(Num: 5); }
2571
2572 static bool classof(const SDNode *N) {
2573 return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD;
2574 }
2575 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2576};
2577
2578/// This class is used to represent a VP_STORE node
2579class VPStoreSDNode : public VPBaseLoadStoreSDNode {
2580public:
2581 friend class SelectionDAG;
2582
2583 VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2584 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2585 EVT MemVT, MachineMemOperand *MMO)
2586 : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
2587 StoreSDNodeBits.IsTruncating = isTrunc;
2588 StoreSDNodeBits.IsCompressing = isCompressing;
2589 }
2590
2591 /// Return true if this is a truncating store.
2592 /// For integers this is the same as doing a TRUNCATE and storing the result.
2593 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2594 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2595
2596 /// Returns true if the op does a compression to the vector before storing.
2597 /// The node contiguously stores the active elements (integers or floats)
2598 /// in src (those with their respective bit set in writemask k) to unaligned
2599 /// memory at base_addr.
2600 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2601
2602 const SDValue &getValue() const { return getOperand(Num: 1); }
2603 const SDValue &getBasePtr() const { return getOperand(Num: 2); }
2604 const SDValue &getOffset() const { return getOperand(Num: 3); }
2605 const SDValue &getMask() const { return getOperand(Num: 4); }
2606 const SDValue &getVectorLength() const { return getOperand(Num: 5); }
2607
2608 static bool classof(const SDNode *N) {
2609 return N->getOpcode() == ISD::VP_STORE;
2610 }
2611};
2612
2613/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
2614class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode {
2615public:
2616 friend class SelectionDAG;
2617
2618 VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
2619 ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing,
2620 EVT MemVT, MachineMemOperand *MMO)
2621 : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL,
2622 VTs, AM, MemVT, MMO) {
2623 StoreSDNodeBits.IsTruncating = IsTrunc;
2624 StoreSDNodeBits.IsCompressing = IsCompressing;
2625 }
2626
2627 /// Return true if this is a truncating store.
2628 /// For integers this is the same as doing a TRUNCATE and storing the result.
2629 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2630 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2631
2632 /// Returns true if the op does a compression to the vector before storing.
2633 /// The node contiguously stores the active elements (integers or floats)
2634 /// in src (those with their respective bit set in writemask k) to unaligned
2635 /// memory at base_addr.
2636 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2637
2638 const SDValue &getValue() const { return getOperand(Num: 1); }
2639 const SDValue &getBasePtr() const { return getOperand(Num: 2); }
2640 const SDValue &getOffset() const { return getOperand(Num: 3); }
2641 const SDValue &getStride() const { return getOperand(Num: 4); }
2642 const SDValue &getMask() const { return getOperand(Num: 5); }
2643 const SDValue &getVectorLength() const { return getOperand(Num: 6); }
2644
2645 static bool classof(const SDNode *N) {
2646 return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE;
2647 }
2648};
2649
2650/// This base class is used to represent MLOAD and MSTORE nodes
2651class MaskedLoadStoreSDNode : public MemSDNode {
2652public:
2653 friend class SelectionDAG;
2654
2655 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2656 const DebugLoc &dl, SDVTList VTs,
2657 ISD::MemIndexedMode AM, EVT MemVT,
2658 MachineMemOperand *MMO)
2659 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2660 LSBaseSDNodeBits.AddressingMode = AM;
2661 assert(getAddressingMode() == AM && "Value truncated");
2662 }
2663
2664 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2665 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2666 // Mask is a vector of i1 elements
2667 const SDValue &getOffset() const {
2668 return getOperand(Num: getOpcode() == ISD::MLOAD ? 2 : 3);
2669 }
2670 const SDValue &getMask() const {
2671 return getOperand(Num: getOpcode() == ISD::MLOAD ? 3 : 4);
2672 }
2673
2674 /// Return the addressing mode for this load or store:
2675 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2676 ISD::MemIndexedMode getAddressingMode() const {
2677 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2678 }
2679
2680 /// Return true if this is a pre/post inc/dec load/store.
2681 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2682
2683 /// Return true if this is NOT a pre/post inc/dec load/store.
2684 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2685
2686 static bool classof(const SDNode *N) {
2687 return N->getOpcode() == ISD::MLOAD ||
2688 N->getOpcode() == ISD::MSTORE;
2689 }
2690};
2691
2692/// This class is used to represent an MLOAD node
2693class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2694public:
2695 friend class SelectionDAG;
2696
2697 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2698 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2699 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2700 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2701 LoadSDNodeBits.ExtTy = ETy;
2702 LoadSDNodeBits.IsExpanding = IsExpanding;
2703 }
2704
2705 ISD::LoadExtType getExtensionType() const {
2706 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2707 }
2708
2709 const SDValue &getBasePtr() const { return getOperand(Num: 1); }
2710 const SDValue &getOffset() const { return getOperand(Num: 2); }
2711 const SDValue &getMask() const { return getOperand(Num: 3); }
2712 const SDValue &getPassThru() const { return getOperand(Num: 4); }
2713
2714 static bool classof(const SDNode *N) {
2715 return N->getOpcode() == ISD::MLOAD;
2716 }
2717
2718 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2719};
2720
2721/// This class is used to represent an MSTORE node
2722class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2723public:
2724 friend class SelectionDAG;
2725
2726 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2727 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2728 EVT MemVT, MachineMemOperand *MMO)
2729 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2730 StoreSDNodeBits.IsTruncating = isTrunc;
2731 StoreSDNodeBits.IsCompressing = isCompressing;
2732 }
2733
2734 /// Return true if the op does a truncation before store.
2735 /// For integers this is the same as doing a TRUNCATE and storing the result.
2736 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2737 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2738
2739 /// Returns true if the op does a compression to the vector before storing.
2740 /// The node contiguously stores the active elements (integers or floats)
2741 /// in src (those with their respective bit set in writemask k) to unaligned
2742 /// memory at base_addr.
2743 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2744
2745 const SDValue &getValue() const { return getOperand(Num: 1); }
2746 const SDValue &getBasePtr() const { return getOperand(Num: 2); }
2747 const SDValue &getOffset() const { return getOperand(Num: 3); }
2748 const SDValue &getMask() const { return getOperand(Num: 4); }
2749
2750 static bool classof(const SDNode *N) {
2751 return N->getOpcode() == ISD::MSTORE;
2752 }
2753};
2754
2755/// This is a base class used to represent
2756/// VP_GATHER and VP_SCATTER nodes
2757///
2758class VPGatherScatterSDNode : public MemSDNode {
2759public:
2760 friend class SelectionDAG;
2761
2762 VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2763 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2764 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2765 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2766 LSBaseSDNodeBits.AddressingMode = IndexType;
2767 assert(getIndexType() == IndexType && "Value truncated");
2768 }
2769
2770 /// How is Index applied to BasePtr when computing addresses.
2771 ISD::MemIndexType getIndexType() const {
2772 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2773 }
2774 bool isIndexScaled() const {
2775 return !cast<ConstantSDNode>(Val: getScale())->isOne();
2776 }
2777 bool isIndexSigned() const { return isIndexTypeSigned(IndexType: getIndexType()); }
2778
2779 // In the both nodes address is Op1, mask is Op2:
2780 // VPGatherSDNode (Chain, base, index, scale, mask, vlen)
2781 // VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
2782 // Mask is a vector of i1 elements
2783 const SDValue &getBasePtr() const {
2784 return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 1 : 2);
2785 }
2786 const SDValue &getIndex() const {
2787 return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 2 : 3);
2788 }
2789 const SDValue &getScale() const {
2790 return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 3 : 4);
2791 }
2792 const SDValue &getMask() const {
2793 return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 4 : 5);
2794 }
2795 const SDValue &getVectorLength() const {
2796 return getOperand(Num: (getOpcode() == ISD::VP_GATHER) ? 5 : 6);
2797 }
2798
2799 static bool classof(const SDNode *N) {
2800 return N->getOpcode() == ISD::VP_GATHER ||
2801 N->getOpcode() == ISD::VP_SCATTER;
2802 }
2803};
2804
2805/// This class is used to represent an VP_GATHER node
2806///
2807class VPGatherSDNode : public VPGatherScatterSDNode {
2808public:
2809 friend class SelectionDAG;
2810
2811 VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2812 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2813 : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
2814 IndexType) {}
2815
2816 static bool classof(const SDNode *N) {
2817 return N->getOpcode() == ISD::VP_GATHER;
2818 }
2819};
2820
2821/// This class is used to represent an VP_SCATTER node
2822///
2823class VPScatterSDNode : public VPGatherScatterSDNode {
2824public:
2825 friend class SelectionDAG;
2826
2827 VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2828 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2829 : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
2830 IndexType) {}
2831
2832 const SDValue &getValue() const { return getOperand(Num: 1); }
2833
2834 static bool classof(const SDNode *N) {
2835 return N->getOpcode() == ISD::VP_SCATTER;
2836 }
2837};
2838
2839/// This is a base class used to represent
2840/// MGATHER and MSCATTER nodes
2841///
2842class MaskedGatherScatterSDNode : public MemSDNode {
2843public:
2844 friend class SelectionDAG;
2845
2846 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2847 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2848 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2849 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2850 LSBaseSDNodeBits.AddressingMode = IndexType;
2851 assert(getIndexType() == IndexType && "Value truncated");
2852 }
2853
2854 /// How is Index applied to BasePtr when computing addresses.
2855 ISD::MemIndexType getIndexType() const {
2856 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2857 }
2858 bool isIndexScaled() const {
2859 return !cast<ConstantSDNode>(Val: getScale())->isOne();
2860 }
2861 bool isIndexSigned() const { return isIndexTypeSigned(IndexType: getIndexType()); }
2862
2863 // In the both nodes address is Op1, mask is Op2:
2864 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2865 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2866 // Mask is a vector of i1 elements
2867 const SDValue &getBasePtr() const { return getOperand(Num: 3); }
2868 const SDValue &getIndex() const { return getOperand(Num: 4); }
2869 const SDValue &getMask() const { return getOperand(Num: 2); }
2870 const SDValue &getScale() const { return getOperand(Num: 5); }
2871
2872 static bool classof(const SDNode *N) {
2873 return N->getOpcode() == ISD::MGATHER ||
2874 N->getOpcode() == ISD::MSCATTER;
2875 }
2876};
2877
2878/// This class is used to represent an MGATHER node
2879///
2880class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2881public:
2882 friend class SelectionDAG;
2883
2884 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2885 EVT MemVT, MachineMemOperand *MMO,
2886 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2887 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2888 IndexType) {
2889 LoadSDNodeBits.ExtTy = ETy;
2890 }
2891
2892 const SDValue &getPassThru() const { return getOperand(Num: 1); }
2893
2894 ISD::LoadExtType getExtensionType() const {
2895 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2896 }
2897
2898 static bool classof(const SDNode *N) {
2899 return N->getOpcode() == ISD::MGATHER;
2900 }
2901};
2902
2903/// This class is used to represent an MSCATTER node
2904///
2905class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2906public:
2907 friend class SelectionDAG;
2908
2909 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2910 EVT MemVT, MachineMemOperand *MMO,
2911 ISD::MemIndexType IndexType, bool IsTrunc)
2912 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2913 IndexType) {
2914 StoreSDNodeBits.IsTruncating = IsTrunc;
2915 }
2916
2917 /// Return true if the op does a truncation before store.
2918 /// For integers this is the same as doing a TRUNCATE and storing the result.
2919 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2920 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2921
2922 const SDValue &getValue() const { return getOperand(Num: 1); }
2923
2924 static bool classof(const SDNode *N) {
2925 return N->getOpcode() == ISD::MSCATTER;
2926 }
2927};
2928
2929class FPStateAccessSDNode : public MemSDNode {
2930public:
2931 friend class SelectionDAG;
2932
2933 FPStateAccessSDNode(unsigned NodeTy, unsigned Order, const DebugLoc &dl,
2934 SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
2935 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2936 assert((NodeTy == ISD::GET_FPENV_MEM || NodeTy == ISD::SET_FPENV_MEM) &&
2937 "Expected FP state access node");
2938 }
2939
2940 static bool classof(const SDNode *N) {
2941 return N->getOpcode() == ISD::GET_FPENV_MEM ||
2942 N->getOpcode() == ISD::SET_FPENV_MEM;
2943 }
2944};
2945
2946/// An SDNode that represents everything that will be needed
2947/// to construct a MachineInstr. These nodes are created during the
2948/// instruction selection proper phase.
2949///
2950/// Note that the only supported way to set the `memoperands` is by calling the
2951/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2952/// inside the DAG rather than in the node.
2953class MachineSDNode : public SDNode {
2954private:
2955 friend class SelectionDAG;
2956
2957 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2958 : SDNode(Opc, Order, DL, VTs) {}
2959
2960 // We use a pointer union between a single `MachineMemOperand` pointer and
2961 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2962 // the number of these is zero, the single pointer variant used when the
2963 // number is one, and the array is used for larger numbers.
2964 //
2965 // The array is allocated via the `SelectionDAG`'s allocator and so will
2966 // always live until the DAG is cleaned up and doesn't require ownership here.
2967 //
2968 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2969 // subclasses aren't managed in a conforming C++ manner. See the comments on
2970 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2971 // constraint here is that these don't manage memory with their constructor or
2972 // destructor and can be initialized to a good state even if they start off
2973 // uninitialized.
2974 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2975
2976 // Note that this could be folded into the above `MemRefs` member if doing so
2977 // is advantageous at some point. We don't need to store this in most cases.
2978 // However, at the moment this doesn't appear to make the allocation any
2979 // smaller and makes the code somewhat simpler to read.
2980 int NumMemRefs = 0;
2981
2982public:
2983 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2984
2985 ArrayRef<MachineMemOperand *> memoperands() const {
2986 // Special case the common cases.
2987 if (NumMemRefs == 0)
2988 return {};
2989 if (NumMemRefs == 1)
2990 return ArrayRef(MemRefs.getAddrOfPtr1(), 1);
2991
2992 // Otherwise we have an actual array.
2993 return ArrayRef(cast<MachineMemOperand **>(Val: MemRefs), NumMemRefs);
2994 }
2995 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2996 mmo_iterator memoperands_end() const { return memoperands().end(); }
2997 bool memoperands_empty() const { return memoperands().empty(); }
2998
2999 /// Clear out the memory reference descriptor list.
3000 void clearMemRefs() {
3001 MemRefs = nullptr;
3002 NumMemRefs = 0;
3003 }
3004
3005 static bool classof(const SDNode *N) {
3006 return N->isMachineOpcode();
3007 }
3008};
3009
3010/// An SDNode that records if a register contains a value that is guaranteed to
3011/// be aligned accordingly.
3012class AssertAlignSDNode : public SDNode {
3013 Align Alignment;
3014
3015public:
3016 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
3017 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
3018
3019 Align getAlign() const { return Alignment; }
3020
3021 static bool classof(const SDNode *N) {
3022 return N->getOpcode() == ISD::AssertAlign;
3023 }
3024};
3025
3026class SDNodeIterator {
3027 const SDNode *Node;
3028 unsigned Operand;
3029
3030 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
3031
3032public:
3033 using iterator_category = std::forward_iterator_tag;
3034 using value_type = SDNode;
3035 using difference_type = std::ptrdiff_t;
3036 using pointer = value_type *;
3037 using reference = value_type &;
3038
3039 bool operator==(const SDNodeIterator& x) const {
3040 return Operand == x.Operand;
3041 }
3042 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
3043
3044 pointer operator*() const {
3045 return Node->getOperand(Num: Operand).getNode();
3046 }
3047 pointer operator->() const { return operator*(); }
3048
3049 SDNodeIterator& operator++() { // Preincrement
3050 ++Operand;
3051 return *this;
3052 }
3053 SDNodeIterator operator++(int) { // Postincrement
3054 SDNodeIterator tmp = *this; ++*this; return tmp;
3055 }
3056 size_t operator-(SDNodeIterator Other) const {
3057 assert(Node == Other.Node &&
3058 "Cannot compare iterators of two different nodes!");
3059 return Operand - Other.Operand;
3060 }
3061
3062 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
3063 static SDNodeIterator end (const SDNode *N) {
3064 return SDNodeIterator(N, N->getNumOperands());
3065 }
3066
3067 unsigned getOperand() const { return Operand; }
3068 const SDNode *getNode() const { return Node; }
3069};
3070
3071template <> struct GraphTraits<SDNode*> {
3072 using NodeRef = SDNode *;
3073 using ChildIteratorType = SDNodeIterator;
3074
3075 static NodeRef getEntryNode(SDNode *N) { return N; }
3076
3077 static ChildIteratorType child_begin(NodeRef N) {
3078 return SDNodeIterator::begin(N);
3079 }
3080
3081 static ChildIteratorType child_end(NodeRef N) {
3082 return SDNodeIterator::end(N);
3083 }
3084};
3085
3086/// A representation of the largest SDNode, for use in sizeof().
3087///
3088/// This needs to be a union because the largest node differs on 32 bit systems
3089/// with 4 and 8 byte pointer alignment, respectively.
3090using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
3091 BlockAddressSDNode,
3092 GlobalAddressSDNode,
3093 PseudoProbeSDNode>;
3094
3095/// The SDNode class with the greatest alignment requirement.
3096using MostAlignedSDNode = GlobalAddressSDNode;
3097
3098namespace ISD {
3099
3100 /// Returns true if the specified node is a non-extending and unindexed load.
3101 inline bool isNormalLoad(const SDNode *N) {
3102 auto *Ld = dyn_cast<LoadSDNode>(Val: N);
3103 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
3104 Ld->getAddressingMode() == ISD::UNINDEXED;
3105 }
3106
3107 /// Returns true if the specified node is a non-extending load.
3108 inline bool isNON_EXTLoad(const SDNode *N) {
3109 auto *Ld = dyn_cast<LoadSDNode>(Val: N);
3110 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD;
3111 }
3112
3113 /// Returns true if the specified node is a EXTLOAD.
3114 inline bool isEXTLoad(const SDNode *N) {
3115 auto *Ld = dyn_cast<LoadSDNode>(Val: N);
3116 return Ld && Ld->getExtensionType() == ISD::EXTLOAD;
3117 }
3118
3119 /// Returns true if the specified node is a SEXTLOAD.
3120 inline bool isSEXTLoad(const SDNode *N) {
3121 auto *Ld = dyn_cast<LoadSDNode>(Val: N);
3122 return Ld && Ld->getExtensionType() == ISD::SEXTLOAD;
3123 }
3124
3125 /// Returns true if the specified node is a ZEXTLOAD.
3126 inline bool isZEXTLoad(const SDNode *N) {
3127 auto *Ld = dyn_cast<LoadSDNode>(Val: N);
3128 return Ld && Ld->getExtensionType() == ISD::ZEXTLOAD;
3129 }
3130
3131 /// Returns true if the specified node is an unindexed load.
3132 inline bool isUNINDEXEDLoad(const SDNode *N) {
3133 auto *Ld = dyn_cast<LoadSDNode>(Val: N);
3134 return Ld && Ld->getAddressingMode() == ISD::UNINDEXED;
3135 }
3136
3137 /// Returns true if the specified node is a non-truncating
3138 /// and unindexed store.
3139 inline bool isNormalStore(const SDNode *N) {
3140 auto *St = dyn_cast<StoreSDNode>(Val: N);
3141 return St && !St->isTruncatingStore() &&
3142 St->getAddressingMode() == ISD::UNINDEXED;
3143 }
3144
3145 /// Returns true if the specified node is an unindexed store.
3146 inline bool isUNINDEXEDStore(const SDNode *N) {
3147 auto *St = dyn_cast<StoreSDNode>(Val: N);
3148 return St && St->getAddressingMode() == ISD::UNINDEXED;
3149 }
3150
3151 /// Attempt to match a unary predicate against a scalar/splat constant or
3152 /// every element of a constant BUILD_VECTOR.
3153 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
3154 template <typename ConstNodeType>
3155 bool matchUnaryPredicateImpl(SDValue Op,
3156 std::function<bool(ConstNodeType *)> Match,
3157 bool AllowUndefs = false);
3158
3159 /// Hook for matching ConstantSDNode predicate
3160 inline bool matchUnaryPredicate(SDValue Op,
3161 std::function<bool(ConstantSDNode *)> Match,
3162 bool AllowUndefs = false) {
3163 return matchUnaryPredicateImpl<ConstantSDNode>(Op, Match, AllowUndefs);
3164 }
3165
3166 /// Hook for matching ConstantFPSDNode predicate
3167 inline bool
3168 matchUnaryFpPredicate(SDValue Op,
3169 std::function<bool(ConstantFPSDNode *)> Match,
3170 bool AllowUndefs = false) {
3171 return matchUnaryPredicateImpl<ConstantFPSDNode>(Op, Match, AllowUndefs);
3172 }
3173
3174 /// Attempt to match a binary predicate against a pair of scalar/splat
3175 /// constants or every element of a pair of constant BUILD_VECTORs.
3176 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
3177 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
3178 bool matchBinaryPredicate(
3179 SDValue LHS, SDValue RHS,
3180 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
3181 bool AllowUndefs = false, bool AllowTypeMismatch = false);
3182
3183 /// Returns true if the specified value is the overflow result from one
3184 /// of the overflow intrinsic nodes.
3185 inline bool isOverflowIntrOpRes(SDValue Op) {
3186 unsigned Opc = Op.getOpcode();
3187 return (Op.getResNo() == 1 &&
3188 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
3189 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
3190 }
3191
3192} // end namespace ISD
3193
3194} // end namespace llvm
3195
3196#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H
3197

source code of llvm/include/llvm/CodeGen/SelectionDAGNodes.h