1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56#include <utility>
57
58namespace llvm {
59
60class APInt;
61class Constant;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125/// Return true if the specified node is FREEZE(UNDEF).
126bool isFreezeUndef(const SDNode *N);
127
128} // end namespace ISD
129
130//===----------------------------------------------------------------------===//
131/// Unlike LLVM values, Selection DAG nodes may return multiple
132/// values as the result of a computation. Many nodes return multiple values,
133/// from loads (which define a token and a return value) to ADDC (which returns
134/// a result and a carry value), to calls (which may return an arbitrary number
135/// of values).
136///
137/// As such, each use of a SelectionDAG computation must indicate the node that
138/// computes it as well as which return value to use from that node. This pair
139/// of information is represented with the SDValue value type.
140///
141class SDValue {
142 friend struct DenseMapInfo<SDValue>;
143
144 SDNode *Node = nullptr; // The node defining the value we are using.
145 unsigned ResNo = 0; // Which return value of the node we are using.
146
147public:
148 SDValue() = default;
149 SDValue(SDNode *node, unsigned resno);
150
151 /// get the index which selects a specific result in the SDNode
152 unsigned getResNo() const { return ResNo; }
153
154 /// get the SDNode which holds the desired result
155 SDNode *getNode() const { return Node; }
156
157 /// set the SDNode
158 void setNode(SDNode *N) { Node = N; }
159
160 inline SDNode *operator->() const { return Node; }
161
162 bool operator==(const SDValue &O) const {
163 return Node == O.Node && ResNo == O.ResNo;
164 }
165 bool operator!=(const SDValue &O) const {
166 return !operator==(O);
167 }
168 bool operator<(const SDValue &O) const {
169 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
170 }
171 explicit operator bool() const {
172 return Node != nullptr;
173 }
174
175 SDValue getValue(unsigned R) const {
176 return SDValue(Node, R);
177 }
178
179 /// Return true if this node is an operand of N.
180 bool isOperandOf(const SDNode *N) const;
181
182 /// Return the ValueType of the referenced return value.
183 inline EVT getValueType() const;
184
185 /// Return the simple ValueType of the referenced return value.
186 MVT getSimpleValueType() const {
187 return getValueType().getSimpleVT();
188 }
189
190 /// Returns the size of the value in bits.
191 ///
192 /// If the value type is a scalable vector type, the scalable property will
193 /// be set and the runtime size will be a positive integer multiple of the
194 /// base size.
195 TypeSize getValueSizeInBits() const {
196 return getValueType().getSizeInBits();
197 }
198
199 uint64_t getScalarValueSizeInBits() const {
200 return getValueType().getScalarType().getFixedSizeInBits();
201 }
202
203 // Forwarding methods - These forward to the corresponding methods in SDNode.
204 inline unsigned getOpcode() const;
205 inline unsigned getNumOperands() const;
206 inline const SDValue &getOperand(unsigned i) const;
207 inline uint64_t getConstantOperandVal(unsigned i) const;
208 inline const APInt &getConstantOperandAPInt(unsigned i) const;
209 inline bool isTargetMemoryOpcode() const;
210 inline bool isTargetOpcode() const;
211 inline bool isMachineOpcode() const;
212 inline bool isUndef() const;
213 inline unsigned getMachineOpcode() const;
214 inline const DebugLoc &getDebugLoc() const;
215 inline void dump() const;
216 inline void dump(const SelectionDAG *G) const;
217 inline void dumpr() const;
218 inline void dumpr(const SelectionDAG *G) const;
219
220 /// Return true if this operand (which must be a chain) reaches the
221 /// specified operand without crossing any side-effecting instructions.
222 /// In practice, this looks through token factors and non-volatile loads.
223 /// In order to remain efficient, this only
224 /// looks a couple of nodes in, it does not do an exhaustive search.
225 bool reachesChainWithoutSideEffects(SDValue Dest,
226 unsigned Depth = 2) const;
227
228 /// Return true if there are no nodes using value ResNo of Node.
229 inline bool use_empty() const;
230
231 /// Return true if there is exactly one node using value ResNo of Node.
232 inline bool hasOneUse() const;
233};
234
235template<> struct DenseMapInfo<SDValue> {
236 static inline SDValue getEmptyKey() {
237 SDValue V;
238 V.ResNo = -1U;
239 return V;
240 }
241
242 static inline SDValue getTombstoneKey() {
243 SDValue V;
244 V.ResNo = -2U;
245 return V;
246 }
247
248 static unsigned getHashValue(const SDValue &Val) {
249 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
250 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
251 }
252
253 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
254 return LHS == RHS;
255 }
256};
257
258/// Allow casting operators to work directly on
259/// SDValues as if they were SDNode*'s.
260template<> struct simplify_type<SDValue> {
261 using SimpleType = SDNode *;
262
263 static SimpleType getSimplifiedValue(SDValue &Val) {
264 return Val.getNode();
265 }
266};
267template<> struct simplify_type<const SDValue> {
268 using SimpleType = /*const*/ SDNode *;
269
270 static SimpleType getSimplifiedValue(const SDValue &Val) {
271 return Val.getNode();
272 }
273};
274
275/// Represents a use of a SDNode. This class holds an SDValue,
276/// which records the SDNode being used and the result number, a
277/// pointer to the SDNode using the value, and Next and Prev pointers,
278/// which link together all the uses of an SDNode.
279///
280class SDUse {
281 /// Val - The value being used.
282 SDValue Val;
283 /// User - The user of this value.
284 SDNode *User = nullptr;
285 /// Prev, Next - Pointers to the uses list of the SDNode referred by
286 /// this operand.
287 SDUse **Prev = nullptr;
288 SDUse *Next = nullptr;
289
290public:
291 SDUse() = default;
292 SDUse(const SDUse &U) = delete;
293 SDUse &operator=(const SDUse &) = delete;
294
295 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
296 operator const SDValue&() const { return Val; }
297
298 /// If implicit conversion to SDValue doesn't work, the get() method returns
299 /// the SDValue.
300 const SDValue &get() const { return Val; }
301
302 /// This returns the SDNode that contains this Use.
303 SDNode *getUser() { return User; }
304
305 /// Get the next SDUse in the use list.
306 SDUse *getNext() const { return Next; }
307
308 /// Convenience function for get().getNode().
309 SDNode *getNode() const { return Val.getNode(); }
310 /// Convenience function for get().getResNo().
311 unsigned getResNo() const { return Val.getResNo(); }
312 /// Convenience function for get().getValueType().
313 EVT getValueType() const { return Val.getValueType(); }
314
315 /// Convenience function for get().operator==
316 bool operator==(const SDValue &V) const {
317 return Val == V;
318 }
319
320 /// Convenience function for get().operator!=
321 bool operator!=(const SDValue &V) const {
322 return Val != V;
323 }
324
325 /// Convenience function for get().operator<
326 bool operator<(const SDValue &V) const {
327 return Val < V;
328 }
329
330private:
331 friend class SelectionDAG;
332 friend class SDNode;
333 // TODO: unfriend HandleSDNode once we fix its operand handling.
334 friend class HandleSDNode;
335
336 void setUser(SDNode *p) { User = p; }
337
338 /// Remove this use from its existing use list, assign it the
339 /// given value, and add it to the new value's node's use list.
340 inline void set(const SDValue &V);
341 /// Like set, but only supports initializing a newly-allocated
342 /// SDUse with a non-null value.
343 inline void setInitial(const SDValue &V);
344 /// Like set, but only sets the Node portion of the value,
345 /// leaving the ResNo portion unmodified.
346 inline void setNode(SDNode *N);
347
348 void addToList(SDUse **List) {
349 Next = *List;
350 if (Next) Next->Prev = &Next;
351 Prev = List;
352 *List = this;
353 }
354
355 void removeFromList() {
356 *Prev = Next;
357 if (Next) Next->Prev = Prev;
358 }
359};
360
361/// simplify_type specializations - Allow casting operators to work directly on
362/// SDValues as if they were SDNode*'s.
363template<> struct simplify_type<SDUse> {
364 using SimpleType = SDNode *;
365
366 static SimpleType getSimplifiedValue(SDUse &Val) {
367 return Val.getNode();
368 }
369};
370
371/// These are IR-level optimization flags that may be propagated to SDNodes.
372/// TODO: This data structure should be shared by the IR optimizer and the
373/// the backend.
374struct SDNodeFlags {
375private:
376 bool NoUnsignedWrap : 1;
377 bool NoSignedWrap : 1;
378 bool Exact : 1;
379 bool NoNaNs : 1;
380 bool NoInfs : 1;
381 bool NoSignedZeros : 1;
382 bool AllowReciprocal : 1;
383 bool AllowContract : 1;
384 bool ApproximateFuncs : 1;
385 bool AllowReassociation : 1;
386
387 // We assume instructions do not raise floating-point exceptions by default,
388 // and only those marked explicitly may do so. We could choose to represent
389 // this via a positive "FPExcept" flags like on the MI level, but having a
390 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
391 // intersection logic more straightforward.
392 bool NoFPExcept : 1;
393
394public:
395 /// Default constructor turns off all optimization flags.
396 SDNodeFlags()
397 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
398 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
399 AllowContract(false), ApproximateFuncs(false),
400 AllowReassociation(false), NoFPExcept(false) {}
401
402 /// Propagate the fast-math-flags from an IR FPMathOperator.
403 void copyFMF(const FPMathOperator &FPMO) {
404 setNoNaNs(FPMO.hasNoNaNs());
405 setNoInfs(FPMO.hasNoInfs());
406 setNoSignedZeros(FPMO.hasNoSignedZeros());
407 setAllowReciprocal(FPMO.hasAllowReciprocal());
408 setAllowContract(FPMO.hasAllowContract());
409 setApproximateFuncs(FPMO.hasApproxFunc());
410 setAllowReassociation(FPMO.hasAllowReassoc());
411 }
412
413 // These are mutators for each flag.
414 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
415 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
416 void setExact(bool b) { Exact = b; }
417 void setNoNaNs(bool b) { NoNaNs = b; }
418 void setNoInfs(bool b) { NoInfs = b; }
419 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
420 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
421 void setAllowContract(bool b) { AllowContract = b; }
422 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
423 void setAllowReassociation(bool b) { AllowReassociation = b; }
424 void setNoFPExcept(bool b) { NoFPExcept = b; }
425
426 // These are accessors for each flag.
427 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
428 bool hasNoSignedWrap() const { return NoSignedWrap; }
429 bool hasExact() const { return Exact; }
430 bool hasNoNaNs() const { return NoNaNs; }
431 bool hasNoInfs() const { return NoInfs; }
432 bool hasNoSignedZeros() const { return NoSignedZeros; }
433 bool hasAllowReciprocal() const { return AllowReciprocal; }
434 bool hasAllowContract() const { return AllowContract; }
435 bool hasApproximateFuncs() const { return ApproximateFuncs; }
436 bool hasAllowReassociation() const { return AllowReassociation; }
437 bool hasNoFPExcept() const { return NoFPExcept; }
438
439 /// Clear any flags in this flag set that aren't also set in Flags. All
440 /// flags will be cleared if Flags are undefined.
441 void intersectWith(const SDNodeFlags Flags) {
442 NoUnsignedWrap &= Flags.NoUnsignedWrap;
443 NoSignedWrap &= Flags.NoSignedWrap;
444 Exact &= Flags.Exact;
445 NoNaNs &= Flags.NoNaNs;
446 NoInfs &= Flags.NoInfs;
447 NoSignedZeros &= Flags.NoSignedZeros;
448 AllowReciprocal &= Flags.AllowReciprocal;
449 AllowContract &= Flags.AllowContract;
450 ApproximateFuncs &= Flags.ApproximateFuncs;
451 AllowReassociation &= Flags.AllowReassociation;
452 NoFPExcept &= Flags.NoFPExcept;
453 }
454};
455
456/// Represents one node in the SelectionDAG.
457///
458class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
459private:
460 /// The operation that this node performs.
461 int16_t NodeType;
462
463protected:
464 // We define a set of mini-helper classes to help us interpret the bits in our
465 // SubclassData. These are designed to fit within a uint16_t so they pack
466 // with NodeType.
467
468#if defined(_AIX) && (!defined(__GNUC__) || defined(__clang__))
469// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
470// and give the `pack` pragma push semantics.
471#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")
472#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")
473#else
474#define BEGIN_TWO_BYTE_PACK()
475#define END_TWO_BYTE_PACK()
476#endif
477
478BEGIN_TWO_BYTE_PACK()
479 class SDNodeBitfields {
480 friend class SDNode;
481 friend class MemIntrinsicSDNode;
482 friend class MemSDNode;
483 friend class SelectionDAG;
484
485 uint16_t HasDebugValue : 1;
486 uint16_t IsMemIntrinsic : 1;
487 uint16_t IsDivergent : 1;
488 };
489 enum { NumSDNodeBits = 3 };
490
491 class ConstantSDNodeBitfields {
492 friend class ConstantSDNode;
493
494 uint16_t : NumSDNodeBits;
495
496 uint16_t IsOpaque : 1;
497 };
498
499 class MemSDNodeBitfields {
500 friend class MemSDNode;
501 friend class MemIntrinsicSDNode;
502 friend class AtomicSDNode;
503
504 uint16_t : NumSDNodeBits;
505
506 uint16_t IsVolatile : 1;
507 uint16_t IsNonTemporal : 1;
508 uint16_t IsDereferenceable : 1;
509 uint16_t IsInvariant : 1;
510 };
511 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
512
513 class LSBaseSDNodeBitfields {
514 friend class LSBaseSDNode;
515 friend class VPBaseLoadStoreSDNode;
516 friend class MaskedLoadStoreSDNode;
517 friend class MaskedGatherScatterSDNode;
518 friend class VPGatherScatterSDNode;
519
520 uint16_t : NumMemSDNodeBits;
521
522 // This storage is shared between disparate class hierarchies to hold an
523 // enumeration specific to the class hierarchy in use.
524 // LSBaseSDNode => enum ISD::MemIndexedMode
525 // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
526 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
527 // VPGatherScatterSDNode => enum ISD::MemIndexType
528 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
529 uint16_t AddressingMode : 3;
530 };
531 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
532
533 class LoadSDNodeBitfields {
534 friend class LoadSDNode;
535 friend class VPLoadSDNode;
536 friend class VPStridedLoadSDNode;
537 friend class MaskedLoadSDNode;
538 friend class MaskedGatherSDNode;
539 friend class VPGatherSDNode;
540
541 uint16_t : NumLSBaseSDNodeBits;
542
543 uint16_t ExtTy : 2; // enum ISD::LoadExtType
544 uint16_t IsExpanding : 1;
545 };
546
547 class StoreSDNodeBitfields {
548 friend class StoreSDNode;
549 friend class VPStoreSDNode;
550 friend class VPStridedStoreSDNode;
551 friend class MaskedStoreSDNode;
552 friend class MaskedScatterSDNode;
553 friend class VPScatterSDNode;
554
555 uint16_t : NumLSBaseSDNodeBits;
556
557 uint16_t IsTruncating : 1;
558 uint16_t IsCompressing : 1;
559 };
560
561 union {
562 char RawSDNodeBits[sizeof(uint16_t)];
563 SDNodeBitfields SDNodeBits;
564 ConstantSDNodeBitfields ConstantSDNodeBits;
565 MemSDNodeBitfields MemSDNodeBits;
566 LSBaseSDNodeBitfields LSBaseSDNodeBits;
567 LoadSDNodeBitfields LoadSDNodeBits;
568 StoreSDNodeBitfields StoreSDNodeBits;
569 };
570END_TWO_BYTE_PACK()
571#undef BEGIN_TWO_BYTE_PACK
572#undef END_TWO_BYTE_PACK
573
574 // RawSDNodeBits must cover the entirety of the union. This means that all of
575 // the union's members must have size <= RawSDNodeBits. We write the RHS as
576 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
577 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
578 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
579 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
580 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
581 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
582 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
583
584private:
585 friend class SelectionDAG;
586 // TODO: unfriend HandleSDNode once we fix its operand handling.
587 friend class HandleSDNode;
588
589 /// Unique id per SDNode in the DAG.
590 int NodeId = -1;
591
592 /// The values that are used by this operation.
593 SDUse *OperandList = nullptr;
594
595 /// The types of the values this node defines. SDNode's may
596 /// define multiple values simultaneously.
597 const EVT *ValueList;
598
599 /// List of uses for this SDNode.
600 SDUse *UseList = nullptr;
601
602 /// The number of entries in the Operand/Value list.
603 unsigned short NumOperands = 0;
604 unsigned short NumValues;
605
606 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
607 // original LLVM instructions.
608 // This is used for turning off scheduling, because we'll forgo
609 // the normal scheduling algorithms and output the instructions according to
610 // this ordering.
611 unsigned IROrder;
612
613 /// Source line information.
614 DebugLoc debugLoc;
615
616 /// Return a pointer to the specified value type.
617 static const EVT *getValueTypeList(EVT VT);
618
619 SDNodeFlags Flags;
620
621public:
622 /// Unique and persistent id per SDNode in the DAG. Used for debug printing.
623 /// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS`
624 /// intentionally because it adds unneeded complexity without noticeable
625 /// benefits (see discussion with @thakis in D120714).
626 uint16_t PersistentId;
627
628 //===--------------------------------------------------------------------===//
629 // Accessors
630 //
631
632 /// Return the SelectionDAG opcode value for this node. For
633 /// pre-isel nodes (those for which isMachineOpcode returns false), these
634 /// are the opcode values in the ISD and <target>ISD namespaces. For
635 /// post-isel opcodes, see getMachineOpcode.
636 unsigned getOpcode() const { return (unsigned short)NodeType; }
637
638 /// Test if this node has a target-specific opcode (in the
639 /// \<target\>ISD namespace).
640 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
641
642 /// Test if this node has a target-specific opcode that may raise
643 /// FP exceptions (in the \<target\>ISD namespace and greater than
644 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
645 /// opcode are currently automatically considered to possibly raise
646 /// FP exceptions as well.
647 bool isTargetStrictFPOpcode() const {
648 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
649 }
650
651 /// Test if this node has a target-specific
652 /// memory-referencing opcode (in the \<target\>ISD namespace and
653 /// greater than FIRST_TARGET_MEMORY_OPCODE).
654 bool isTargetMemoryOpcode() const {
655 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
656 }
657
658 /// Return true if the type of the node type undefined.
659 bool isUndef() const { return NodeType == ISD::UNDEF; }
660
661 /// Test if this node is a memory intrinsic (with valid pointer information).
662 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
663 /// non-memory intrinsics (with chains) that are not really instances of
664 /// MemSDNode. For such nodes, we need some extra state to determine the
665 /// proper classof relationship.
666 bool isMemIntrinsic() const {
667 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
668 NodeType == ISD::INTRINSIC_VOID) &&
669 SDNodeBits.IsMemIntrinsic;
670 }
671
672 /// Test if this node is a strict floating point pseudo-op.
673 bool isStrictFPOpcode() {
674 switch (NodeType) {
675 default:
676 return false;
677 case ISD::STRICT_FP16_TO_FP:
678 case ISD::STRICT_FP_TO_FP16:
679#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
680 case ISD::STRICT_##DAGN:
681#include "llvm/IR/ConstrainedOps.def"
682 return true;
683 }
684 }
685
686 /// Test if this node is a vector predication operation.
687 bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }
688
689 /// Test if this node has a post-isel opcode, directly
690 /// corresponding to a MachineInstr opcode.
691 bool isMachineOpcode() const { return NodeType < 0; }
692
693 /// This may only be called if isMachineOpcode returns
694 /// true. It returns the MachineInstr opcode value that the node's opcode
695 /// corresponds to.
696 unsigned getMachineOpcode() const {
697 assert(isMachineOpcode() && "Not a MachineInstr opcode!");
698 return ~NodeType;
699 }
700
701 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
702 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
703
704 bool isDivergent() const { return SDNodeBits.IsDivergent; }
705
706 /// Return true if there are no uses of this node.
707 bool use_empty() const { return UseList == nullptr; }
708
709 /// Return true if there is exactly one use of this node.
710 bool hasOneUse() const { return hasSingleElement(uses()); }
711
712 /// Return the number of uses of this node. This method takes
713 /// time proportional to the number of uses.
714 size_t use_size() const { return std::distance(use_begin(), use_end()); }
715
716 /// Return the unique node id.
717 int getNodeId() const { return NodeId; }
718
719 /// Set unique node id.
720 void setNodeId(int Id) { NodeId = Id; }
721
722 /// Return the node ordering.
723 unsigned getIROrder() const { return IROrder; }
724
725 /// Set the node ordering.
726 void setIROrder(unsigned Order) { IROrder = Order; }
727
728 /// Return the source location info.
729 const DebugLoc &getDebugLoc() const { return debugLoc; }
730
731 /// Set source location info. Try to avoid this, putting
732 /// it in the constructor is preferable.
733 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
734
735 /// This class provides iterator support for SDUse
736 /// operands that use a specific SDNode.
737 class use_iterator {
738 friend class SDNode;
739
740 SDUse *Op = nullptr;
741
742 explicit use_iterator(SDUse *op) : Op(op) {}
743
744 public:
745 using iterator_category = std::forward_iterator_tag;
746 using value_type = SDUse;
747 using difference_type = std::ptrdiff_t;
748 using pointer = value_type *;
749 using reference = value_type &;
750
751 use_iterator() = default;
752 use_iterator(const use_iterator &I) = default;
753
754 bool operator==(const use_iterator &x) const { return Op == x.Op; }
755 bool operator!=(const use_iterator &x) const {
756 return !operator==(x);
757 }
758
759 /// Return true if this iterator is at the end of uses list.
760 bool atEnd() const { return Op == nullptr; }
761
762 // Iterator traversal: forward iteration only.
763 use_iterator &operator++() { // Preincrement
764 assert(Op && "Cannot increment end iterator!");
765 Op = Op->getNext();
766 return *this;
767 }
768
769 use_iterator operator++(int) { // Postincrement
770 use_iterator tmp = *this; ++*this; return tmp;
771 }
772
773 /// Retrieve a pointer to the current user node.
774 SDNode *operator*() const {
775 assert(Op && "Cannot dereference end iterator!");
776 return Op->getUser();
777 }
778
779 SDNode *operator->() const { return operator*(); }
780
781 SDUse &getUse() const { return *Op; }
782
783 /// Retrieve the operand # of this use in its user.
784 unsigned getOperandNo() const {
785 assert(Op && "Cannot dereference end iterator!");
786 return (unsigned)(Op - Op->getUser()->OperandList);
787 }
788 };
789
790 /// Provide iteration support to walk over all uses of an SDNode.
791 use_iterator use_begin() const {
792 return use_iterator(UseList);
793 }
794
795 static use_iterator use_end() { return use_iterator(nullptr); }
796
797 inline iterator_range<use_iterator> uses() {
798 return make_range(use_begin(), use_end());
799 }
800 inline iterator_range<use_iterator> uses() const {
801 return make_range(use_begin(), use_end());
802 }
803
804 /// Return true if there are exactly NUSES uses of the indicated value.
805 /// This method ignores uses of other values defined by this operation.
806 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
807
808 /// Return true if there are any use of the indicated value.
809 /// This method ignores uses of other values defined by this operation.
810 bool hasAnyUseOfValue(unsigned Value) const;
811
812 /// Return true if this node is the only use of N.
813 bool isOnlyUserOf(const SDNode *N) const;
814
815 /// Return true if this node is an operand of N.
816 bool isOperandOf(const SDNode *N) const;
817
818 /// Return true if this node is a predecessor of N.
819 /// NOTE: Implemented on top of hasPredecessor and every bit as
820 /// expensive. Use carefully.
821 bool isPredecessorOf(const SDNode *N) const {
822 return N->hasPredecessor(this);
823 }
824
825 /// Return true if N is a predecessor of this node.
826 /// N is either an operand of this node, or can be reached by recursively
827 /// traversing up the operands.
828 /// NOTE: This is an expensive method. Use it carefully.
829 bool hasPredecessor(const SDNode *N) const;
830
831 /// Returns true if N is a predecessor of any node in Worklist. This
832 /// helper keeps Visited and Worklist sets externally to allow unions
833 /// searches to be performed in parallel, caching of results across
834 /// queries and incremental addition to Worklist. Stops early if N is
835 /// found but will resume. Remember to clear Visited and Worklists
836 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
837 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
838 /// topologically ordered (Operands have strictly smaller node id) and search
839 /// can be pruned leveraging this.
840 static bool hasPredecessorHelper(const SDNode *N,
841 SmallPtrSetImpl<const SDNode *> &Visited,
842 SmallVectorImpl<const SDNode *> &Worklist,
843 unsigned int MaxSteps = 0,
844 bool TopologicalPrune = false) {
845 SmallVector<const SDNode *, 8> DeferredNodes;
846 if (Visited.count(N))
847 return true;
848
849 // Node Id's are assigned in three places: As a topological
850 // ordering (> 0), during legalization (results in values set to
851 // 0), new nodes (set to -1). If N has a topolgical id then we
852 // know that all nodes with ids smaller than it cannot be
853 // successors and we need not check them. Filter out all node
854 // that can't be matches. We add them to the worklist before exit
855 // in case of multiple calls. Note that during selection the topological id
856 // may be violated if a node's predecessor is selected before it. We mark
857 // this at selection negating the id of unselected successors and
858 // restricting topological pruning to positive ids.
859
860 int NId = N->getNodeId();
861 // If we Invalidated the Id, reconstruct original NId.
862 if (NId < -1)
863 NId = -(NId + 1);
864
865 bool Found = false;
866 while (!Worklist.empty()) {
867 const SDNode *M = Worklist.pop_back_val();
868 int MId = M->getNodeId();
869 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
870 (MId > 0) && (MId < NId)) {
871 DeferredNodes.push_back(M);
872 continue;
873 }
874 for (const SDValue &OpV : M->op_values()) {
875 SDNode *Op = OpV.getNode();
876 if (Visited.insert(Op).second)
877 Worklist.push_back(Op);
878 if (Op == N)
879 Found = true;
880 }
881 if (Found)
882 break;
883 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
884 break;
885 }
886 // Push deferred nodes back on worklist.
887 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
888 // If we bailed early, conservatively return found.
889 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
890 return true;
891 return Found;
892 }
893
894 /// Return true if all the users of N are contained in Nodes.
895 /// NOTE: Requires at least one match, but doesn't require them all.
896 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
897
898 /// Return the number of values used by this operation.
899 unsigned getNumOperands() const { return NumOperands; }
900
901 /// Return the maximum number of operands that a SDNode can hold.
902 static constexpr size_t getMaxNumOperands() {
903 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
904 }
905
906 /// Helper method returns the integer value of a ConstantSDNode operand.
907 inline uint64_t getConstantOperandVal(unsigned Num) const;
908
909 /// Helper method returns the APInt of a ConstantSDNode operand.
910 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
911
912 const SDValue &getOperand(unsigned Num) const {
913 assert(Num < NumOperands && "Invalid child # of SDNode!");
914 return OperandList[Num];
915 }
916
917 using op_iterator = SDUse *;
918
919 op_iterator op_begin() const { return OperandList; }
920 op_iterator op_end() const { return OperandList+NumOperands; }
921 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
922
923 /// Iterator for directly iterating over the operand SDValue's.
924 struct value_op_iterator
925 : iterator_adaptor_base<value_op_iterator, op_iterator,
926 std::random_access_iterator_tag, SDValue,
927 ptrdiff_t, value_op_iterator *,
928 value_op_iterator *> {
929 explicit value_op_iterator(SDUse *U = nullptr)
930 : iterator_adaptor_base(U) {}
931
932 const SDValue &operator*() const { return I->get(); }
933 };
934
935 iterator_range<value_op_iterator> op_values() const {
936 return make_range(value_op_iterator(op_begin()),
937 value_op_iterator(op_end()));
938 }
939
940 SDVTList getVTList() const {
941 SDVTList X = { ValueList, NumValues };
942 return X;
943 }
944
945 /// If this node has a glue operand, return the node
946 /// to which the glue operand points. Otherwise return NULL.
947 SDNode *getGluedNode() const {
948 if (getNumOperands() != 0 &&
949 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
950 return getOperand(getNumOperands()-1).getNode();
951 return nullptr;
952 }
953
954 /// If this node has a glue value with a user, return
955 /// the user (there is at most one). Otherwise return NULL.
956 SDNode *getGluedUser() const {
957 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
958 if (UI.getUse().get().getValueType() == MVT::Glue)
959 return *UI;
960 return nullptr;
961 }
962
963 SDNodeFlags getFlags() const { return Flags; }
964 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
965
966 /// Clear any flags in this node that aren't also set in Flags.
967 /// If Flags is not in a defined state then this has no effect.
968 void intersectFlagsWith(const SDNodeFlags Flags);
969
970 /// Return the number of values defined/returned by this operator.
971 unsigned getNumValues() const { return NumValues; }
972
973 /// Return the type of a specified result.
974 EVT getValueType(unsigned ResNo) const {
975 assert(ResNo < NumValues && "Illegal result number!");
976 return ValueList[ResNo];
977 }
978
979 /// Return the type of a specified result as a simple type.
980 MVT getSimpleValueType(unsigned ResNo) const {
981 return getValueType(ResNo).getSimpleVT();
982 }
983
984 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
985 ///
986 /// If the value type is a scalable vector type, the scalable property will
987 /// be set and the runtime size will be a positive integer multiple of the
988 /// base size.
989 TypeSize getValueSizeInBits(unsigned ResNo) const {
990 return getValueType(ResNo).getSizeInBits();
991 }
992
993 using value_iterator = const EVT *;
994
995 value_iterator value_begin() const { return ValueList; }
996 value_iterator value_end() const { return ValueList+NumValues; }
997 iterator_range<value_iterator> values() const {
998 return llvm::make_range(value_begin(), value_end());
999 }
1000
1001 /// Return the opcode of this operation for printing.
1002 std::string getOperationName(const SelectionDAG *G = nullptr) const;
1003 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
1004 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
1005 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
1006 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1007 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1008
1009 /// Print a SelectionDAG node and all children down to
1010 /// the leaves. The given SelectionDAG allows target-specific nodes
1011 /// to be printed in human-readable form. Unlike printr, this will
1012 /// print the whole DAG, including children that appear multiple
1013 /// times.
1014 ///
1015 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1016
1017 /// Print a SelectionDAG node and children up to
1018 /// depth "depth." The given SelectionDAG allows target-specific
1019 /// nodes to be printed in human-readable form. Unlike printr, this
1020 /// will print children that appear multiple times wherever they are
1021 /// used.
1022 ///
1023 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1024 unsigned depth = 100) const;
1025
1026 /// Dump this node, for debugging.
1027 void dump() const;
1028
1029 /// Dump (recursively) this node and its use-def subgraph.
1030 void dumpr() const;
1031
1032 /// Dump this node, for debugging.
1033 /// The given SelectionDAG allows target-specific nodes to be printed
1034 /// in human-readable form.
1035 void dump(const SelectionDAG *G) const;
1036
1037 /// Dump (recursively) this node and its use-def subgraph.
1038 /// The given SelectionDAG allows target-specific nodes to be printed
1039 /// in human-readable form.
1040 void dumpr(const SelectionDAG *G) const;
1041
1042 /// printrFull to dbgs(). The given SelectionDAG allows
1043 /// target-specific nodes to be printed in human-readable form.
1044 /// Unlike dumpr, this will print the whole DAG, including children
1045 /// that appear multiple times.
1046 void dumprFull(const SelectionDAG *G = nullptr) const;
1047
1048 /// printrWithDepth to dbgs(). The given
1049 /// SelectionDAG allows target-specific nodes to be printed in
1050 /// human-readable form. Unlike dumpr, this will print children
1051 /// that appear multiple times wherever they are used.
1052 ///
1053 void dumprWithDepth(const SelectionDAG *G = nullptr,
1054 unsigned depth = 100) const;
1055
1056 /// Gather unique data for the node.
1057 void Profile(FoldingSetNodeID &ID) const;
1058
1059 /// This method should only be used by the SDUse class.
1060 void addUse(SDUse &U) { U.addToList(&UseList); }
1061
1062protected:
1063 static SDVTList getSDVTList(EVT VT) {
1064 SDVTList Ret = { getValueTypeList(VT), 1 };
1065 return Ret;
1066 }
1067
1068 /// Create an SDNode.
1069 ///
1070 /// SDNodes are created without any operands, and never own the operand
1071 /// storage. To add operands, see SelectionDAG::createOperands.
1072 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1073 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1074 IROrder(Order), debugLoc(std::move(dl)) {
1075 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1076 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
1077 assert(NumValues == VTs.NumVTs &&
1078 "NumValues wasn't wide enough for its operands!");
1079 }
1080
1081 /// Release the operands and set this node to have zero operands.
1082 void DropOperands();
1083};
1084
1085/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1086/// into SDNode creation functions.
1087/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1088/// from the original Instruction, and IROrder is the ordinal position of
1089/// the instruction.
1090/// When an SDNode is created after the DAG is being built, both DebugLoc and
1091/// the IROrder are propagated from the original SDNode.
1092/// So SDLoc class provides two constructors besides the default one, one to
1093/// be used by the DAGBuilder, the other to be used by others.
1094class SDLoc {
1095private:
1096 DebugLoc DL;
1097 int IROrder = 0;
1098
1099public:
1100 SDLoc() = default;
1101 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1102 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1103 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1104 assert(Order >= 0 && "bad IROrder");
1105 if (I)
1106 DL = I->getDebugLoc();
1107 }
1108
1109 unsigned getIROrder() const { return IROrder; }
1110 const DebugLoc &getDebugLoc() const { return DL; }
1111};
1112
1113// Define inline functions from the SDValue class.
1114
1115inline SDValue::SDValue(SDNode *node, unsigned resno)
1116 : Node(node), ResNo(resno) {
1117 // Explicitly check for !ResNo to avoid use-after-free, because there are
1118 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1119 // combines.
1120 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&
1121 "Invalid result number for the given node!");
1122 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.");
1123}
1124
1125inline unsigned SDValue::getOpcode() const {
1126 return Node->getOpcode();
1127}
1128
1129inline EVT SDValue::getValueType() const {
1130 return Node->getValueType(ResNo);
1131}
1132
1133inline unsigned SDValue::getNumOperands() const {
1134 return Node->getNumOperands();
1135}
1136
1137inline const SDValue &SDValue::getOperand(unsigned i) const {
1138 return Node->getOperand(i);
1139}
1140
1141inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1142 return Node->getConstantOperandVal(i);
1143}
1144
1145inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1146 return Node->getConstantOperandAPInt(i);
1147}
1148
1149inline bool SDValue::isTargetOpcode() const {
1150 return Node->isTargetOpcode();
1151}
1152
1153inline bool SDValue::isTargetMemoryOpcode() const {
1154 return Node->isTargetMemoryOpcode();
1155}
1156
1157inline bool SDValue::isMachineOpcode() const {
1158 return Node->isMachineOpcode();
1159}
1160
1161inline unsigned SDValue::getMachineOpcode() const {
1162 return Node->getMachineOpcode();
1163}
1164
1165inline bool SDValue::isUndef() const {
1166 return Node->isUndef();
1167}
1168
1169inline bool SDValue::use_empty() const {
1170 return !Node->hasAnyUseOfValue(ResNo);
1171}
1172
1173inline bool SDValue::hasOneUse() const {
1174 return Node->hasNUsesOfValue(1, ResNo);
1175}
1176
1177inline const DebugLoc &SDValue::getDebugLoc() const {
1178 return Node->getDebugLoc();
1179}
1180
1181inline void SDValue::dump() const {
1182 return Node->dump();
1183}
1184
1185inline void SDValue::dump(const SelectionDAG *G) const {
1186 return Node->dump(G);
1187}
1188
1189inline void SDValue::dumpr() const {
1190 return Node->dumpr();
1191}
1192
1193inline void SDValue::dumpr(const SelectionDAG *G) const {
1194 return Node->dumpr(G);
1195}
1196
1197// Define inline functions from the SDUse class.
1198
1199inline void SDUse::set(const SDValue &V) {
1200 if (Val.getNode()) removeFromList();
1201 Val = V;
1202 if (V.getNode())
1203 V->addUse(*this);
1204}
1205
1206inline void SDUse::setInitial(const SDValue &V) {
1207 Val = V;
1208 V->addUse(*this);
1209}
1210
1211inline void SDUse::setNode(SDNode *N) {
1212 if (Val.getNode()) removeFromList();
1213 Val.setNode(N);
1214 if (N) N->addUse(*this);
1215}
1216
1217/// This class is used to form a handle around another node that
1218/// is persistent and is updated across invocations of replaceAllUsesWith on its
1219/// operand. This node should be directly created by end-users and not added to
1220/// the AllNodes list.
1221class HandleSDNode : public SDNode {
1222 SDUse Op;
1223
1224public:
1225 explicit HandleSDNode(SDValue X)
1226 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1227 // HandleSDNodes are never inserted into the DAG, so they won't be
1228 // auto-numbered. Use ID 65535 as a sentinel.
1229 PersistentId = 0xffff;
1230
1231 // Manually set up the operand list. This node type is special in that it's
1232 // always stack allocated and SelectionDAG does not manage its operands.
1233 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1234 // be so special.
1235 Op.setUser(this);
1236 Op.setInitial(X);
1237 NumOperands = 1;
1238 OperandList = &Op;
1239 }
1240 ~HandleSDNode();
1241
1242 const SDValue &getValue() const { return Op; }
1243};
1244
1245class AddrSpaceCastSDNode : public SDNode {
1246private:
1247 unsigned SrcAddrSpace;
1248 unsigned DestAddrSpace;
1249
1250public:
1251 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1252 unsigned SrcAS, unsigned DestAS);
1253
1254 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1255 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1256
1257 static bool classof(const SDNode *N) {
1258 return N->getOpcode() == ISD::ADDRSPACECAST;
1259 }
1260};
1261
1262/// This is an abstract virtual class for memory operations.
1263class MemSDNode : public SDNode {
1264private:
1265 // VT of in-memory value.
1266 EVT MemoryVT;
1267
1268protected:
1269 /// Memory reference information.
1270 MachineMemOperand *MMO;
1271
1272public:
1273 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1274 EVT memvt, MachineMemOperand *MMO);
1275
1276 bool readMem() const { return MMO->isLoad(); }
1277 bool writeMem() const { return MMO->isStore(); }
1278
1279 /// Returns alignment and volatility of the memory access
1280 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1281 Align getAlign() const { return MMO->getAlign(); }
1282 // FIXME: Remove once transition to getAlign is over.
1283 unsigned getAlignment() const { return MMO->getAlign().value(); }
1284
1285 /// Return the SubclassData value, without HasDebugValue. This contains an
1286 /// encoding of the volatile flag, as well as bits used by subclasses. This
1287 /// function should only be used to compute a FoldingSetNodeID value.
1288 /// The HasDebugValue bit is masked out because CSE map needs to match
1289 /// nodes with debug info with nodes without debug info. Same is about
1290 /// isDivergent bit.
1291 unsigned getRawSubclassData() const {
1292 uint16_t Data;
1293 union {
1294 char RawSDNodeBits[sizeof(uint16_t)];
1295 SDNodeBitfields SDNodeBits;
1296 };
1297 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1298 SDNodeBits.HasDebugValue = 0;
1299 SDNodeBits.IsDivergent = false;
1300 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1301 return Data;
1302 }
1303
1304 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1305 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1306 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1307 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1308
1309 // Returns the offset from the location of the access.
1310 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1311
1312 /// Returns the AA info that describes the dereference.
1313 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1314
1315 /// Returns the Ranges that describes the dereference.
1316 const MDNode *getRanges() const { return MMO->getRanges(); }
1317
1318 /// Returns the synchronization scope ID for this memory operation.
1319 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1320
1321 /// Return the atomic ordering requirements for this memory operation. For
1322 /// cmpxchg atomic operations, return the atomic ordering requirements when
1323 /// store occurs.
1324 AtomicOrdering getSuccessOrdering() const {
1325 return MMO->getSuccessOrdering();
1326 }
1327
1328 /// Return a single atomic ordering that is at least as strong as both the
1329 /// success and failure orderings for an atomic operation. (For operations
1330 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1331 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1332
1333 /// Return true if the memory operation ordering is Unordered or higher.
1334 bool isAtomic() const { return MMO->isAtomic(); }
1335
1336 /// Returns true if the memory operation doesn't imply any ordering
1337 /// constraints on surrounding memory operations beyond the normal memory
1338 /// aliasing rules.
1339 bool isUnordered() const { return MMO->isUnordered(); }
1340
1341 /// Returns true if the memory operation is neither atomic or volatile.
1342 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1343
1344 /// Return the type of the in-memory value.
1345 EVT getMemoryVT() const { return MemoryVT; }
1346
1347 /// Return a MachineMemOperand object describing the memory
1348 /// reference performed by operation.
1349 MachineMemOperand *getMemOperand() const { return MMO; }
1350
1351 const MachinePointerInfo &getPointerInfo() const {
1352 return MMO->getPointerInfo();
1353 }
1354
1355 /// Return the address space for the associated pointer
1356 unsigned getAddressSpace() const {
1357 return getPointerInfo().getAddrSpace();
1358 }
1359
1360 /// Update this MemSDNode's MachineMemOperand information
1361 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1362 /// This must only be used when the new alignment applies to all users of
1363 /// this MachineMemOperand.
1364 void refineAlignment(const MachineMemOperand *NewMMO) {
1365 MMO->refineAlignment(NewMMO);
1366 }
1367
1368 const SDValue &getChain() const { return getOperand(0); }
1369
1370 const SDValue &getBasePtr() const {
1371 switch (getOpcode()) {
1372 case ISD::STORE:
1373 case ISD::VP_STORE:
1374 case ISD::MSTORE:
1375 case ISD::VP_SCATTER:
1376 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1377 return getOperand(2);
1378 case ISD::MGATHER:
1379 case ISD::MSCATTER:
1380 return getOperand(3);
1381 default:
1382 return getOperand(1);
1383 }
1384 }
1385
1386 // Methods to support isa and dyn_cast
1387 static bool classof(const SDNode *N) {
1388 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1389 // with either an intrinsic or a target opcode.
1390 switch (N->getOpcode()) {
1391 case ISD::LOAD:
1392 case ISD::STORE:
1393 case ISD::PREFETCH:
1394 case ISD::ATOMIC_CMP_SWAP:
1395 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1396 case ISD::ATOMIC_SWAP:
1397 case ISD::ATOMIC_LOAD_ADD:
1398 case ISD::ATOMIC_LOAD_SUB:
1399 case ISD::ATOMIC_LOAD_AND:
1400 case ISD::ATOMIC_LOAD_CLR:
1401 case ISD::ATOMIC_LOAD_OR:
1402 case ISD::ATOMIC_LOAD_XOR:
1403 case ISD::ATOMIC_LOAD_NAND:
1404 case ISD::ATOMIC_LOAD_MIN:
1405 case ISD::ATOMIC_LOAD_MAX:
1406 case ISD::ATOMIC_LOAD_UMIN:
1407 case ISD::ATOMIC_LOAD_UMAX:
1408 case ISD::ATOMIC_LOAD_FADD:
1409 case ISD::ATOMIC_LOAD_FSUB:
1410 case ISD::ATOMIC_LOAD_FMAX:
1411 case ISD::ATOMIC_LOAD_FMIN:
1412 case ISD::ATOMIC_LOAD:
1413 case ISD::ATOMIC_STORE:
1414 case ISD::MLOAD:
1415 case ISD::MSTORE:
1416 case ISD::MGATHER:
1417 case ISD::MSCATTER:
1418 case ISD::VP_LOAD:
1419 case ISD::VP_STORE:
1420 case ISD::VP_GATHER:
1421 case ISD::VP_SCATTER:
1422 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1423 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1424 return true;
1425 default:
1426 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1427 }
1428 }
1429};
1430
1431/// This is an SDNode representing atomic operations.
1432class AtomicSDNode : public MemSDNode {
1433public:
1434 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1435 EVT MemVT, MachineMemOperand *MMO)
1436 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1437 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||
1438 MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
1439 }
1440
1441 const SDValue &getBasePtr() const { return getOperand(1); }
1442 const SDValue &getVal() const { return getOperand(2); }
1443
1444 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1445 /// otherwise.
1446 bool isCompareAndSwap() const {
1447 unsigned Op = getOpcode();
1448 return Op == ISD::ATOMIC_CMP_SWAP ||
1449 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1450 }
1451
1452 /// For cmpxchg atomic operations, return the atomic ordering requirements
1453 /// when store does not occur.
1454 AtomicOrdering getFailureOrdering() const {
1455 assert(isCompareAndSwap() && "Must be cmpxchg operation");
1456 return MMO->getFailureOrdering();
1457 }
1458
1459 // Methods to support isa and dyn_cast
1460 static bool classof(const SDNode *N) {
1461 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1462 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1463 N->getOpcode() == ISD::ATOMIC_SWAP ||
1464 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1465 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1466 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1467 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1468 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1469 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1470 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1471 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1472 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1473 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1474 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1475 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1476 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1477 N->getOpcode() == ISD::ATOMIC_LOAD_FMAX ||
1478 N->getOpcode() == ISD::ATOMIC_LOAD_FMIN ||
1479 N->getOpcode() == ISD::ATOMIC_LOAD ||
1480 N->getOpcode() == ISD::ATOMIC_STORE;
1481 }
1482};
1483
1484/// This SDNode is used for target intrinsics that touch
1485/// memory and need an associated MachineMemOperand. Its opcode may be
1486/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1487/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1488class MemIntrinsicSDNode : public MemSDNode {
1489public:
1490 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1491 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1492 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1493 SDNodeBits.IsMemIntrinsic = true;
1494 }
1495
1496 // Methods to support isa and dyn_cast
1497 static bool classof(const SDNode *N) {
1498 // We lower some target intrinsics to their target opcode
1499 // early a node with a target opcode can be of this class
1500 return N->isMemIntrinsic() ||
1501 N->getOpcode() == ISD::PREFETCH ||
1502 N->isTargetMemoryOpcode();
1503 }
1504};
1505
1506/// This SDNode is used to implement the code generator
1507/// support for the llvm IR shufflevector instruction. It combines elements
1508/// from two input vectors into a new input vector, with the selection and
1509/// ordering of elements determined by an array of integers, referred to as
1510/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1511/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1512/// An index of -1 is treated as undef, such that the code generator may put
1513/// any value in the corresponding element of the result.
1514class ShuffleVectorSDNode : public SDNode {
1515 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1516 // is freed when the SelectionDAG object is destroyed.
1517 const int *Mask;
1518
1519protected:
1520 friend class SelectionDAG;
1521
1522 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1523 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1524
1525public:
1526 ArrayRef<int> getMask() const {
1527 EVT VT = getValueType(0);
1528 return makeArrayRef(Mask, VT.getVectorNumElements());
1529 }
1530
1531 int getMaskElt(unsigned Idx) const {
1532 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!");
1533 return Mask[Idx];
1534 }
1535
1536 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1537
1538 int getSplatIndex() const {
1539 assert(isSplat() && "Cannot get splat index for non-splat!");
1540 EVT VT = getValueType(0);
1541 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1542 if (Mask[i] >= 0)
1543 return Mask[i];
1544
1545 // We can choose any index value here and be correct because all elements
1546 // are undefined. Return 0 for better potential for callers to simplify.
1547 return 0;
1548 }
1549
1550 static bool isSplatMask(const int *Mask, EVT VT);
1551
1552 /// Change values in a shuffle permute mask assuming
1553 /// the two vector operands have swapped position.
1554 static void commuteMask(MutableArrayRef<int> Mask) {
1555 unsigned NumElems = Mask.size();
1556 for (unsigned i = 0; i != NumElems; ++i) {
1557 int idx = Mask[i];
1558 if (idx < 0)
1559 continue;
1560 else if (idx < (int)NumElems)
1561 Mask[i] = idx + NumElems;
1562 else
1563 Mask[i] = idx - NumElems;
1564 }
1565 }
1566
1567 static bool classof(const SDNode *N) {
1568 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1569 }
1570};
1571
1572class ConstantSDNode : public SDNode {
1573 friend class SelectionDAG;
1574
1575 const ConstantInt *Value;
1576
1577 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1578 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1579 getSDVTList(VT)),
1580 Value(val) {
1581 ConstantSDNodeBits.IsOpaque = isOpaque;
1582 }
1583
1584public:
1585 const ConstantInt *getConstantIntValue() const { return Value; }
1586 const APInt &getAPIntValue() const { return Value->getValue(); }
1587 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1588 int64_t getSExtValue() const { return Value->getSExtValue(); }
1589 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) {
1590 return Value->getLimitedValue(Limit);
1591 }
1592 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1593 Align getAlignValue() const { return Value->getAlignValue(); }
1594
1595 bool isOne() const { return Value->isOne(); }
1596 bool isZero() const { return Value->isZero(); }
1597 // NOTE: This is soft-deprecated. Please use `isZero()` instead.
1598 bool isNullValue() const { return isZero(); }
1599 bool isAllOnes() const { return Value->isMinusOne(); }
1600 // NOTE: This is soft-deprecated. Please use `isAllOnes()` instead.
1601 bool isAllOnesValue() const { return isAllOnes(); }
1602 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1603 bool isMinSignedValue() const { return Value->isMinValue(true); }
1604
1605 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1606
1607 static bool classof(const SDNode *N) {
1608 return N->getOpcode() == ISD::Constant ||
1609 N->getOpcode() == ISD::TargetConstant;
1610 }
1611};
1612
1613uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1614 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1615}
1616
1617const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1618 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1619}
1620
1621class ConstantFPSDNode : public SDNode {
1622 friend class SelectionDAG;
1623
1624 const ConstantFP *Value;
1625
1626 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1627 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1628 DebugLoc(), getSDVTList(VT)),
1629 Value(val) {}
1630
1631public:
1632 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1633 const ConstantFP *getConstantFPValue() const { return Value; }
1634
1635 /// Return true if the value is positive or negative zero.
1636 bool isZero() const { return Value->isZero(); }
1637
1638 /// Return true if the value is a NaN.
1639 bool isNaN() const { return Value->isNaN(); }
1640
1641 /// Return true if the value is an infinity
1642 bool isInfinity() const { return Value->isInfinity(); }
1643
1644 /// Return true if the value is negative.
1645 bool isNegative() const { return Value->isNegative(); }
1646
1647 /// We don't rely on operator== working on double values, as
1648 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1649 /// As such, this method can be used to do an exact bit-for-bit comparison of
1650 /// two floating point values.
1651
1652 /// We leave the version with the double argument here because it's just so
1653 /// convenient to write "2.0" and the like. Without this function we'd
1654 /// have to duplicate its logic everywhere it's called.
1655 bool isExactlyValue(double V) const {
1656 return Value->getValueAPF().isExactlyValue(V);
1657 }
1658 bool isExactlyValue(const APFloat& V) const;
1659
1660 static bool isValueValidForType(EVT VT, const APFloat& Val);
1661
1662 static bool classof(const SDNode *N) {
1663 return N->getOpcode() == ISD::ConstantFP ||
1664 N->getOpcode() == ISD::TargetConstantFP;
1665 }
1666};
1667
1668/// Returns true if \p V is a constant integer zero.
1669bool isNullConstant(SDValue V);
1670
1671/// Returns true if \p V is an FP constant with a value of positive zero.
1672bool isNullFPConstant(SDValue V);
1673
1674/// Returns true if \p V is an integer constant with all bits set.
1675bool isAllOnesConstant(SDValue V);
1676
1677/// Returns true if \p V is a constant integer one.
1678bool isOneConstant(SDValue V);
1679
1680/// Returns true if \p V is a constant min signed integer value.
1681bool isMinSignedConstant(SDValue V);
1682
1683/// Return the non-bitcasted source operand of \p V if it exists.
1684/// If \p V is not a bitcasted value, it is returned as-is.
1685SDValue peekThroughBitcasts(SDValue V);
1686
1687/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1688/// If \p V is not a bitcasted one-use value, it is returned as-is.
1689SDValue peekThroughOneUseBitcasts(SDValue V);
1690
1691/// Return the non-extracted vector source operand of \p V if it exists.
1692/// If \p V is not an extracted subvector, it is returned as-is.
1693SDValue peekThroughExtractSubvectors(SDValue V);
1694
1695/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1696/// constant is canonicalized to be operand 1.
1697bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1698
1699/// If \p V is a bitwise not, returns the inverted operand. Otherwise returns
1700/// an empty SDValue. Only bits set in \p Mask are required to be inverted,
1701/// other bits may be arbitrary.
1702SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs);
1703
1704/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1705ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1706 bool AllowTruncation = false);
1707
1708/// Returns the SDNode if it is a demanded constant splat BuildVector or
1709/// constant int.
1710ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1711 bool AllowUndefs = false,
1712 bool AllowTruncation = false);
1713
1714/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1715ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1716
1717/// Returns the SDNode if it is a demanded constant splat BuildVector or
1718/// constant float.
1719ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1720 bool AllowUndefs = false);
1721
1722/// Return true if the value is a constant 0 integer or a splatted vector of
1723/// a constant 0 integer (with no undefs by default).
1724/// Build vector implicit truncation is not an issue for null values.
1725bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1726
1727/// Return true if the value is a constant 1 integer or a splatted vector of a
1728/// constant 1 integer (with no undefs).
1729/// Does not permit build vector implicit truncation.
1730bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1731
1732/// Return true if the value is a constant -1 integer or a splatted vector of a
1733/// constant -1 integer (with no undefs).
1734/// Does not permit build vector implicit truncation.
1735bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1736
1737/// Return true if \p V is either a integer or FP constant.
1738inline bool isIntOrFPConstant(SDValue V) {
1739 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1740}
1741
1742class GlobalAddressSDNode : public SDNode {
1743 friend class SelectionDAG;
1744
1745 const GlobalValue *TheGlobal;
1746 int64_t Offset;
1747 unsigned TargetFlags;
1748
1749 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1750 const GlobalValue *GA, EVT VT, int64_t o,
1751 unsigned TF);
1752
1753public:
1754 const GlobalValue *getGlobal() const { return TheGlobal; }
1755 int64_t getOffset() const { return Offset; }
1756 unsigned getTargetFlags() const { return TargetFlags; }
1757 // Return the address space this GlobalAddress belongs to.
1758 unsigned getAddressSpace() const;
1759
1760 static bool classof(const SDNode *N) {
1761 return N->getOpcode() == ISD::GlobalAddress ||
1762 N->getOpcode() == ISD::TargetGlobalAddress ||
1763 N->getOpcode() == ISD::GlobalTLSAddress ||
1764 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1765 }
1766};
1767
1768class FrameIndexSDNode : public SDNode {
1769 friend class SelectionDAG;
1770
1771 int FI;
1772
1773 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1774 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1775 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1776 }
1777
1778public:
1779 int getIndex() const { return FI; }
1780
1781 static bool classof(const SDNode *N) {
1782 return N->getOpcode() == ISD::FrameIndex ||
1783 N->getOpcode() == ISD::TargetFrameIndex;
1784 }
1785};
1786
1787/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1788/// the offet and size that are started/ended in the underlying FrameIndex.
1789class LifetimeSDNode : public SDNode {
1790 friend class SelectionDAG;
1791 int64_t Size;
1792 int64_t Offset; // -1 if offset is unknown.
1793
1794 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1795 SDVTList VTs, int64_t Size, int64_t Offset)
1796 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1797public:
1798 int64_t getFrameIndex() const {
1799 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1800 }
1801
1802 bool hasOffset() const { return Offset >= 0; }
1803 int64_t getOffset() const {
1804 assert(hasOffset() && "offset is unknown");
1805 return Offset;
1806 }
1807 int64_t getSize() const {
1808 assert(hasOffset() && "offset is unknown");
1809 return Size;
1810 }
1811
1812 // Methods to support isa and dyn_cast
1813 static bool classof(const SDNode *N) {
1814 return N->getOpcode() == ISD::LIFETIME_START ||
1815 N->getOpcode() == ISD::LIFETIME_END;
1816 }
1817};
1818
1819/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1820/// the index of the basic block being probed. A pseudo probe serves as a place
1821/// holder and will be removed at the end of compilation. It does not have any
1822/// operand because we do not want the instruction selection to deal with any.
1823class PseudoProbeSDNode : public SDNode {
1824 friend class SelectionDAG;
1825 uint64_t Guid;
1826 uint64_t Index;
1827 uint32_t Attributes;
1828
1829 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1830 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1831 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1832 Attributes(Attr) {}
1833
1834public:
1835 uint64_t getGuid() const { return Guid; }
1836 uint64_t getIndex() const { return Index; }
1837 uint32_t getAttributes() const { return Attributes; }
1838
1839 // Methods to support isa and dyn_cast
1840 static bool classof(const SDNode *N) {
1841 return N->getOpcode() == ISD::PSEUDO_PROBE;
1842 }
1843};
1844
1845class JumpTableSDNode : public SDNode {
1846 friend class SelectionDAG;
1847
1848 int JTI;
1849 unsigned TargetFlags;
1850
1851 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1852 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1853 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1854 }
1855
1856public:
1857 int getIndex() const { return JTI; }
1858 unsigned getTargetFlags() const { return TargetFlags; }
1859
1860 static bool classof(const SDNode *N) {
1861 return N->getOpcode() == ISD::JumpTable ||
1862 N->getOpcode() == ISD::TargetJumpTable;
1863 }
1864};
1865
1866class ConstantPoolSDNode : public SDNode {
1867 friend class SelectionDAG;
1868
1869 union {
1870 const Constant *ConstVal;
1871 MachineConstantPoolValue *MachineCPVal;
1872 } Val;
1873 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1874 Align Alignment; // Minimum alignment requirement of CP.
1875 unsigned TargetFlags;
1876
1877 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1878 Align Alignment, unsigned TF)
1879 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1880 DebugLoc(), getSDVTList(VT)),
1881 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1882 assert(Offset >= 0 && "Offset is too large");
1883 Val.ConstVal = c;
1884 }
1885
1886 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1887 Align Alignment, unsigned TF)
1888 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1889 DebugLoc(), getSDVTList(VT)),
1890 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1891 assert(Offset >= 0 && "Offset is too large");
1892 Val.MachineCPVal = v;
1893 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1);
1894 }
1895
1896public:
1897 bool isMachineConstantPoolEntry() const {
1898 return Offset < 0;
1899 }
1900
1901 const Constant *getConstVal() const {
1902 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type");
1903 return Val.ConstVal;
1904 }
1905
1906 MachineConstantPoolValue *getMachineCPVal() const {
1907 assert(isMachineConstantPoolEntry() && "Wrong constantpool type");
1908 return Val.MachineCPVal;
1909 }
1910
1911 int getOffset() const {
1912 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT-1));
1913 }
1914
1915 // Return the alignment of this constant pool object, which is either 0 (for
1916 // default alignment) or the desired value.
1917 Align getAlign() const { return Alignment; }
1918 unsigned getTargetFlags() const { return TargetFlags; }
1919
1920 Type *getType() const;
1921
1922 static bool classof(const SDNode *N) {
1923 return N->getOpcode() == ISD::ConstantPool ||
1924 N->getOpcode() == ISD::TargetConstantPool;
1925 }
1926};
1927
1928/// Completely target-dependent object reference.
1929class TargetIndexSDNode : public SDNode {
1930 friend class SelectionDAG;
1931
1932 unsigned TargetFlags;
1933 int Index;
1934 int64_t Offset;
1935
1936public:
1937 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1938 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1939 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1940
1941 unsigned getTargetFlags() const { return TargetFlags; }
1942 int getIndex() const { return Index; }
1943 int64_t getOffset() const { return Offset; }
1944
1945 static bool classof(const SDNode *N) {
1946 return N->getOpcode() == ISD::TargetIndex;
1947 }
1948};
1949
1950class BasicBlockSDNode : public SDNode {
1951 friend class SelectionDAG;
1952
1953 MachineBasicBlock *MBB;
1954
1955 /// Debug info is meaningful and potentially useful here, but we create
1956 /// blocks out of order when they're jumped to, which makes it a bit
1957 /// harder. Let's see if we need it first.
1958 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1959 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1960 {}
1961
1962public:
1963 MachineBasicBlock *getBasicBlock() const { return MBB; }
1964
1965 static bool classof(const SDNode *N) {
1966 return N->getOpcode() == ISD::BasicBlock;
1967 }
1968};
1969
1970/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1971class BuildVectorSDNode : public SDNode {
1972public:
1973 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1974 explicit BuildVectorSDNode() = delete;
1975
1976 /// Check if this is a constant splat, and if so, find the
1977 /// smallest element size that splats the vector. If MinSplatBits is
1978 /// nonzero, the element size must be at least that large. Note that the
1979 /// splat element may be the entire vector (i.e., a one element vector).
1980 /// Returns the splat element value in SplatValue. Any undefined bits in
1981 /// that value are zero, and the corresponding bits in the SplatUndef mask
1982 /// are set. The SplatBitSize value is set to the splat element size in
1983 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1984 /// undefined. isBigEndian describes the endianness of the target.
1985 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1986 unsigned &SplatBitSize, bool &HasAnyUndefs,
1987 unsigned MinSplatBits = 0,
1988 bool isBigEndian = false) const;
1989
1990 /// Returns the demanded splatted value or a null value if this is not a
1991 /// splat.
1992 ///
1993 /// The DemandedElts mask indicates the elements that must be in the splat.
1994 /// If passed a non-null UndefElements bitvector, it will resize it to match
1995 /// the vector width and set the bits where elements are undef.
1996 SDValue getSplatValue(const APInt &DemandedElts,
1997 BitVector *UndefElements = nullptr) const;
1998
1999 /// Returns the splatted value or a null value if this is not a splat.
2000 ///
2001 /// If passed a non-null UndefElements bitvector, it will resize it to match
2002 /// the vector width and set the bits where elements are undef.
2003 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
2004
2005 /// Find the shortest repeating sequence of values in the build vector.
2006 ///
2007 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
2008 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
2009 ///
2010 /// Currently this must be a power-of-2 build vector.
2011 /// The DemandedElts mask indicates the elements that must be present,
2012 /// undemanded elements in Sequence may be null (SDValue()). If passed a
2013 /// non-null UndefElements bitvector, it will resize it to match the original
2014 /// vector width and set the bits where elements are undef. If result is
2015 /// false, Sequence will be empty.
2016 bool getRepeatedSequence(const APInt &DemandedElts,
2017 SmallVectorImpl<SDValue> &Sequence,
2018 BitVector *UndefElements = nullptr) const;
2019
2020 /// Find the shortest repeating sequence of values in the build vector.
2021 ///
2022 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
2023 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
2024 ///
2025 /// Currently this must be a power-of-2 build vector.
2026 /// If passed a non-null UndefElements bitvector, it will resize it to match
2027 /// the original vector width and set the bits where elements are undef.
2028 /// If result is false, Sequence will be empty.
2029 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
2030 BitVector *UndefElements = nullptr) const;
2031
2032 /// Returns the demanded splatted constant or null if this is not a constant
2033 /// splat.
2034 ///
2035 /// The DemandedElts mask indicates the elements that must be in the splat.
2036 /// If passed a non-null UndefElements bitvector, it will resize it to match
2037 /// the vector width and set the bits where elements are undef.
2038 ConstantSDNode *
2039 getConstantSplatNode(const APInt &DemandedElts,
2040 BitVector *UndefElements = nullptr) const;
2041
2042 /// Returns the splatted constant or null if this is not a constant
2043 /// splat.
2044 ///
2045 /// If passed a non-null UndefElements bitvector, it will resize it to match
2046 /// the vector width and set the bits where elements are undef.
2047 ConstantSDNode *
2048 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2049
2050 /// Returns the demanded splatted constant FP or null if this is not a
2051 /// constant FP splat.
2052 ///
2053 /// The DemandedElts mask indicates the elements that must be in the splat.
2054 /// If passed a non-null UndefElements bitvector, it will resize it to match
2055 /// the vector width and set the bits where elements are undef.
2056 ConstantFPSDNode *
2057 getConstantFPSplatNode(const APInt &DemandedElts,
2058 BitVector *UndefElements = nullptr) const;
2059
2060 /// Returns the splatted constant FP or null if this is not a constant
2061 /// FP splat.
2062 ///
2063 /// If passed a non-null UndefElements bitvector, it will resize it to match
2064 /// the vector width and set the bits where elements are undef.
2065 ConstantFPSDNode *
2066 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2067
2068 /// If this is a constant FP splat and the splatted constant FP is an
2069 /// exact power or 2, return the log base 2 integer value. Otherwise,
2070 /// return -1.
2071 ///
2072 /// The BitWidth specifies the necessary bit precision.
2073 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2074 uint32_t BitWidth) const;
2075
2076 /// Extract the raw bit data from a build vector of Undef, Constant or
2077 /// ConstantFP node elements. Each raw bit element will be \p
2078 /// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
2079 /// undefined elements are flagged in \p UndefElements.
2080 bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
2081 SmallVectorImpl<APInt> &RawBitElements,
2082 BitVector &UndefElements) const;
2083
2084 bool isConstant() const;
2085
2086 /// If this BuildVector is constant and represents the numerical series
2087 /// "<a, a+n, a+2n, a+3n, ...>" where a is integer and n is a non-zero integer,
2088 /// the value "<a,n>" is returned.
2089 Optional<std::pair<APInt, APInt>> isConstantSequence() const;
2090
2091 /// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements.
2092 /// Undef elements are treated as zero, and entirely undefined elements are
2093 /// flagged in \p DstUndefElements.
2094 static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
2095 SmallVectorImpl<APInt> &DstBitElements,
2096 ArrayRef<APInt> SrcBitElements,
2097 BitVector &DstUndefElements,
2098 const BitVector &SrcUndefElements);
2099
2100 static bool classof(const SDNode *N) {
2101 return N->getOpcode() == ISD::BUILD_VECTOR;
2102 }
2103};
2104
2105/// An SDNode that holds an arbitrary LLVM IR Value. This is
2106/// used when the SelectionDAG needs to make a simple reference to something
2107/// in the LLVM IR representation.
2108///
2109class SrcValueSDNode : public SDNode {
2110 friend class SelectionDAG;
2111
2112 const Value *V;
2113
2114 /// Create a SrcValue for a general value.
2115 explicit SrcValueSDNode(const Value *v)
2116 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2117
2118public:
2119 /// Return the contained Value.
2120 const Value *getValue() const { return V; }
2121
2122 static bool classof(const SDNode *N) {
2123 return N->getOpcode() == ISD::SRCVALUE;
2124 }
2125};
2126
2127class MDNodeSDNode : public SDNode {
2128 friend class SelectionDAG;
2129
2130 const MDNode *MD;
2131
2132 explicit MDNodeSDNode(const MDNode *md)
2133 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2134 {}
2135
2136public:
2137 const MDNode *getMD() const { return MD; }
2138
2139 static bool classof(const SDNode *N) {
2140 return N->getOpcode() == ISD::MDNODE_SDNODE;
2141 }
2142};
2143
2144class RegisterSDNode : public SDNode {
2145 friend class SelectionDAG;
2146
2147 Register Reg;
2148
2149 RegisterSDNode(Register reg, EVT VT)
2150 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2151
2152public:
2153 Register getReg() const { return Reg; }
2154
2155 static bool classof(const SDNode *N) {
2156 return N->getOpcode() == ISD::Register;
2157 }
2158};
2159
2160class RegisterMaskSDNode : public SDNode {
2161 friend class SelectionDAG;
2162
2163 // The memory for RegMask is not owned by the node.
2164 const uint32_t *RegMask;
2165
2166 RegisterMaskSDNode(const uint32_t *mask)
2167 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2168 RegMask(mask) {}
2169
2170public:
2171 const uint32_t *getRegMask() const { return RegMask; }
2172
2173 static bool classof(const SDNode *N) {
2174 return N->getOpcode() == ISD::RegisterMask;
2175 }
2176};
2177
2178class BlockAddressSDNode : public SDNode {
2179 friend class SelectionDAG;
2180
2181 const BlockAddress *BA;
2182 int64_t Offset;
2183 unsigned TargetFlags;
2184
2185 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2186 int64_t o, unsigned Flags)
2187 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2188 BA(ba), Offset(o), TargetFlags(Flags) {}
2189
2190public:
2191 const BlockAddress *getBlockAddress() const { return BA; }
2192 int64_t getOffset() const { return Offset; }
2193 unsigned getTargetFlags() const { return TargetFlags; }
2194
2195 static bool classof(const SDNode *N) {
2196 return N->getOpcode() == ISD::BlockAddress ||
2197 N->getOpcode() == ISD::TargetBlockAddress;
2198 }
2199};
2200
2201class LabelSDNode : public SDNode {
2202 friend class SelectionDAG;
2203
2204 MCSymbol *Label;
2205
2206 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2207 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2208 assert(LabelSDNode::classof(this) && "not a label opcode");
2209 }
2210
2211public:
2212 MCSymbol *getLabel() const { return Label; }
2213
2214 static bool classof(const SDNode *N) {
2215 return N->getOpcode() == ISD::EH_LABEL ||
2216 N->getOpcode() == ISD::ANNOTATION_LABEL;
2217 }
2218};
2219
2220class ExternalSymbolSDNode : public SDNode {
2221 friend class SelectionDAG;
2222
2223 const char *Symbol;
2224 unsigned TargetFlags;
2225
2226 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2227 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2228 DebugLoc(), getSDVTList(VT)),
2229 Symbol(Sym), TargetFlags(TF) {}
2230
2231public:
2232 const char *getSymbol() const { return Symbol; }
2233 unsigned getTargetFlags() const { return TargetFlags; }
2234
2235 static bool classof(const SDNode *N) {
2236 return N->getOpcode() == ISD::ExternalSymbol ||
2237 N->getOpcode() == ISD::TargetExternalSymbol;
2238 }
2239};
2240
2241class MCSymbolSDNode : public SDNode {
2242 friend class SelectionDAG;
2243
2244 MCSymbol *Symbol;
2245
2246 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2247 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2248
2249public:
2250 MCSymbol *getMCSymbol() const { return Symbol; }
2251
2252 static bool classof(const SDNode *N) {
2253 return N->getOpcode() == ISD::MCSymbol;
2254 }
2255};
2256
2257class CondCodeSDNode : public SDNode {
2258 friend class SelectionDAG;
2259
2260 ISD::CondCode Condition;
2261
2262 explicit CondCodeSDNode(ISD::CondCode Cond)
2263 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2264 Condition(Cond) {}
2265
2266public:
2267 ISD::CondCode get() const { return Condition; }
2268
2269 static bool classof(const SDNode *N) {
2270 return N->getOpcode() == ISD::CONDCODE;
2271 }
2272};
2273
2274/// This class is used to represent EVT's, which are used
2275/// to parameterize some operations.
2276class VTSDNode : public SDNode {
2277 friend class SelectionDAG;
2278
2279 EVT ValueType;
2280
2281 explicit VTSDNode(EVT VT)
2282 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2283 ValueType(VT) {}
2284
2285public:
2286 EVT getVT() const { return ValueType; }
2287
2288 static bool classof(const SDNode *N) {
2289 return N->getOpcode() == ISD::VALUETYPE;
2290 }
2291};
2292
2293/// Base class for LoadSDNode and StoreSDNode
2294class LSBaseSDNode : public MemSDNode {
2295public:
2296 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2297 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2298 MachineMemOperand *MMO)
2299 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2300 LSBaseSDNodeBits.AddressingMode = AM;
2301 assert(getAddressingMode() == AM && "Value truncated");
2302 }
2303
2304 const SDValue &getOffset() const {
2305 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2306 }
2307
2308 /// Return the addressing mode for this load or store:
2309 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2310 ISD::MemIndexedMode getAddressingMode() const {
2311 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2312 }
2313
2314 /// Return true if this is a pre/post inc/dec load/store.
2315 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2316
2317 /// Return true if this is NOT a pre/post inc/dec load/store.
2318 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2319
2320 static bool classof(const SDNode *N) {
2321 return N->getOpcode() == ISD::LOAD ||
2322 N->getOpcode() == ISD::STORE;
2323 }
2324};
2325
2326/// This class is used to represent ISD::LOAD nodes.
2327class LoadSDNode : public LSBaseSDNode {
2328 friend class SelectionDAG;
2329
2330 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2331 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2332 MachineMemOperand *MMO)
2333 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2334 LoadSDNodeBits.ExtTy = ETy;
2335 assert(readMem() && "Load MachineMemOperand is not a load!");
2336 assert(!writeMem() && "Load MachineMemOperand is a store!");
2337 }
2338
2339public:
2340 /// Return whether this is a plain node,
2341 /// or one of the varieties of value-extending loads.
2342 ISD::LoadExtType getExtensionType() const {
2343 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2344 }
2345
2346 const SDValue &getBasePtr() const { return getOperand(1); }
2347 const SDValue &getOffset() const { return getOperand(2); }
2348
2349 static bool classof(const SDNode *N) {
2350 return N->getOpcode() == ISD::LOAD;
2351 }
2352};
2353
2354/// This class is used to represent ISD::STORE nodes.
2355class StoreSDNode : public LSBaseSDNode {
2356 friend class SelectionDAG;
2357
2358 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2359 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2360 MachineMemOperand *MMO)
2361 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2362 StoreSDNodeBits.IsTruncating = isTrunc;
2363 assert(!readMem() && "Store MachineMemOperand is a load!");
2364 assert(writeMem() && "Store MachineMemOperand is not a store!");
2365 }
2366
2367public:
2368 /// Return true if the op does a truncation before store.
2369 /// For integers this is the same as doing a TRUNCATE and storing the result.
2370 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2371 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2372 void setTruncatingStore(bool Truncating) {
2373 StoreSDNodeBits.IsTruncating = Truncating;
2374 }
2375
2376 const SDValue &getValue() const { return getOperand(1); }
2377 const SDValue &getBasePtr() const { return getOperand(2); }
2378 const SDValue &getOffset() const { return getOperand(3); }
2379
2380 static bool classof(const SDNode *N) {
2381 return N->getOpcode() == ISD::STORE;
2382 }
2383};
2384
2385/// This base class is used to represent VP_LOAD, VP_STORE,
2386/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes
2387class VPBaseLoadStoreSDNode : public MemSDNode {
2388public:
2389 friend class SelectionDAG;
2390
2391 VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2392 const DebugLoc &DL, SDVTList VTs,
2393 ISD::MemIndexedMode AM, EVT MemVT,
2394 MachineMemOperand *MMO)
2395 : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) {
2396 LSBaseSDNodeBits.AddressingMode = AM;
2397 assert(getAddressingMode() == AM && "Value truncated");
2398 }
2399
2400 // VPStridedStoreSDNode (Chain, Data, Ptr, Offset, Stride, Mask, EVL)
2401 // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
2402 // VPStridedLoadSDNode (Chain, Ptr, Offset, Stride, Mask, EVL)
2403 // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
2404 // Mask is a vector of i1 elements;
2405 // the type of EVL is TLI.getVPExplicitVectorLengthTy().
2406 const SDValue &getOffset() const {
2407 return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
2408 getOpcode() == ISD::VP_LOAD)
2409 ? 2
2410 : 3);
2411 }
2412 const SDValue &getBasePtr() const {
2413 return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
2414 getOpcode() == ISD::VP_LOAD)
2415 ? 1
2416 : 2);
2417 }
2418 const SDValue &getMask() const {
2419 switch (getOpcode()) {
2420 default:
2421 llvm_unreachable("Invalid opcode");
2422 case ISD::VP_LOAD:
2423 return getOperand(3);
2424 case ISD::VP_STORE:
2425 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2426 return getOperand(4);
2427 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2428 return getOperand(5);
2429 }
2430 }
2431 const SDValue &getVectorLength() const {
2432 switch (getOpcode()) {
2433 default:
2434 llvm_unreachable("Invalid opcode");
2435 case ISD::VP_LOAD:
2436 return getOperand(4);
2437 case ISD::VP_STORE:
2438 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2439 return getOperand(5);
2440 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2441 return getOperand(6);
2442 }
2443 }
2444
2445 /// Return the addressing mode for this load or store:
2446 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2447 ISD::MemIndexedMode getAddressingMode() const {
2448 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2449 }
2450
2451 /// Return true if this is a pre/post inc/dec load/store.
2452 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2453
2454 /// Return true if this is NOT a pre/post inc/dec load/store.
2455 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2456
2457 static bool classof(const SDNode *N) {
2458 return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
2459 N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE ||
2460 N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
2461 }
2462};
2463
2464/// This class is used to represent a VP_LOAD node
2465class VPLoadSDNode : public VPBaseLoadStoreSDNode {
2466public:
2467 friend class SelectionDAG;
2468
2469 VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2470 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
2471 EVT MemVT, MachineMemOperand *MMO)
2472 : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2473 LoadSDNodeBits.ExtTy = ETy;
2474 LoadSDNodeBits.IsExpanding = isExpanding;
2475 }
2476
2477 ISD::LoadExtType getExtensionType() const {
2478 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2479 }
2480
2481 const SDValue &getBasePtr() const { return getOperand(1); }
2482 const SDValue &getOffset() const { return getOperand(2); }
2483 const SDValue &getMask() const { return getOperand(3); }
2484 const SDValue &getVectorLength() const { return getOperand(4); }
2485
2486 static bool classof(const SDNode *N) {
2487 return N->getOpcode() == ISD::VP_LOAD;
2488 }
2489 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2490};
2491
2492/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
2493class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode {
2494public:
2495 friend class SelectionDAG;
2496
2497 VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
2498 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2499 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2500 : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs,
2501 AM, MemVT, MMO) {
2502 LoadSDNodeBits.ExtTy = ETy;
2503 LoadSDNodeBits.IsExpanding = IsExpanding;
2504 }
2505
2506 ISD::LoadExtType getExtensionType() const {
2507 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2508 }
2509
2510 const SDValue &getBasePtr() const { return getOperand(1); }
2511 const SDValue &getOffset() const { return getOperand(2); }
2512 const SDValue &getStride() const { return getOperand(3); }
2513 const SDValue &getMask() const { return getOperand(4); }
2514 const SDValue &getVectorLength() const { return getOperand(5); }
2515
2516 static bool classof(const SDNode *N) {
2517 return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD;
2518 }
2519 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2520};
2521
2522/// This class is used to represent a VP_STORE node
2523class VPStoreSDNode : public VPBaseLoadStoreSDNode {
2524public:
2525 friend class SelectionDAG;
2526
2527 VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2528 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2529 EVT MemVT, MachineMemOperand *MMO)
2530 : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
2531 StoreSDNodeBits.IsTruncating = isTrunc;
2532 StoreSDNodeBits.IsCompressing = isCompressing;
2533 }
2534
2535 /// Return true if this is a truncating store.
2536 /// For integers this is the same as doing a TRUNCATE and storing the result.
2537 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2538 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2539
2540 /// Returns true if the op does a compression to the vector before storing.
2541 /// The node contiguously stores the active elements (integers or floats)
2542 /// in src (those with their respective bit set in writemask k) to unaligned
2543 /// memory at base_addr.
2544 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2545
2546 const SDValue &getValue() const { return getOperand(1); }
2547 const SDValue &getBasePtr() const { return getOperand(2); }
2548 const SDValue &getOffset() const { return getOperand(3); }
2549 const SDValue &getMask() const { return getOperand(4); }
2550 const SDValue &getVectorLength() const { return getOperand(5); }
2551
2552 static bool classof(const SDNode *N) {
2553 return N->getOpcode() == ISD::VP_STORE;
2554 }
2555};
2556
2557/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
2558class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode {
2559public:
2560 friend class SelectionDAG;
2561
2562 VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
2563 ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing,
2564 EVT MemVT, MachineMemOperand *MMO)
2565 : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL,
2566 VTs, AM, MemVT, MMO) {
2567 StoreSDNodeBits.IsTruncating = IsTrunc;
2568 StoreSDNodeBits.IsCompressing = IsCompressing;
2569 }
2570
2571 /// Return true if this is a truncating store.
2572 /// For integers this is the same as doing a TRUNCATE and storing the result.
2573 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2574 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2575
2576 /// Returns true if the op does a compression to the vector before storing.
2577 /// The node contiguously stores the active elements (integers or floats)
2578 /// in src (those with their respective bit set in writemask k) to unaligned
2579 /// memory at base_addr.
2580 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2581
2582 const SDValue &getValue() const { return getOperand(1); }
2583 const SDValue &getBasePtr() const { return getOperand(2); }
2584 const SDValue &getOffset() const { return getOperand(3); }
2585 const SDValue &getStride() const { return getOperand(4); }
2586 const SDValue &getMask() const { return getOperand(5); }
2587 const SDValue &getVectorLength() const { return getOperand(6); }
2588
2589 static bool classof(const SDNode *N) {
2590 return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE;
2591 }
2592};
2593
2594/// This base class is used to represent MLOAD and MSTORE nodes
2595class MaskedLoadStoreSDNode : public MemSDNode {
2596public:
2597 friend class SelectionDAG;
2598
2599 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2600 const DebugLoc &dl, SDVTList VTs,
2601 ISD::MemIndexedMode AM, EVT MemVT,
2602 MachineMemOperand *MMO)
2603 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2604 LSBaseSDNodeBits.AddressingMode = AM;
2605 assert(getAddressingMode() == AM && "Value truncated");
2606 }
2607
2608 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2609 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2610 // Mask is a vector of i1 elements
2611 const SDValue &getOffset() const {
2612 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2613 }
2614 const SDValue &getMask() const {
2615 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2616 }
2617
2618 /// Return the addressing mode for this load or store:
2619 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2620 ISD::MemIndexedMode getAddressingMode() const {
2621 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2622 }
2623
2624 /// Return true if this is a pre/post inc/dec load/store.
2625 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2626
2627 /// Return true if this is NOT a pre/post inc/dec load/store.
2628 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2629
2630 static bool classof(const SDNode *N) {
2631 return N->getOpcode() == ISD::MLOAD ||
2632 N->getOpcode() == ISD::MSTORE;
2633 }
2634};
2635
2636/// This class is used to represent an MLOAD node
2637class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2638public:
2639 friend class SelectionDAG;
2640
2641 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2642 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2643 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2644 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2645 LoadSDNodeBits.ExtTy = ETy;
2646 LoadSDNodeBits.IsExpanding = IsExpanding;
2647 }
2648
2649 ISD::LoadExtType getExtensionType() const {
2650 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2651 }
2652
2653 const SDValue &getBasePtr() const { return getOperand(1); }
2654 const SDValue &getOffset() const { return getOperand(2); }
2655 const SDValue &getMask() const { return getOperand(3); }
2656 const SDValue &getPassThru() const { return getOperand(4); }
2657
2658 static bool classof(const SDNode *N) {
2659 return N->getOpcode() == ISD::MLOAD;
2660 }
2661
2662 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2663};
2664
2665/// This class is used to represent an MSTORE node
2666class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2667public:
2668 friend class SelectionDAG;
2669
2670 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2671 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2672 EVT MemVT, MachineMemOperand *MMO)
2673 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2674 StoreSDNodeBits.IsTruncating = isTrunc;
2675 StoreSDNodeBits.IsCompressing = isCompressing;
2676 }
2677
2678 /// Return true if the op does a truncation before store.
2679 /// For integers this is the same as doing a TRUNCATE and storing the result.
2680 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2681 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2682
2683 /// Returns true if the op does a compression to the vector before storing.
2684 /// The node contiguously stores the active elements (integers or floats)
2685 /// in src (those with their respective bit set in writemask k) to unaligned
2686 /// memory at base_addr.
2687 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2688
2689 const SDValue &getValue() const { return getOperand(1); }
2690 const SDValue &getBasePtr() const { return getOperand(2); }
2691 const SDValue &getOffset() const { return getOperand(3); }
2692 const SDValue &getMask() const { return getOperand(4); }
2693
2694 static bool classof(const SDNode *N) {
2695 return N->getOpcode() == ISD::MSTORE;
2696 }
2697};
2698
2699/// This is a base class used to represent
2700/// VP_GATHER and VP_SCATTER nodes
2701///
2702class VPGatherScatterSDNode : public MemSDNode {
2703public:
2704 friend class SelectionDAG;
2705
2706 VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2707 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2708 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2709 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2710 LSBaseSDNodeBits.AddressingMode = IndexType;
2711 assert(getIndexType() == IndexType && "Value truncated");
2712 }
2713
2714 /// How is Index applied to BasePtr when computing addresses.
2715 ISD::MemIndexType getIndexType() const {
2716 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2717 }
2718 bool isIndexScaled() const {
2719 return !cast<ConstantSDNode>(getScale())->isOne();
2720 }
2721 bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); }
2722
2723 // In the both nodes address is Op1, mask is Op2:
2724 // VPGatherSDNode (Chain, base, index, scale, mask, vlen)
2725 // VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
2726 // Mask is a vector of i1 elements
2727 const SDValue &getBasePtr() const {
2728 return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
2729 }
2730 const SDValue &getIndex() const {
2731 return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
2732 }
2733 const SDValue &getScale() const {
2734 return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
2735 }
2736 const SDValue &getMask() const {
2737 return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
2738 }
2739 const SDValue &getVectorLength() const {
2740 return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
2741 }
2742
2743 static bool classof(const SDNode *N) {
2744 return N->getOpcode() == ISD::VP_GATHER ||
2745 N->getOpcode() == ISD::VP_SCATTER;
2746 }
2747};
2748
2749/// This class is used to represent an VP_GATHER node
2750///
2751class VPGatherSDNode : public VPGatherScatterSDNode {
2752public:
2753 friend class SelectionDAG;
2754
2755 VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2756 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2757 : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
2758 IndexType) {}
2759
2760 static bool classof(const SDNode *N) {
2761 return N->getOpcode() == ISD::VP_GATHER;
2762 }
2763};
2764
2765/// This class is used to represent an VP_SCATTER node
2766///
2767class VPScatterSDNode : public VPGatherScatterSDNode {
2768public:
2769 friend class SelectionDAG;
2770
2771 VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2772 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2773 : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
2774 IndexType) {}
2775
2776 const SDValue &getValue() const { return getOperand(1); }
2777
2778 static bool classof(const SDNode *N) {
2779 return N->getOpcode() == ISD::VP_SCATTER;
2780 }
2781};
2782
2783/// This is a base class used to represent
2784/// MGATHER and MSCATTER nodes
2785///
2786class MaskedGatherScatterSDNode : public MemSDNode {
2787public:
2788 friend class SelectionDAG;
2789
2790 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2791 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2792 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2793 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2794 LSBaseSDNodeBits.AddressingMode = IndexType;
2795 assert(getIndexType() == IndexType && "Value truncated");
2796 }
2797
2798 /// How is Index applied to BasePtr when computing addresses.
2799 ISD::MemIndexType getIndexType() const {
2800 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2801 }
2802 bool isIndexScaled() const {
2803 return !cast<ConstantSDNode>(getScale())->isOne();
2804 }
2805 bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); }
2806
2807 // In the both nodes address is Op1, mask is Op2:
2808 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2809 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2810 // Mask is a vector of i1 elements
2811 const SDValue &getBasePtr() const { return getOperand(3); }
2812 const SDValue &getIndex() const { return getOperand(4); }
2813 const SDValue &getMask() const { return getOperand(2); }
2814 const SDValue &getScale() const { return getOperand(5); }
2815
2816 static bool classof(const SDNode *N) {
2817 return N->getOpcode() == ISD::MGATHER ||
2818 N->getOpcode() == ISD::MSCATTER;
2819 }
2820};
2821
2822/// This class is used to represent an MGATHER node
2823///
2824class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2825public:
2826 friend class SelectionDAG;
2827
2828 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2829 EVT MemVT, MachineMemOperand *MMO,
2830 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2831 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2832 IndexType) {
2833 LoadSDNodeBits.ExtTy = ETy;
2834 }
2835
2836 const SDValue &getPassThru() const { return getOperand(1); }
2837
2838 ISD::LoadExtType getExtensionType() const {
2839 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2840 }
2841
2842 static bool classof(const SDNode *N) {
2843 return N->getOpcode() == ISD::MGATHER;
2844 }
2845};
2846
2847/// This class is used to represent an MSCATTER node
2848///
2849class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2850public:
2851 friend class SelectionDAG;
2852
2853 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2854 EVT MemVT, MachineMemOperand *MMO,
2855 ISD::MemIndexType IndexType, bool IsTrunc)
2856 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2857 IndexType) {
2858 StoreSDNodeBits.IsTruncating = IsTrunc;
2859 }
2860
2861 /// Return true if the op does a truncation before store.
2862 /// For integers this is the same as doing a TRUNCATE and storing the result.
2863 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2864 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2865
2866 const SDValue &getValue() const { return getOperand(1); }
2867
2868 static bool classof(const SDNode *N) {
2869 return N->getOpcode() == ISD::MSCATTER;
2870 }
2871};
2872
2873/// An SDNode that represents everything that will be needed
2874/// to construct a MachineInstr. These nodes are created during the
2875/// instruction selection proper phase.
2876///
2877/// Note that the only supported way to set the `memoperands` is by calling the
2878/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2879/// inside the DAG rather than in the node.
2880class MachineSDNode : public SDNode {
2881private:
2882 friend class SelectionDAG;
2883
2884 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2885 : SDNode(Opc, Order, DL, VTs) {}
2886
2887 // We use a pointer union between a single `MachineMemOperand` pointer and
2888 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2889 // the number of these is zero, the single pointer variant used when the
2890 // number is one, and the array is used for larger numbers.
2891 //
2892 // The array is allocated via the `SelectionDAG`'s allocator and so will
2893 // always live until the DAG is cleaned up and doesn't require ownership here.
2894 //
2895 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2896 // subclasses aren't managed in a conforming C++ manner. See the comments on
2897 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2898 // constraint here is that these don't manage memory with their constructor or
2899 // destructor and can be initialized to a good state even if they start off
2900 // uninitialized.
2901 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2902
2903 // Note that this could be folded into the above `MemRefs` member if doing so
2904 // is advantageous at some point. We don't need to store this in most cases.
2905 // However, at the moment this doesn't appear to make the allocation any
2906 // smaller and makes the code somewhat simpler to read.
2907 int NumMemRefs = 0;
2908
2909public:
2910 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2911
2912 ArrayRef<MachineMemOperand *> memoperands() const {
2913 // Special case the common cases.
2914 if (NumMemRefs == 0)
2915 return {};
2916 if (NumMemRefs == 1)
2917 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2918
2919 // Otherwise we have an actual array.
2920 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2921 }
2922 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2923 mmo_iterator memoperands_end() const { return memoperands().end(); }
2924 bool memoperands_empty() const { return memoperands().empty(); }
2925
2926 /// Clear out the memory reference descriptor list.
2927 void clearMemRefs() {
2928 MemRefs = nullptr;
2929 NumMemRefs = 0;
2930 }
2931
2932 static bool classof(const SDNode *N) {
2933 return N->isMachineOpcode();
2934 }
2935};
2936
2937/// An SDNode that records if a register contains a value that is guaranteed to
2938/// be aligned accordingly.
2939class AssertAlignSDNode : public SDNode {
2940 Align Alignment;
2941
2942public:
2943 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2944 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2945
2946 Align getAlign() const { return Alignment; }
2947
2948 static bool classof(const SDNode *N) {
2949 return N->getOpcode() == ISD::AssertAlign;
2950 }
2951};
2952
2953class SDNodeIterator {
2954 const SDNode *Node;
2955 unsigned Operand;
2956
2957 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2958
2959public:
2960 using iterator_category = std::forward_iterator_tag;
2961 using value_type = SDNode;
2962 using difference_type = std::ptrdiff_t;
2963 using pointer = value_type *;
2964 using reference = value_type &;
2965
2966 bool operator==(const SDNodeIterator& x) const {
2967 return Operand == x.Operand;
2968 }
2969 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2970
2971 pointer operator*() const {
2972 return Node->getOperand(Operand).getNode();
2973 }
2974 pointer operator->() const { return operator*(); }
2975
2976 SDNodeIterator& operator++() { // Preincrement
2977 ++Operand;
2978 return *this;
2979 }
2980 SDNodeIterator operator++(int) { // Postincrement
2981 SDNodeIterator tmp = *this; ++*this; return tmp;
2982 }
2983 size_t operator-(SDNodeIterator Other) const {
2984 assert(Node == Other.Node &&
2985 "Cannot compare iterators of two different nodes!");
2986 return Operand - Other.Operand;
2987 }
2988
2989 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2990 static SDNodeIterator end (const SDNode *N) {
2991 return SDNodeIterator(N, N->getNumOperands());
2992 }
2993
2994 unsigned getOperand() const { return Operand; }
2995 const SDNode *getNode() const { return Node; }
2996};
2997
2998template <> struct GraphTraits<SDNode*> {
2999 using NodeRef = SDNode *;
3000 using ChildIteratorType = SDNodeIterator;
3001
3002 static NodeRef getEntryNode(SDNode *N) { return N; }
3003
3004 static ChildIteratorType child_begin(NodeRef N) {
3005 return SDNodeIterator::begin(N);
3006 }
3007
3008 static ChildIteratorType child_end(NodeRef N) {
3009 return SDNodeIterator::end(N);
3010 }
3011};
3012
3013/// A representation of the largest SDNode, for use in sizeof().
3014///
3015/// This needs to be a union because the largest node differs on 32 bit systems
3016/// with 4 and 8 byte pointer alignment, respectively.
3017using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
3018 BlockAddressSDNode,
3019 GlobalAddressSDNode,
3020 PseudoProbeSDNode>;
3021
3022/// The SDNode class with the greatest alignment requirement.
3023using MostAlignedSDNode = GlobalAddressSDNode;
3024
3025namespace ISD {
3026
3027 /// Returns true if the specified node is a non-extending and unindexed load.
3028 inline bool isNormalLoad(const SDNode *N) {
3029 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
3030 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
3031 Ld->getAddressingMode() == ISD::UNINDEXED;
3032 }
3033
3034 /// Returns true if the specified node is a non-extending load.
3035 inline bool isNON_EXTLoad(const SDNode *N) {
3036 return isa<LoadSDNode>(N) &&
3037 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
3038 }
3039
3040 /// Returns true if the specified node is a EXTLOAD.
3041 inline bool isEXTLoad(const SDNode *N) {
3042 return isa<LoadSDNode>(N) &&
3043 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
3044 }
3045
3046 /// Returns true if the specified node is a SEXTLOAD.
3047 inline bool isSEXTLoad(const SDNode *N) {
3048 return isa<LoadSDNode>(N) &&
3049 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
3050 }
3051
3052 /// Returns true if the specified node is a ZEXTLOAD.
3053 inline bool isZEXTLoad(const SDNode *N) {
3054 return isa<LoadSDNode>(N) &&
3055 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
3056 }
3057
3058 /// Returns true if the specified node is an unindexed load.
3059 inline bool isUNINDEXEDLoad(const SDNode *N) {
3060 return isa<LoadSDNode>(N) &&
3061 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
3062 }
3063
3064 /// Returns true if the specified node is a non-truncating
3065 /// and unindexed store.
3066 inline bool isNormalStore(const SDNode *N) {
3067 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
3068 return St && !St->isTruncatingStore() &&
3069 St->getAddressingMode() == ISD::UNINDEXED;
3070 }
3071
3072 /// Returns true if the specified node is an unindexed store.
3073 inline bool isUNINDEXEDStore(const SDNode *N) {
3074 return isa<StoreSDNode>(N) &&
3075 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
3076 }
3077
3078 /// Attempt to match a unary predicate against a scalar/splat constant or
3079 /// every element of a constant BUILD_VECTOR.
3080 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
3081 bool matchUnaryPredicate(SDValue Op,
3082 std::function<bool(ConstantSDNode *)> Match,
3083 bool AllowUndefs = false);
3084
3085 /// Attempt to match a binary predicate against a pair of scalar/splat
3086 /// constants or every element of a pair of constant BUILD_VECTORs.
3087 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
3088 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
3089 bool matchBinaryPredicate(
3090 SDValue LHS, SDValue RHS,
3091 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
3092 bool AllowUndefs = false, bool AllowTypeMismatch = false);
3093
3094 /// Returns true if the specified value is the overflow result from one
3095 /// of the overflow intrinsic nodes.
3096 inline bool isOverflowIntrOpRes(SDValue Op) {
3097 unsigned Opc = Op.getOpcode();
3098 return (Op.getResNo() == 1 &&
3099 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
3100 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
3101 }
3102
3103} // end namespace ISD
3104
3105} // end namespace llvm
3106
3107#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H
3108

source code of llvm/include/llvm/CodeGen/SelectionDAGNodes.h